diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4516ef8..4e0fd500 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null echo "deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main" | sudo tee /etc/apt/sources.list.d/llvm-23.list sudo apt-get update - sudo apt-get install -y --no-install-recommends clang-23 lld-23 libclang-rt-23-dev + sudo apt-get install -y --no-install-recommends clang-23 lld-23 llvm-23 libclang-rt-23-dev - name: Install GCC ${{ matrix.cxx }} if: matrix.cxx == 'g++-16' @@ -186,6 +186,9 @@ jobs: export PROJ_DATA="${{ github.workspace }}/build/alp_external/proj/share/proj" export PROJ_LIB="${PROJ_DATA}" export GDAL_DATA="${{ github.workspace }}/build/alp_external/gdal/share/gdal" + if [[ "${{ matrix.sanitizer }}" == "asan" ]]; then + export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-23 + fi if [[ "${{ matrix.sanitizer }}" == "tsan" ]]; then export TSAN_OPTIONS="suppressions=${{ github.workspace }}/extern/tbb/cmake/suppressions/tsan.suppressions" fi diff --git a/README.md b/README.md index e4af0bdb..011b8cd6 100644 --- a/README.md +++ b/README.md @@ -24,17 +24,24 @@ The tools typically only handle one tile per command which makes it infeasible t In this example, we'll show how to build the hierarchy for Vienna's city center (Zoom: 13, X: 4468, Y:2840). ### 1. Downloading tiles -The following command will download the basemap tiles from our mirror with the following format: -https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{Y}/{X}.jpeg +Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`. + +The `basemap` and `gataki` providers select their complete URL pattern and Y direction automatically. Both currently use downward Y with the URL order `{zoom}/{y}/{x}`. The `basemap` provider downloads the basemap.at orthophoto, and the Gataki mirror uses: + +`https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg` Example for the root tile: https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/13/2840/4468.jpeg -The tiles will be downloaded into the folder `./tiles/`. +For another service, pass a quoted `--url` pattern containing `{zoom}`, `{x}`, and `{y}`. Placeholder placement selects URL coordinate order. Custom URLs default to downward Y; use `--url-y-direction up` for legacy TMS. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`, independently of the remote URL. + +The following command downloads the mirror's `{zoom}/{y}/{x}` URLs and writes the root tile to `./tiles/13/4468/2840.jpeg`: ``` -./tile-downloader --provider gataki --zoom 13 --row 2840 --col 4468 --max-zoom-level 19 +./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --max-zoom-level 19 ``` +For example, a custom Google/Mapbox URL can be selected with `--url 'https://example.test/{zoom}/{x}/{y}.jpeg'`. + ### 2. Download heightmap dataset The meshes are built from a heightmap dataset, therefore we need to download one. For this example we'll only use a small part of the complete dataset for the whole of austria (available at https://gataki.cg.tuwien.ac.at/raw/Oe_2020/, 268 GB to 1.1 TB). The part we're gonna use contains Vienna's city center and it is available at https://gataki.cg.tuwien.ac.at/raw/vienna/innenstadt_gs_1m_mgi.tif (228 MB). @@ -84,4 +91,4 @@ In order to build, you need to install: - tbb (intel threading building blocks) sudo apt-get install libcgal-dev libopencv-dev libfmt-dev libglm-dev libgdal-dev catch2 libfreeimage-dev libtbb-dev libcurl4-openssl-dev -(libgmp-dev libmpfr-dev libsqlite3-dev) \ No newline at end of file +(libgmp-dev libmpfr-dev libsqlite3-dev) diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index a5c51da6..fdc85178 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -21,17 +21,25 @@ if(NOT COMMAND alp_setup_cmake_project) endif() function(alp_setup_gdal) - set(oneValueArgs GDAL_VERSION PROJ_VERSION) + set(oneValueArgs GDAL_VERSION GEOS_VERSION PROJ_VERSION) cmake_parse_arguments(ARG "" "${oneValueArgs}" "" ${ARGN}) - if(NOT ARG_GDAL_VERSION OR NOT ARG_PROJ_VERSION) - message(FATAL_ERROR "alp_setup_gdal() needs: GDAL_VERSION PROJ_VERSION ") + if(NOT ARG_GDAL_VERSION OR NOT ARG_GEOS_VERSION OR NOT ARG_PROJ_VERSION) + message(FATAL_ERROR "alp_setup_gdal() needs: GDAL_VERSION GEOS_VERSION PROJ_VERSION ") endif() alp_setup_cmake_project(proj URL https://github.com/OSGeo/PROJ.git COMMITISH ${ARG_PROJ_VERSION} CMAKE_ARGUMENTS -DBUILD_TESTING=OFF -DBUILD_APPS=OFF) find_package(PROJ CONFIG REQUIRED) - set(_proj_install "${ALP_PROJ_INSTALL_DIR}") + alp_setup_cmake_project(geos + URL https://github.com/libgeos/geos.git + COMMITISH ${ARG_GEOS_VERSION} + CMAKE_ARGUMENTS + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DGEOS_BUILD_DEVELOPER=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + ) alp_setup_cmake_project(gdal URL https://github.com/OSGeo/gdal.git @@ -40,6 +48,8 @@ function(alp_setup_gdal) -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DGDAL_ENABLE_DRIVER_HFA=ON -DOGR_BUILD_OPTIONAL_DRIVERS=OFF + -DOGR_ENABLE_DRIVER_GPKG=ON + -DOGR_ENABLE_DRIVER_SQLITE=ON -DBUILD_APPS=OFF -DBUILD_TESTING=OFF -DBUILD_PYTHON_BINDINGS=OFF @@ -47,6 +57,8 @@ function(alp_setup_gdal) -DBUILD_CSHARP_BINDINGS=OFF -DGDAL_USE_ICONV=OFF -DGDAL_USE_EXTERNAL_LIBS=OFF + -DGDAL_USE_GEOS=ON + -DGDAL_USE_SQLITE3=ON "-DCMAKE_INSTALL_RPATH=\$ORIGIN/../../proj/lib" ) diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md new file mode 100644 index 00000000..04429875 --- /dev/null +++ b/docs/raster-store/README.md @@ -0,0 +1,26 @@ +# Raster store design + +This directory describes a proposed authoritative raster store and the +generation of delivery tile pyramids from it. The documents are a design +baseline, not a finalized binary-format specification. + +## Documents + +- [Terminology](terminology.md) +- [Status quo and reuse assessment](status-quo.md) +- [Architecture](architecture.md) +- [Storage format](storage-format.md) +- [Sampling and pyramid generation](sampling-and-generation.md) + +## Plans + +- [Store refactoring plan](refactor-plan.md) +- [Raster store TODO](todo.md) +- [DRAFT RF builder plan archive](rf_builder.md) +- [DRAFT RF merger plan archive](rf_merger.md) + +## Scope + +The documents mostly hold format information. The `rf_builder` and +`rf_merger` documents are explicitly non-authoritative idea parking lots, not +tool specifications or implementation plans. diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md new file mode 100644 index 00000000..204c7ac7 --- /dev/null +++ b/docs/raster-store/architecture.md @@ -0,0 +1,154 @@ +# Architecture + +## System boundary + +The design separates authoritative data management from delivery generation: + +```text +Input rasters (GDAL) + │ + │ inspect, transform to Web Mercator, define source, one source per pixel -> rf_builder + ▼ +raster-fundamentalis (one rf per source at the beginning) + │ + │ rf_merger: merge two rf stores based on (vecrtor) mask, take one tile if no overlap / far from vector border, merge strategy otherwise + │ at the beginning, have xor strategy, and exact border, later we may implement linear blending + ▼ +Authoritative rf raster store + │ + │ generate overviews using defined filtering strategy, all area pixels. for now simple averaging, later maybe larger filter sizes / more complex filters + ▼ +tile-base store (one per layer, one per data version. user visible server should only need one version per layer) + ├── read by tile-server + ├── area/vertex pixel tile generation + └── select resolution, type etc by url +``` + +## Dataset organization + +### Proposed + +A store root contains immutable snapshots. Each snapshot contains an index, a source attribution table and the data: + +```text +store/ +└── snapshot-id/ + ├── source_attribution_table.ard + ├── raster_store.index + └── //.amort +``` + +The shown payload path is the default `zoom/x/y_google` layout. Other layouts +may map the same tile IDs differently; there is no mandatory `chunks/` +directory. + +## Sparse quadtree index + +The index uses the same four logical states as the octree index: + +| State | Chunk exists | Indexed descendants | +|-----------|-------------:|--------------------:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | +| `Missing` | no | no | + +`Missing` is represented by absence from the index, not serialized as an +entry. + +The index answers structural questions only. It does not claim that every +child exists and does not mark a virtual subtree as spatially complete. + +### Required operations + +- Look up a node without probing the filesystem. +- Add and remove physical nodes while maintaining virtual ancestors. +- Traverse only indexed branches. +- Enumerate physical descendants of a subtree. +- Find the nearest physical ancestor for fallback. +- Determine whether descendants may improve a requested output. +- Serialize and validate a versioned 2D topology. + +The last two operations may require aggregate metadata beyond +`Leaf/Inner/Virtual`, such as best descendant resolution or coverage. Such +metadata is an optimization and should be derivable from authoritative +entries. + +## Chunk model + +Each physical node owns one logical chunk, see storage-format.md + +## Source selection and fallback + +Source selection is a policy of the tools, not a property of the raster container. +Its initial comparison is expected to prioritize effective pixel resolution, and can be represented as a per zoom level or global ordered vector of attribution indices. + +At the beginning the selection will be binary, later we may introduce blending (over pixels of one zoom level, or several zoom levels). + +## Snapshot lifecycle + +A snapshot is never mutated, instead, operations build new snapshots, while reducing disk usage by using hard links: + +when adding data, we would: +1. create a new sf from the new data +2. define a validity mask for the new data +3. define a source merging priority (an ordering of sources) +4. using these two, a new snapshot is generated from the new data and the existing / authoritative snapshot, hardlinking tiles without change. + +Because hard links share inodes, a linked container must never be opened for +in-place modification. Existing snapshots are considered immutable. Obsolete snapshots can be deleted, the data will be preserved if necessary due to reference counting in the inodes. +When merging, we need to create new hardlinks for unchanged rf tiles (taken completely from either snapshot), and we need to create new rf tiles if the new tile shares information from both. + +### Publication + +A new snapshot is assembled in a sibling directory named +`.part`. Publication follows this protocol: + +1. Write all payload and metadata files into the `.part` directory. +2. Write the index last and validate the completed snapshot. +3. Flush and close every file. +4. Atomically rename the directory to `` on the same filesystem. + +The final destination must not already exist. A `.part` directory is +incomplete and is never considered published. The rename removes the suffix; +there is no separate marker or manifest. During normal operation this gives +readers atomic visibility: they see either no final snapshot or the completed +one. + +Cross-filesystem publication is unsupported because the final rename and any +hard links must remain on one filesystem. A builder or merger must reject that +configuration before starting a long operation. + +Publication does not guarantee durability or safe recovery across a power +failure, operating-system crash, or storage failure. Flushing and closing +files before the rename is required for normal-operation correctness, but is +not a crash-durability guarantee. The implementation does not require +`fsync()`, `fdatasync()`, `FlushFileBuffers()`, or equivalent +platform-specific synchronization. After such a failure, either a `.part` +directory or a final snapshot may be unusable and must be validated and +rebuilt. + +## Pyramid generator interface (to be confirmed, LLM, do not use the following without consultation) + +A generator requests a layer over a target tile and sampling specification. +The store reader supplies selected authoritative values and provenance over a +window large enough for the generator's filter support. + +The generator owns: + +- target output zoom and dimensions; +- vertex-pixel or area-pixel placement; +- low-pass/reconstruction filter; +- NoData normalization during filtering; +- colour-space and alpha treatment; +- border construction; +- output codec; and +- tile-level contributing-source metadata. + +The store owns: + +- chunk location and decoding; +- sparse hierarchy and physical fallback; +- exact stored source map; +- source catalog lookup; and +- consistent window access across chunk boundaries. diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md new file mode 100644 index 00000000..f9d3d9f0 --- /dev/null +++ b/docs/raster-store/before-refactor/README.md @@ -0,0 +1,433 @@ +# SF storage and merge architecture before the 2D/3D refactor + +This report describes the existing Structura Fundamentalis (SF) builder, +sparse octree index, storage stack, and mask-based dataset merger. Its purpose +is to identify the mechanisms that can be generalised from 3D octrees to a +shared 2D/3D hierarchy without carrying mesh- and ECEF-specific behaviour into +the common storage layer. + +The central conclusion is: + +> Generalise the sparse hierarchy, storage, traversal, merge decisions, and +> unchanged-subtree reuse. Keep GDAL ingestion, raster processing, mesh +> construction, spatial transforms, and mask geometry in dimension-specific +> adapters. + +The current storage templates are already payload-generic, but they are not +hierarchy-generic: `octree::Id` is embedded throughout indexing, traversal, +caches, disk paths, and serialization. + +## Current architecture + +```mermaid +flowchart TB + subgraph Applications + SFB[sf-builder] + SFM[sf-merger] + end + + subgraph Input + GDALR[GDAL raster datasource] + GDALV[GDAL vector mask] + OLD[Base SF dataset] + NEW[New SF dataset] + end + + subgraph Domain3D["3D and mesh policy"] + SPACE[octree::Space and ECEF bounds] + MESHBUILD[Height raster to SimpleMesh] + MESHMASK[Polygon to spherical extruded MeshMask] + MESHMERGE[Clip, combine, and texture atlas] + FALLBACK[Reconstruct child by clipping ancestor mesh] + end + + subgraph GenericCandidate["Mostly generalisable mechanisms"] + INDEX[IndexMap and NodeStatus] + WALK[Depth-first and breadth-first traversal] + DRIVER[Merger and merge result actions] + STORAGE[Storage and IndexedStorage] + RAW[RawStorage] + CODEC[Codec policy] + LAYOUT[Layout strategy] + LINK[Hard-link unchanged payload] + end + + GDALR --> SFB + SFB --> SPACE + SFB --> MESHBUILD + MESHBUILD --> STORAGE + + OLD --> SFM + NEW --> SFM + GDALV --> MESHMASK + MESHMASK --> SFM + SFM --> DRIVER + DRIVER --> FALLBACK + DRIVER --> MESHMERGE + DRIVER --> STORAGE + + STORAGE --> INDEX + STORAGE --> RAW + RAW --> CODEC + RAW --> LAYOUT + RAW --> LINK + WALK --> INDEX + DRIVER --> WALK +``` + +The significant code boundaries are: + +- hierarchy identity: [`octree::Id`](../../../src/terrainlib/octree/Id.h); +- sparse topology: [`IndexMap`](../../../src/terrainlib/octree/IndexMap.h); +- traversal: [`octree::traverse`](../../../src/terrainlib/octree/traverse.h); +- logical storage: + [`Storage_`](../../../src/terrainlib/octree/storage/Storage.h); +- raw file and hard-link storage: + [`RawStorage_`](../../../src/terrainlib/octree/storage/RawStorage.h); +- disk paths: [`disk::Layout`](../../../src/terrainlib/octree/disk/Layout.h); +- merge driver: [`Merger`](../../../src/sf_merger/merge.h). + +## Sparse index model + +The index has four logical states: + +| State | Physical payload | Indexed descendants | +|---|---:|---:| +| `Missing` | no | no | +| `Virtual` | no | yes | +| `Leaf` | yes | no | +| `Inner` | yes | yes | + +`Missing` is represented by absence from `IndexMap`. The other states are +defined by +[`NodeStatus`](../../../src/terrainlib/octree/NodeStatus.h). + +```mermaid +stateDiagram-v2 + [*] --> Missing + + Missing --> Leaf: add physical root or node + Missing --> Virtual: add deeper descendant + + Virtual --> Inner: add physical payload here + Leaf --> Inner: add physical descendant + + Inner --> Leaf: remove final descendant + Leaf --> Missing: remove payload and no descendants + Virtual --> Missing: remove final descendant + Inner --> Virtual: remove payload but keep descendants +``` + +This state machine is dimension-independent. What is currently 3D-specific is +how a node finds its parent and children: `octree::Id` uses three +Morton-interleaved coordinates and eight children. + +Traversal is also reusable in concept. It: + +1. looks up the root in the sparse index; +2. visits only entries that exist; +3. enumerates children through the concrete ID type; and +4. supports depth-first or breadth-first order plus a refinement predicate. + +The only reason +[`traverse()`](../../../src/terrainlib/octree/traverse.h) +is 3D is its dependency on `octree::Id` and `octree::Id::children()`. + +## Creating an SF dataset from a GDAL datasource + +The current SF builder creates physical mesh nodes at one requested octree +level. + +```mermaid +sequenceDiagram + participant CLI as sf-builder CLI + participant DS as Dataset and GDAL + participant Space as octree::Space + participant Build as terrainbuilder + participant Reader as RawDatasetReader + participant Raster as Raster and RasterMask + participant Mesh as mesh operations + participant Store as Storage + participant Index as IndexMap and terrain.index + + CLI->>DS: Open raster datasource + DS-->>CLI: SRS, 2D bounds, and height range + + CLI->>Space: Transform bounds to ECEF + Space-->>CLI: Smallest enclosing octree node + + loop Intersecting children until target level + CLI->>Space: Node bounds and intersection tests + end + + loop Each target-level node + CLI->>Build: build_patch using node bounds + Build->>Reader: Map SRS bounds to source pixels + Reader->>DS: RasterIO band 1 as float + Reader-->>Raster: Height raster + Raster->>Raster: Build NoData validity mask + Raster->>Mesh: Generate positions and triangle grid + Mesh->>Mesh: Clip to node volume + Mesh->>Mesh: Generate UVs and transform output SRS + Mesh-->>Store: SimpleMesh with optional texture + Store->>Store: MeshCodec writes node file + end + + Store->>Index: Scan output paths + Index->>Index: Add leaves and virtual ancestors + Index->>Store: Write terrain.index +``` + +### Current components + +| Responsibility | Current component | +|---|---| +| Own and open a GDAL datasource | [`Dataset`](../../../src/terrainlib/Dataset.h) | +| One-time GDAL registration | [`initialize_gdal_once()`](../../../src/terrainlib/init.cpp) | +| ECEF octree geometry | [`octree::Space`](../../../src/terrainlib/octree/Space.h) | +| Batch enumeration and orchestration | [`build_all_patches()`](../../../src/sf_builder/terrainbuilder.cpp) | +| Direct source-window reading | [`RawDatasetReader`](../../../src/sf_builder/raw_dataset_reader.h) | +| Height-to-mesh conversion | [`build_reference_mesh_patch()`](../../../src/sf_builder/mesh_builder.cpp) | +| Optional imagery | [`TileProvider`](../../../src/sf_builder/tile_provider.h) and [`texture_assembler.h`](../../../src/sf_builder/texture_assembler.h) | +| Mesh persistence | `Storage_` | +| Sparse topology | `IndexMap` | +| Index creation and serialization | `save_or_create_index()` and `terrain.index` | + +For a new output, `build_all_patches()` opens unindexed storage. Individual +saves therefore do not build the index incrementally. The final +`save_or_create_index()` recursively scans the output directory, parses node +paths, and calls `IndexMap::add()`. + +### More suitable 2D GDAL path + +The separate +[`DatasetReader`](../../../src/tile_builder/DatasetReader.h) +is closer to what a 2D SF or raster-store builder needs: + +- it accepts requested target-SRS bounds and output dimensions; +- it creates a GDAL warped VRT; +- it reprojects into the target grid; and +- it returns `radix::Raster`. + +[`Tiler`](../../../src/tile_builder/Tiler.h) and +[`ParallelTiler`](../../../src/tile_builder/ParallelTiler.h) +already enumerate `radix::tile::Id` values over dataset bounds. + +Their grid calculations are useful, but `ParallelTileGenerator` is a +delivery-file writer rather than an SF snapshot builder. + +## Mask-based SF dataset merging + +```mermaid +flowchart TD + START[Open base and new datasets as IndexedStorage] + MASK[Read vector mask through GDAL and OGR] + PREP[Polygon repair, sphere projection, triangulation, radial extrusion] + ROOT[Start at octree root] + STATUS[Read left and right NodeStatus] + POLICY[Masked visitor evaluates node] + RECURSE[Clip mask to node bounds and recurse into 8 children] + LEFT[Keep base subtree] + RIGHT[Keep new subtree] + MERGE[Clip base outside mask and new inside mask] + ATLAS[Combine meshes and rebuild texture atlas] + WRITE[Write new physical node] + COPY[Traverse unchanged source subtree] + LINK[Hard-link each physical payload] + INDEX[Scan and save new output index] + + START --> ROOT + MASK --> PREP --> POLICY + ROOT --> STATUS --> POLICY + + POLICY -->|virtual or refinement required| RECURSE + RECURSE --> STATUS + + POLICY -->|unchanged left| LEFT --> COPY + POLICY -->|unchanged right| RIGHT --> COPY + COPY --> LINK + + POLICY -->|boundary node| MERGE --> ATLAS --> WRITE + POLICY -->|no retained data| INDEX + + LINK --> INDEX + WRITE --> INDEX +``` + +### Merge components + +- [`Merger`](../../../src/sf_merger/merge.h) is the recursive dispatcher. +- [`NodeLoader`](../../../src/sf_merger/NodeLoader.h) reports status and loads + payloads. +- When an exact payload is missing, `NodeLoader` searches physical ancestors + and reconstructs the requested child by clipping the ancestor mesh. That + reconstruction is 3D-specific. +- [`NodeData`](../../../src/sf_merger/merge/NodeData.h) lazily exposes a node + payload to a visitor. +- [`Result`](../../../src/sf_merger/merge/Result.h) gives the driver four + actions: recurse, ignore, preserve one source unchanged, or write a merged + payload. +- [`Masked`](../../../src/sf_merger/merge/visitor/Masked.h) implements + mesh-and-mask policy. +- [`NodeWriter`](../../../src/sf_merger/NodeWriter.h) writes changed nodes or + copies unchanged subtrees. + +The vector-mask pipeline in +[`mask.h`](../../../src/sf_merger/mask.h) +is almost entirely 3D and Earth-specific after OGR polygon loading: + +```text +OGR polygons + -> referenced 2D polygon mask + -> ECEF and spherical projection + -> triangulated surface + -> extrusion over an Earth-radius interval + -> closed 3D MeshMask +``` + +For 2D raster merging, only the OGR polygon loading and CRS transformation +ideas remain relevant. Triangulation, spherical projection, extrusion, 3D +clipping, and texture atlases should not enter the generic core. + +## Hard-link behaviour + +Hard links are implemented at the lowest storage layer in +[`RawStorage_::copy_from()`](../../../src/terrainlib/octree/storage/RawStorage.h): + +```mermaid +flowchart TD + COPY[copy_from node] + EXISTS{Source payload exists?} + EXT{Source and target extensions match?} + DECODE[Decode source payload] + ENCODE[Encode into target format] + REMOVE[Remove existing target] + DIRS[Create parent directories] + HARDLINK[Create hard link] + DONE[Add physical node to target index] + + COPY --> EXISTS + EXISTS -->|no| ERROR1[FileNotFound] + EXISTS -->|yes| EXT + EXT -->|no| DECODE --> ENCODE --> DONE + EXT -->|yes| REMOVE --> DIRS --> HARDLINK --> DONE +``` + +Properties: + +- Hard linking is payload-agnostic and fully generalisable. +- It happens only when source and target filename extensions match. +- Different formats cause decode and re-encode through the codec. +- There is no copy fallback if hard-link creation fails, including across + filesystems. +- Snapshot immutability is not enforced by `Storage_`; it is a caller-level + convention. +- The existing target file is removed before the link is created. +- The new output receives a fresh index; `terrain.index` itself is not linked. + +## Generalisation assessment + +| Mechanism | Assessment | Required change | +|---|---|---| +| `NodeStatus` state model | Reuse essentially unchanged | Move outside `octree` naming | +| `IndexMap` state transitions | Generalise | Template over node key and tree traits | +| Sparse traversal | Generalise | Obtain children through tree traits | +| `Storage_` | Good starting point | Also template over node key and layout | +| `RawStorage_` hard-link behaviour | Generalise | Key-neutral paths and explicit fallback policy | +| Codec concept | Reuse | No dimensional dependency | +| Cache interface | Generalise | Key type is currently `octree::Id` | +| Layout strategy | Generalise | Key-neutral path API | +| Index file | Replace or version | Record topology kind, format version, and validated key | +| Merge result algebra | Generalise | `Merged` must hold generic payload, not `SimpleMesh` | +| Recursive merge driver | Generalise | Tree traits, payload, loader/writer, and policy | +| Unchanged-subtree reuse | Reuse | Enumerate every physical state, including `Inner` | +| GDAL datasource ownership | Reuse or adapt | Typed, multi-band reads and explicit NoData | +| Mask loading through OGR | Reuse or adapt | Produce dimension-specific mask representation | +| `octree::Space` | Keep 3D | Add a sibling 2D grid or space policy | +| Mesh clipping, atlas, and UV code | Keep 3D | Do not place in generic storage | +| ECEF and spherical mask conversion | Keep 3D | 2D uses polygon classification or rasterisation | +| `MeshCodec` | Keep 3D | Add a raster chunk codec or container | + +## Existing `Inner` limitation + +The index supports `Inner`, but the SF merger effectively does not: + +- `Merger::call_merge()` dispatches only `Missing`, `Leaf`, and `Virtual` + combinations. +- `Masked::visit()` declares `Inner` unreachable. +- `NodeWriter::copy_subtree_to_output()` skips `Virtual` and asserts every + other visited node is `Leaf`. + +This matters because a physical coarse node coexisting with physical +descendants is exactly the `Inner` case. A general 2D/3D implementation should +make "has a physical payload" and "has indexed descendants" independent +properties and handle all four states throughout traversal and merging. + +## Recommended target architecture + +```mermaid +flowchart TB + subgraph Core["Dimension-neutral hierarchical store"] + TRAITS["TreeTraits<Key>
root, parent, children, validation"] + SI["SparseIndex<Key>"] + TR["Traversal<Key>"] + ST["Storage<Key, Payload, Codec>"] + DL["DiskLayout<Key>"] + ME["MergeEngine<Key, Payload, Policy>"] + SNAP[Snapshot and subtree reuse] + end + + subgraph D2["2D adapter"] + TILE[radix::tile::Id] + GRID[Web Mercator grid] + RASTER[RasterChunk] + GDAL[GDAL warped-window reader] + MASK2[2D polygon or raster mask policy] + RC[Raster codec and container] + end + + subgraph D3["3D adapter"] + OCT[octree::Id] + ECEF[ECEF octree space] + MESH[SimpleMesh] + MASK3[Extruded mesh-mask policy] + MC[MeshCodec] + end + + TILE --> TRAITS + OCT --> TRAITS + + RASTER --> ST + MESH --> ST + RC --> ST + MC --> ST + + MASK2 --> ME + MASK3 --> ME + + SI --> TR + SI --> ST + DL --> ST + TR --> ME + ST --> SNAP + ME --> SNAP +``` + +The central abstractions should therefore be: + +1. `TreeTraits`: root, parent, children, maximum depth, and key + validation. +2. `SparseIndex`: the current `IndexMap` algorithm. +3. `Traversal`: DFS, BFS, and refinement independent of child + count. +4. `DiskLayout`: key-to-path and path-to-key conversion. +5. `Storage`: logical storage and index maintenance. +6. `MergeEngine`: paired sparse-tree walking. +7. Generic merge actions such as `Recurse`, `Ignore`, `KeepLeft`, `KeepRight`, + and `Write`. +8. A snapshot copier that hard-links unchanged physical nodes without knowing + their payload type. +9. Separate 2D and 3D spatial, mask, and fallback policies. + +The GDAL builder and mask merger should be clients of this core, not part of +it. diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh new file mode 100755 index 00000000..b919938c --- /dev/null +++ b/docs/raster-store/golden-e2e.sh @@ -0,0 +1,310 @@ +#!/usr/bin/env bash + +# Repeatable, non-unit end-to-end verification for the current 3D SF/DAG path. +# Inputs are prepared once under INPUT_ROOT. Completed steps are skipped through +# explicit state markers, so an interrupted SF or DAG run can be resumed. + +set -Eeuo pipefail + +readonly SOURCE_DIR="/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store" +readonly BUILD_DIR="${SOURCE_DIR}/build/golden-e2e" +readonly RUN_ROOT="/data/scratch/codex/alpine-terrain-builder-golden-e2e" +readonly INPUT_ROOT="${RUN_ROOT}/inputs" +readonly REFERENCE_ROOT="${RUN_ROOT}/reference" +readonly LOG_ROOT="${RUN_ROOT}/logs" +readonly STATE_ROOT="${RUN_ROOT}/state" +readonly RUN_ID="${RUN_ID:-$(date +%Y%m%dT%H%M%S)}" +readonly RUN_LOG_ROOT="${LOG_ROOT}/runs/${RUN_ID}" +readonly TIMINGS_FILE="${RUN_LOG_ROOT}/timings.tsv" + +readonly SF_BUILDER="${BUILD_DIR}/src/sf_builder/sf-builder" +readonly SF_MERGER="${BUILD_DIR}/src/sf_merger/sf-merger" +readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" + +# These VRTs are identical 4x4 km, native-resolution windows into the GS and +# GT rasters. They are recreated from the raw datasets at the start of a run. +readonly RAW_GS="/data/raw/raster_data/Oe_2020/OeRect_01m_gs_31287.img" +readonly RAW_GT="/data/raw/raster_data/Oe_2020/OeRect_01m_gt_31287.img" +readonly ELEVATION_ROOT="${INPUT_ROOT}/grossglockner-elevation-4km" +readonly GS_VRT="${ELEVATION_ROOT}/grossglockner-gs-4km.vrt" +readonly GT_VRT="${ELEVATION_ROOT}/grossglockner-gt-4km.vrt" +readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" +readonly GATAKI_TILES="${INPUT_ROOT}/gataki" +# A 0.01 m simplification preserves the Tirol border to sub-centimetre area +# accuracy while avoiding the exact geometry's pathological merge time. +readonly TIROL_MASK="${INPUT_ROOT}/tirol-boundary/benchmark-shape/0_01m/tirol.shp" + +readonly SF_ROOT="${REFERENCE_ROOT}/sf" +readonly DAG_ROOT="${REFERENCE_ROOT}/dag" +readonly SF_BASEMAP="${SF_ROOT}/grossglockner-basemap-gs-terrain" +readonly SF_GATAKI="${SF_ROOT}/grossglockner-gataki-gt-terrain" +readonly SF_MERGED="${SF_ROOT}/grossglockner-merged-terrain" +readonly DAG_MERGED="${DAG_ROOT}/grossglockner-merged-terrain" + +readonly SF_TARGET_LEVEL="${SF_TARGET_LEVEL:-15}" +readonly SF_THREADS="${SF_THREADS:-12}" +readonly MIN_TEXTURE_LEVEL=12 +readonly MAX_TEXTURE_LEVEL=19 + +mkdir -p "${SF_ROOT}" "${DAG_ROOT}" "${RUN_LOG_ROOT}" "${STATE_ROOT}" +exec > >(tee -a "${RUN_LOG_ROOT}/golden-e2e.log") 2>&1 + +timestamp() +{ + date --iso-8601=seconds +} + +run_timed() +{ + local name="$1" + shift + + local start_epoch end_epoch elapsed status + start_epoch="$(date +%s)" + printf '[%s] START %s\n' "$(timestamp)" "${name}" + + set +e + "$@" + status=$? + set -e + + end_epoch="$(date +%s)" + elapsed=$((end_epoch - start_epoch)) + printf '%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${start_epoch}" "${end_epoch}" "${elapsed}" "${status}" \ + >> "${TIMINGS_FILE}" + printf '[%s] END %s status=%s elapsed_seconds=%s\n' \ + "$(timestamp)" "${name}" "${status}" "${elapsed}" + + if ((status != 0)); then + return "${status}" + fi +} + +require_file() +{ + if [[ ! -f "$1" ]]; then + printf 'Required file is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +require_directory() +{ + if [[ ! -d "$1" ]]; then + printf 'Required directory is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +require_executable() +{ + if [[ ! -x "$1" ]]; then + printf 'Required executable is missing: %s\n' "$1" >&2 + exit 2 + fi +} + +verify_snapshot() +{ + local snapshot="$1" + local extension="$2" + local payload_count + + require_file "${snapshot}/terrain.index" + if [[ ! -s "${snapshot}/terrain.index" ]]; then + printf 'Index is empty: %s\n' "${snapshot}/terrain.index" >&2 + return 1 + fi + + payload_count="$(find "${snapshot}" -type f -name "*${extension}" | wc -l)" + if ((payload_count == 0)); then + printf 'No %s payloads found in %s\n' "${extension}" "${snapshot}" >&2 + return 1 + fi + printf 'Verified %s: %s payloads\n' "${snapshot}" "${payload_count}" +} + +build_sf() +{ + local name="$1" + local dataset="$2" + local textures="$3" + local output="$4" + local marker="${STATE_ROOT}/${name}.complete" + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP %s: completion marker exists\n' "$(timestamp)" "${name}" + verify_snapshot "${output}" ".terrain" + return + fi + + mkdir -p "${output}" + run_timed "${name}" \ + "${SF_BUILDER}" \ + --dataset "${dataset}" \ + --textures "${textures}" \ + --min-texture-level "${MIN_TEXTURE_LEVEL}" \ + --max-texture-level "${MAX_TEXTURE_LEVEL}" \ + --mesh-srs EPSG:4978 \ + --verbosity info \ + batch \ + --target-level "${SF_TARGET_LEVEL}" \ + --output "${output}" \ + --format .terrain \ + --threads "${SF_THREADS}" + + verify_snapshot "${output}" ".terrain" + touch "${marker}" +} + +merge_sf() +{ + local marker="${STATE_ROOT}/merge_sf.complete" + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP merge_sf: completion marker exists\n' "$(timestamp)" + verify_snapshot "${SF_MERGED}" ".terrain" + return + fi + + if [[ -d "${SF_MERGED}" ]] && [[ -n "$(find "${SF_MERGED}" -mindepth 1 -print -quit)" ]]; then + printf 'Partial merge output exists at %s.\n' "${SF_MERGED}" >&2 + printf 'The current sf-merger cannot safely resume this output; refusing to overwrite it.\n' >&2 + return 1 + fi + + mkdir -p "${SF_MERGED}" + run_timed merge_sf \ + "${SF_MERGER}" merge \ + --base "${SF_BASEMAP}" \ + --new "${SF_GATAKI}" \ + --mask "${TIROL_MASK}" \ + --output "${SF_MERGED}" \ + --verbosity info + + verify_snapshot "${SF_MERGED}" ".terrain" + touch "${marker}" +} + +build_dag() +{ + local marker="${STATE_ROOT}/build_dag.complete" + local continuation=(--overwrite) + + if [[ -f "${marker}" ]]; then + printf '[%s] SKIP build_dag: completion marker exists\n' "$(timestamp)" + verify_snapshot "${DAG_MERGED}" ".bin" + return + fi + + mkdir -p "${DAG_MERGED}" + if [[ -s "${DAG_MERGED}/terrain.index" ]]; then + continuation=(--resume) + fi + + run_timed build_dag \ + "${DAG_BUILDER}" \ + --input "${SF_MERGED}" \ + --output "${DAG_MERGED}" \ + "${continuation[@]}" \ + --verbosity info + + verify_snapshot "${DAG_MERGED}" ".bin" + touch "${marker}" +} + +write_manifest() +{ + local name="$1" + local snapshot="$2" + local output="${RUN_LOG_ROOT}/${name}.sha256" + + ( + cd "${snapshot}" + find . -type f ! -name terrain.index -print0 \ + | sort -z \ + | xargs -0 -r sha256sum + ) > "${output}" +} + +record_hard_links() +{ + local merged_count=0 + local linked_to_gataki=0 + local linked_to_basemap=0 + local newly_written=0 + local merged_file relative merged_inode candidate_inode + + while IFS= read -r -d '' merged_file; do + relative="${merged_file#"${SF_MERGED}/"}" + merged_inode="$(stat -c '%d:%i' "${merged_file}")" + ((merged_count += 1)) + + if [[ -f "${SF_GATAKI}/${relative}" ]]; then + candidate_inode="$(stat -c '%d:%i' "${SF_GATAKI}/${relative}")" + if [[ "${merged_inode}" == "${candidate_inode}" ]]; then + ((linked_to_gataki += 1)) + continue + fi + fi + + if [[ -f "${SF_BASEMAP}/${relative}" ]]; then + candidate_inode="$(stat -c '%d:%i' "${SF_BASEMAP}/${relative}")" + if [[ "${merged_inode}" == "${candidate_inode}" ]]; then + ((linked_to_basemap += 1)) + continue + fi + fi + + ((newly_written += 1)) + done < <(find "${SF_MERGED}" -type f -name '*.terrain' -print0) + + printf 'merged_payloads=%s\nlinked_to_gataki=%s\nlinked_to_basemap=%s\nnewly_written=%s\n' \ + "${merged_count}" "${linked_to_gataki}" "${linked_to_basemap}" "${newly_written}" \ + | tee "${RUN_LOG_ROOT}/merge-hard-links.txt" +} + +prepare_elevation_vrts() +{ + mkdir -p "${ELEVATION_ROOT}" + gdal_translate -q -of VRT -srcwin 259507 245043 4000 4000 "${RAW_GS}" "${GS_VRT}" + gdal_translate -q -of VRT -srcwin 259507 245043 4000 4000 "${RAW_GT}" "${GT_VRT}" + gdal_edit.py -units m "${GS_VRT}" + gdal_edit.py -units m "${GT_VRT}" +} + +require_file "${RAW_GS}" +require_file "${RAW_GT}" +run_timed prepare_elevation_vrts prepare_elevation_vrts +require_file "${GS_VRT}" +require_file "${GT_VRT}" +require_file "${TIROL_MASK}" +require_directory "${BASEMAP_TILES}" +require_directory "${GATAKI_TILES}" +require_executable "${SF_BUILDER}" +require_executable "${SF_MERGER}" +require_executable "${DAG_BUILDER}" + +{ + printf 'run_started=%s\n' "$(timestamp)" + printf 'git_commit=%s\n' "$(git -C "${SOURCE_DIR}" rev-parse HEAD)" + printf 'git_status=%q\n' "$(git -C "${SOURCE_DIR}" status --porcelain=v1 --branch)" + printf 'sf_target_level=%s\n' "${SF_TARGET_LEVEL}" + printf 'sf_threads=%s\n' "${SF_THREADS}" + printf 'min_texture_level=%s\n' "${MIN_TEXTURE_LEVEL}" + printf 'max_texture_level=%s\n' "${MAX_TEXTURE_LEVEL}" + printf 'cpu_count=%s\n' "$(nproc)" +} >> "${RUN_LOG_ROOT}/run-metadata.txt" + +build_sf build_sf_basemap_gs_terrain "${GS_VRT}" "${BASEMAP_TILES}" "${SF_BASEMAP}" +build_sf build_sf_gataki_gt_terrain "${GT_VRT}" "${GATAKI_TILES}" "${SF_GATAKI}" +merge_sf +record_hard_links +build_dag +run_timed manifest_sf_basemap write_manifest sf-basemap "${SF_BASEMAP}" +run_timed manifest_sf_gataki write_manifest sf-gataki "${SF_GATAKI}" +run_timed manifest_sf_merged write_manifest sf-merged "${SF_MERGED}" +run_timed manifest_dag_merged write_manifest dag-merged "${DAG_MERGED}" + +printf '[%s] Golden end-to-end run complete\n' "$(timestamp)" diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md new file mode 100644 index 00000000..f783e5d0 --- /dev/null +++ b/docs/raster-store/refactor-plan.md @@ -0,0 +1,1141 @@ +# 2D/3D hierarchical store refactor plan + +Status: proposal for review. This document is an implementation plan, not a +record of completed work. + +## Purpose + +This plan describes how to extract the existing octree-specific index, +traversal, storage, and codec code into a shared 2D/3D store. The refactor +must preserve existing 3D datasets and prove, using test-only mappings and +codecs where necessary, that the shared mechanisms work with a 2D key. +Persistent raster-fundamentalis formats, adapters, and tools are later work. + +## Decisions already made + +- The shared implementation will live in `src/terrainlib/store` and use the + `store` namespace. +- The minimal 2D traits adapter used to exercise the shared hierarchy will + live in `src/terrainlib/raster_store` and use the `raster_store` namespace. +- Common Structura Fundamentalis validation and errors will live in + `src/terrainlib/sf` and use the `sf` namespace. +- Existing 3D Structura Fundamentalis datasets must remain readable and + writable without changing their on-disk contract. +- DAG datasets written by the current code when Phase 0 begins must remain + readable and writable throughout the refactor without changing their index + or payload serialization. Older DAG payload schemas are not supported. +- `Inner` is a valid shared topology state and is supported by DAG, + raster-fundamentalis, and tile-base datasets. It is not valid in Structura + Fundamentalis datasets. +- SF producer and processing boundaries validate indexed SF data and return a + typed `sf::InvalidTopology` error containing an offending key when `Inner` + is present. The diagnostic `sf_index_browser` is exempt. This is an SF data + invariant, not a generic store or octree-format rule. +- 3D compatibility includes both existing path layouts: + `flat` and `level_and_coordinate_directories`. +- A persistent 2D raster tile format is not defined by this refactor. + [architecture.md](architecture.md) and + [storage-format.md](storage-format.md) describe intended direction and + provisional requirements that will be finalized in later RF work. They must + not be retrofitted onto existing 3D datasets. +- A path layout strategy should contain a stable identifier and two + operations: key to extensionless `NodePath`, and `NodePath` to key. It + should not require an inheritance hierarchy, RTTI, global + self-registration, or heap allocation. +- A configured codec owns all filename endings and maps one `NodePath` to one + or more physical files. `Codec::paths()` must not need a payload. +- Codecs are stateful runtime objects behind a small interface. They may + support reading, writing, or both. Reading and writing return + `std::expected`; unsupported operations and other operational failures are + reported as error values. Reading and writing must be reentrant (callable + concurrently from different threads). +- The runtime glTF codec catches exceptions from the existing glTF mesh writer + and converts them to `CodecError`; changing the mesh I/O API is out of scope. +- `store::RawStorage` exclusively owns the configured + `std::unique_ptr>`. `store::Storage` owns the raw + storage and therefore owns the codec transitively. Storage consumers do not + receive a codec template parameter and application call sites do not manage + codec objects. +- This refactor does not add synchronization to storage, indexes, or caches + and does not change their concurrency guarantees. Existing caller-side + synchronization and concurrency behaviour are preserved; any concurrency + bug fix is separate work. +- Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around + DAG output storage. DAG storage remains cacheless while shared-lock reads are + used. +- Legacy index metadata selects a codec through an explicit, caller-supplied + resolver supplied by the payload-domain opening function. The octree format + adapter does not contain a global codec registry or depend on mesh or DAG + payload types. Application-level storage consumers do not call the resolver + or handle the resulting codec object. +- Dimension-specific index persistence uses a small runtime + `store::IndexFormat` value containing ordinary function pointers. It + is not an inheritance hierarchy and does not use global registration. +- `copy_from()` hard-links every file when the input and output codecs return + the same path list for a common dummy `NodePath`. Otherwise it decodes with + the input codec and encodes with the output codec. +- Public store operations reject invalid hierarchy keys through + `std::expected`; invalid keys are not represented by assertions or generic + booleans. +- Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; + rejected overwrites return `AlreadyExists` through `std::expected`, and + enabling it preserves the existing DAG-builder overwrite and debug-export + behaviour. +- The existing cache implementations are currently non-functional. Port their + public API where it remains useful, keep application call sites building, and + provide compile coverage only. Cache behaviour is not a compatibility + requirement of this refactor. +- The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an + unmanaged debug artifact. It is not part of a logical node, is not returned + by `Codec::paths()`, and is not indexed or copied by storage. Its existing + application-local behaviour is preserved. +- 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, + and raster-specific processing remain outside the shared store. + +## Goals + +1. Use one sparse hierarchy implementation for `octree::Id` and + `radix::tile::Id`. +2. Make traversal, storage, cache, and the runtime codec boundary + dimension-neutral while keeping production format adapters 3D-only in this + refactor. +3. Replace the current layout-strategy class hierarchy with small path-mapping + values backed by function pairs. +4. Allow one logical node payload to consist of multiple files without making + layouts aware of those files. +5. Preserve all valid existing 3D index files and payload paths. +6. Prove the shared topology and traversal with `radix::tile::Id` without + defining a persistent 2D adapter or format. +7. Land the refactor in small, testable steps. Every phase should build and + pass tests before the next phase begins. + +## Non-goals + +- Changing `octree::Id`, `octree::Space`, `IdRect`, `OddLevelShifted`, or + other 3D spatial calculations. +- Defining or implementing GDAL ingestion, raster resampling, filtering, + source selection, or mask rasterisation. +- Defining or implementing a raster-fundamentalis index, payload format, + layout ID, codec, publication lifecycle, or persistent storage adapter. +- Implementing an `rf_builder`, `rf_merger`, tile-base generator, tile server, + snapshot-reuse operation, or other RF tool in this refactor. +- Defining the raster-fundamentalis merge policy or the final paired-hierarchy + walker/action algebra. That work is deferred until `rf_merger` requirements + are defined. +- Defining a shared subtree-copy abstraction, `Inner` subtree-copy behaviour, + or forced re-encoding policy for RF. Those decisions are deferred until + `rf_merger`. +- Adding merge semantics for `Inner` nodes in SF. Such nodes are invalid SF + input and must be rejected before merge dispatch. +- Changing the existing 3D hard-link policy by adding a silent file-copy + fallback. +- Refactoring unrelated octree, DAG, mesh, or tile-builder code. +- Changing the serialized schema of the current DAG `.bin` payloads. + +## Compatibility contract + +Before moving code, tests must lock down the following 3D behaviour: + +| Item | Required compatibility | +|---|---| +| Index filename | `terrain.index` | +| Index field order | layout ID, preferred extension, index map | +| Node-key encoding | existing `octree::Id` level/index serialization | +| Node-status encoding | `Leaf = 0`, `Inner = 1`, `Virtual = 2` | +| Valid SF statuses | `Leaf` and `Virtual`; reject `Inner` with `sf::InvalidTopology` | +| Valid DAG statuses | `Leaf`, `Inner`, and `Virtual` | +| Flat layout ID | `flat` | +| Flat path | `-` | +| Coordinate layout ID | `level_and_coordinate_directories` | +| Coordinate path | `///` | +| Default layout | existing level/coordinate layout | +| Layout detection | both existing layouts remain detectable | +| Mesh codec selection | legacy preferred extension selects terrain or configured glTF codec | +| DAG codec selection | legacy `.bin` preferred extension selects the ZPP Bits codec | +| DAG payload encoding | current `dag::ClusterBatch` ZPP Bits serialization: metadata, then clustering | +| Equal codec path lists | hard-link every file, or report an explicit error | +| Different codec path lists | decode with input codec and encode with output codec | +| Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | + +Compatibility means that each refactor phase can open the Phase 0 fixtures +written by the current code. DAG formats older than the Phase 0 baseline and +pre-refactor readers opening post-refactor output are not tested. Exact +byte-for-byte rewriting of an unordered index map is not required, but the +serialized schema and values must remain compatible during the migration. + +The 3D index disk type should remain a versioned 3D adapter. The shared store +must not add a topology field, new header, checksum, or compression layer to +`terrain.index`. + +## Proposed source boundary + +```text +src/terrainlib/ +├── store/ +│ ├── NodeStatus.h +│ ├── NodeStatusOrMissing.h +│ ├── Traits.h +│ ├── InvalidKey.h +│ ├── Index.h +│ ├── traverse.h +│ ├── NodePath.h +│ ├── PathMapping.h +│ ├── Layout.h +│ ├── IndexFormat.h +│ ├── Codec.h +│ ├── CodecError.h +│ ├── OpenError.h +│ ├── CopyError.h +│ ├── StorageSettings.h +│ ├── RawStorage.h +│ ├── Storage.h +│ ├── IndexedStorage.h +│ ├── cache/ +│ │ ├── Interface.h +│ │ ├── Dummy.h +│ │ └── Lru.h +│ └── codec/ +│ └── ZppBits.h +├── mesh/ +│ └── codec/ +│ ├── Terrain.h +│ └── Gltf.h +├── sf/ +│ ├── InvalidTopology.h +│ └── validate_index.h +├── octree/ +│ ├── Id.h +│ ├── StoreTraits.h +│ ├── store_layout/ +│ │ ├── Flat.h +│ │ ├── LevelAndCoordinateDirectories.h +│ │ └── Mappings.h +│ └── storage/ +│ ├── IndexFile.h +│ └── open.h +└── raster_store/ + └── StoreTraits.h +``` + +The exact file grouping may be collapsed if a file would only contain a few +lines. The important boundaries are: + +- `store` contains dimension- and payload-neutral mechanisms; +- `store::codec::ZppBits` is the reusable concrete codec for payload types + that provide ZPP Bits serialization; +- `mesh::codec` contains the separately configured terrain and glTF codecs; +- `sf` contains SF-specific topology validation and errors shared by + `sf_builder`, `sf_merger`, and `dag_builder`; +- `octree` contains the 3D format and key adapters; +- `raster_store` contains only the minimal 2D hierarchy traits adapter in this + refactor; and +- subdirectory names match their namespaces where a subnamespace is used. + +Shared topology and octree-format adapters accept `Inner`. The SF restriction +is enforced by `sf::validate_index()` at SF producer/consumer boundaries and +reported as `sf::InvalidTopology`. It must not be embedded in `store::Index`, +traversal, or the generic 3D disk adapter. `sf_index_browser` is a diagnostic +tool and intentionally does not apply SF validation, so it can display invalid +trees including `Inner`. + +`sf::validate_index()` returns +`std::expected`. SF application-level error types +must retain this error and `CopyError` when propagating failures; neither is +reduced to a log message, assertion, or generic boolean. + +For this refactor, `sf::validate_index()` checks only for `Inner`. It does not +validate other structural or disk/filesystem invariants. Possible future +extensions are recorded in [todo.md](todo.md). + +Temporary forwarding headers and aliases under `octree` are allowed during +migration. They must not contain a second implementation. + +DAG serialization remains owned by `dag_builder`. Consolidate the serializers +for `dag::Id`, `dag::Group`, `dag::NodeMetadata`, `dag::ClusterBatch`, +`radix::geometry::Aabb3d`, GLM vectors, `Clustering`, `Cluster`, and +`TextureSet` in `src/dag_builder/serialization.h`. The DAG storage adapter +includes that header explicitly so template instantiation does not depend on +caller include order. Preserve the current tuples exactly: + +- `ClusterBatch`: metadata, clustering; +- `NodeMetadata`: group assignment, groups; and +- `Group`: children, error, bounds. + +`Group::child_errors` is currently neither populated nor serialized. It remains +non-persistent in this refactor and the Phase 0 fixture locks down its omission. + +## Shared interfaces + +The names below are the intended shape, not signatures that must be copied +verbatim without testing. + +### Hierarchy traits + +The store should be parameterized by one traits type rather than assuming that +all key classes expose identical member functions: + +```cpp +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + { Traits::parent(key) }; + { Traits::children(key) }; + { Traits::is_valid(key) } -> std::same_as; +}; +``` + +The concrete names should be: + +```cpp +store::Index +store::Index +``` + +`octree::StoreTraits` adapts the existing optional parent/children API without +changing `octree::Id`. + +`raster_store::StoreTraits` adapts `radix::tile::Id` and must: + +- treat zoom zero as the only root; +- never call `radix::tile::Id::parent()` at zoom zero, where it underflows; +- reject coordinates outside `[0, 2^zoom)`; +- accept zoom levels 0 through + `std::numeric_limits::digits`, inclusive; +- treat that maximum zoom as terminal because a child cannot be represented + by the `uint32_t` x/y coordinates; +- validate the maximum zoom without evaluating an overflowing + `uint32_t{1} << 32`; +- use `radix::tile::Id::Hasher`; and +- return children in the deterministic order produced by + `radix::tile::Id::children()`. + +This traits adapter defines only hierarchy operations. It does not define +persistent coordinates, a path layout, or an RF disk format. + +The shared code must obtain roots, parents, children, validation, and hashing +through the traits. It must not use dimension checks or specialize behaviour +on key types internally. + +Operations accepting a key validate it through `Traits::is_valid()`. Index +lookup and mutation, traversal with an explicit root, and storage operations +return an `std::expected` retaining an `InvalidKey` when validation fails. +Keys produced internally by `Traits::root()`, `Traits::parent()`, and +`Traits::children()` are trusted only after trait-specific tests establish that +they preserve validity. The child order affects traversal order and is locked +down by the 2D and 3D trait tests; it is not serialized as separate metadata. + +The API result shapes are: + +- index lookup returns `expected, InvalidKey>`; +- index mutation and predicates return `expected>`; +- traversal returns `expected>`; and +- storage `load`, `save`, `has`, `remove`, path lookup, and `copy_from` return + operation-specific `expected` types which retain `InvalidKey` and any + applicable codec, filesystem, missing-source, or overwrite error. + +During Phase 1, a forwarding `octree::IndexMap` compatibility wrapper preserves +the old optional/bool API for existing 3D callers with already-valid +`octree::Id` values. It delegates to `store::Index`, is not +a second implementation, and is removed after callers migrate. + +### Sparse index and traversal + +Move the existing four-state model to: + +```cpp +store::NodeStatus +store::NodeStatusOrMissing +store::Index +store::traverse(index, visitor, refine, root, order) +``` + +The index algorithm remains the current one: + +- adding a physical descendant creates virtual ancestors; +- adding a payload to a virtual node makes it `Inner`; +- removing a payload from an `Inner` node makes it `Virtual`; +- removing the final descendant collapses virtual ancestors; and +- a physical parent becomes `Leaf` after its last descendant is removed. + +Traversal must follow only indexed nodes and use `Traits::children`. Child +order is the order supplied by the traits and is therefore deterministic per +hierarchy, not universally fixed by `store`. + +### Node paths and path mappings + +`store::NodePath` is an extensionless logical location for one hierarchy +node. For example: + +```text +octree flat 12-123456 +octree coordinates 12/34/56/78 +``` + +It does not necessarily name a physical file. Replace +`octree::disk::layout::Strategy` and +`octree::disk::layout::StrategyRegister` with a value similar to: + +```cpp +template +struct store::PathMapping { + std::string_view id; + NodePath (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const NodePath&); +}; +``` + +`store::Layout` owns the base directory and one `PathMapping`. It +does not own a preferred extension. A configured codec expands the +extensionless `NodePath` into the physical file or files. + +The stable ID is format metadata, not a third strategy operation. The +dimension adapters provide ordinary lookup functions: + +```cpp +octree::store_layout::flat() +octree::store_layout::level_and_coordinate_directories() +octree::store_layout::from_id(id) +octree::store_layout::all() +``` + +This retains runtime selection from an index file while removing virtual +dispatch, RTTI type-to-ID lookup, static registration, and ownership through +`unique_ptr`. + +Path parsers validate the complete logical `NodePath`, not a file ending. +Codec or format-adapter code removes and validates physical file endings +before asking the layout to recover a key. Invalid disk input returns an error +or `nullopt`; it must not trigger an assertion. + +### Codec interface + +A codec is a configured runtime object for one logical payload type. It owns +all physical filename endings and may map one `NodePath` to several files: + +```cpp +template +class store::Codec { +public: + virtual ~Codec() = default; + + virtual std::vector + paths(const NodePath& node_path) const = 0; + + virtual std::expected + read(const NodePath&) const { + return std::unexpected( + CodecError::unsupported_operation("read")); + } + + virtual std::expected write( + const NodePath&, + const NodeData&) const { + return std::unexpected( + CodecError::unsupported_operation("write")); + } +}; +``` + +Concrete codecs contain their configuration and are constructed before +storage use. `CodecError` is a payload-neutral operational error that records +the failed operation, an error category, and a diagnostic message. Concrete +codecs convert their domain errors to it. Unsupported read or write operations +may use the base implementation and return its `UnsupportedOperation` error. +Codec writes preserve the current directory-creation behaviour: they create +the parent directories required by their output paths before writing. The +storage hard-link path continues to create its target parent directories before +linking. + +`Codec::paths()` has the following contract: + +- it needs no NodeData payload and performs no filesystem access; +- it returns every physical file belonging to the logical node; +- results depend only on codec configuration and the supplied `NodePath`; +- result order is stable and pairs corresponding input/output files; +- two codecs returning the same path list for the same `NodePath` must produce + mutually compatible files; and +- different artifact counts or filename endings produce different lists. + +Examples: + +```text +Terrain codec + 12/34/56/78 + -> 12/34/56/78.terrain + +glTF codec configured for binary output + 12/34/56/78 + -> 12/34/56/78.glb + +glTF codec configured for JSON output + 12/34/56/78 + -> 12/34/56/78.gltf + +Multi-file test codec + 12/34/56/78 + -> 12/34/56/78.data + -> 12/34/56/78.metadata +``` + +The mesh side has separate terrain and glTF codecs because they use different +format implementations. Binary `.glb` and JSON `.gltf` remain configurations +of one glTF codec because both use the same `cgltf` implementation. + +The shared module also provides: + +```cpp +template +struct store::codec::ZppBits : store::Codec { .. }; +``` + +It uses the existing `io::read_from_path()` and `io::write_to_path()` +functions, maps one node to `.bin`, and converts `io::Error` to +`CodecError`. It contains no DAG-specific serialization logic. DAG payload +serialization remains in `dag_builder/serialization.h`, and its field order +and meshoptimizer/JPEG encoding remain unchanged. + +### Storage and format adapters + +Generalize storage over traits and NodeData. It owns a configured codec through +the runtime interface: + +```cpp +store::RawStorage +store::Storage +store::IndexedStorage +store::cache::Interface +``` + +`RawStorage` owns the +`std::unique_ptr>`. `Storage` owns `RawStorage`, and +`IndexedStorage` owns or derives from `Storage`; no other layer shares codec +ownership. Moving storage transfers ownership. Caches, layouts, index formats, +resolvers, and application consumers never own the codec. + +A move disarms the source so its destructor cannot save moved-out index state. +Move assignment must finalize a displaced dirty destination through the +existing destructor-save policy rather than silently discard it. Tests cover +the DAG builder's move into and release from `dag::ThreadSafeStorage`. + +Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits +codec remains under `store::codec`. RF codecs are deferred. + +Index serialization is not a responsibility of `store::Index`. Opening and +saving a dataset receives the following small runtime values: + +```cpp +template +struct store::IndexMetadata { + store::Index index; + std::string layout_id; + std::string codec_selector; +}; + +template +struct store::IndexFormat { + std::string_view index_filename; + + std::expected, IndexFormatError> + (*read)(const std::filesystem::path& index_path); + + std::expected + (*write)( + const std::filesystem::path& index_path, + const IndexMetadata& metadata); + + std::optional> + (*mapping_from_id)(std::string_view id); + + PathMapping + (*default_mapping)(); +}; +``` + +The value provides: + +- the index filename; +- index read/write conversion; +- mapping lookup by stable ID; +- the default mapping. + +Legacy directory discovery is an octree opening helper which composes the +octree index format, the supplied payload-domain codec resolver, and the known +octree mappings. It is not a generic `IndexFormat` operation. Neither the +format value nor discovery may reintroduce a layout class hierarchy or global +registration. + +For 3D, the adapter reads and writes the current `octree` index DTO unchanged. +Its `codec_selector` is exactly the legacy `preferred_extension`, including +the leading dot. Storage retains the selected `IndexFormat`, index path, +layout ID, and codec selector as its index-persistence state, so explicit and +destructor-triggered index saves can reproduce the same metadata. + +When opening indexed storage or discovering a legacy unindexed directory, it +passes the legacy `preferred_extension` to a caller-supplied codec resolver. +The resolver is an ordinary callable and returns +`std::expected>, CodecError>`. It is not +a global registry. + +The payload domains provide ordinary resolver functions: + +```text +mesh::codec::from_extension + .terrain -> terrain codec + .glb -> glTF codec with binary container + .gltf -> glTF codec with JSON container + +dag::codec::from_extension + .bin -> store::codec::ZppBits + +dag::codec::metadata_from_extension + .bin -> read-only dag::codec::MetadataView +``` + +An unknown extension returns an explicit `UnsupportedCodec` error. Opening a +new empty store receives an explicit legacy codec selector plus an already +constructed codec at the payload-domain opening boundary; the generic storage +does not infer persistent metadata from `Codec::paths()`. Mesh and DAG +convenience functions select or resolve the codec and pass ownership into raw +storage, so application storage consumers do not handle codec objects. +Convenience functions in `src/dag_builder/storage.h` supply the DAG resolvers. +Preserve `DagMetaStorage` and `IndexedDagMetaStorage` as independently readable +views of the metadata prefix in each `ClusterBatch` `.bin` file. Their codec +returns `UnsupportedOperation` from writes instead of terminating or replacing +a full batch with metadata alone. `dag::codec::MetadataView` implements this +domain-specific read-only prefix view over the generic ZPP Bits codec. + +Opening functions return their requested storage type through +`std::expected<..., OpenError>`. `OpenError` is a typed sum which retains the +failing path and the underlying error where applicable: + +- index I/O or malformed index metadata (`IndexFormatError`); +- filesystem failure; +- unknown layout ID; +- unsupported codec selector or codec construction failure (`CodecError`); and +- invalid hierarchy key (`InvalidKey`). + +Loading and saving likewise return storage-level expected errors which retain +an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected +save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, +and codec failures. The SF call chains changed in Phase 4 retain these errors +to their application boundary. DAG callers preserve their existing per-node +log-and-continue and command-line policies; making the DAG builder fail-fast is +not part of this refactor. + +Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes +candidate endings by asking the supplied resolver, removes an accepted ending +to obtain a `NodePath`, and then invokes the selected layout parser. The +generic layout does not recover keys directly from codec-owned file paths. +Only a missing index triggers this discovery path. An unreadable or malformed +index, unknown layout, or unsupported codec selector returns `OpenError` and +must not silently fall back to directory discovery. + +Automatic dirty-index saving currently happens in the 3D storage destructor. +Preserve that behaviour for existing 3D entry points during the migration. + +### Copying one node and SF subtree reuse + +There are two separate responsibilities in the current implementation: + +1. `sf_merger::NodeWriter` traverses a source subtree and + `sf_merger::cut_leaf_node()` identifies an unchanged leaf. +2. `octree::Storage::copy_from()` delegates to + `octree::RawStorage::copy_from()`, where + `std::filesystem::create_hard_link()` performs the actual hard link and the + target index is updated on success. + +The filesystem hard-link implementation is therefore already in terrainlib. +Move the one-node storage operation into the shared store, but keep subtree +selection and traversal in `sf_merger`. There is no current DAG caller, and RF +subtree-copy requirements will be defined with `rf_merger`. + +The migrated call chains are: + +```text +sf_merger decides to keep a source subtree unchanged + -> NodeWriter traverses the source index + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode + +sf_merger determines that a cut leaf is unchanged + -> cut_leaf_node() + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode +``` + +`Storage::copy_from()` remains the operation for copying one logical node. +For one key, `copy_from()`: + +1. calls the input and output `Codec::paths()` with the same fixed dummy + `NodePath`; +2. when the lists are equal, calls both codecs again with their actual source + and target `NodePath` values and hard-links every source path to the + corresponding target path; +3. when the dummy lists differ, reads the payload with the input codec and + writes it with the output codec; and +4. updates the target index only after all links or the write complete. + +The dummy path must be fixed and collision-free, for example +`__codec_probe__/node`. Path lists are compared exactly, including count, +order, and filename endings. + +If overwrite is enabled for an indexed target, remove the target index entry +immediately before the first target file is modified. If linking several files +then fails partway through, return the error without a transactional rollback +guarantee; target links or old files may remain, but the logical node stays +unindexed. The copy operation stops immediately and propagates the failure +until the application aborts the overall operation. There is no journal, +rollback, cleanup guarantee, or silent copy fallback. An unsupported read or +write needed for re-encoding is returned through `CopyError`, retaining the +underlying `CodecError`. + +Hard-link rules: + +- never modify an existing linked payload in place; +- a matching codec path list hard-links every file; +- a different path list decodes with the input codec and encodes with the + output codec; +- hard-link failure is explicit; +- no silent file-copy fallback is introduced. + +#### SF-local subtree traversal + +`sf_merger::NodeWriter::copy_subtree_to_output()` remains in `sf_merger`. It +uses shared traversal and `Storage::copy_from()`, but it is not promoted to a +generic store API during this refactor. + +SF validation guarantees that its indexed inputs contain only `Leaf` and +`Virtual` nodes. The SF-local traversal skips `Virtual`, copies `Leaf`, and +does not define behaviour for `Inner`. An `Inner` node is rejected before +merge or cut processing with `sf::InvalidTopology` containing the offending +key. + +When `rf_merger` is designed, it can initially compose `store::traverse` and +`Storage::copy_from()`. At that point, the SF and RF implementations provide +enough evidence to decide whether a shared subtree copier is useful and how it +must handle RF `Inner` nodes. + +#### Error propagation + +The lower storage layer already represents ordinary copy failures, including +missing source files, directory creation failure, hard-link failure, decode +failure, and encode failure. The current `NodeWriter` consumes +`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, as does the unchanged-leaf path +in `cut_leaf_node()`, while some overwrite paths terminate through +`LOG_ERROR_AND_EXIT()`. Their `void` call chains prevent the application from +reporting or handling these failures. + +Return failures from the SF-local subtree and cut functions through their +callers until the application boundary can report the affected key and path. +Codec, filesystem, unsupported-conversion, malformed-dataset, and overwrite +failures are propagated with `std::expected`; `CopyError` retains any +underlying `CodecError`. Assertions remain appropriate for internal +invariants, but operational failures are not assertion failures or +intentionally thrown exceptions. + +### Paired hierarchy walking is deferred + +This refactor does not extract `sf_merger::Merger` into a shared paired-tree +walker. SF only permits `Missing`, `Leaf`, and `Virtual`, and its current +recursion does not provide enough evidence to define the `Inner` behaviour +needed by raster-fundamentalis and tile-base merging. + +The previously proposed mutually exclusive actions `Recurse`, `Ignore`, +`KeepLeft`, `KeepRight`, and `Write` cannot express both an action for the +current physical payload and recursion into descendants. `Inner` merging may +require both. Whether the future interface uses a combined `WriteAndRecurse` +action or independent current-node and descendant decisions belongs to the +`rf_merger` design. + +For this refactor: + +- keep recursion, subtree traversal, and mesh policy in `sf_merger`; +- validate SF inputs and reject `Inner` through `std::expected`; +- move only the one-node `Storage::copy_from()` mechanism into `store`; and +- do not add a shared `store::merge` namespace. + +A future `rf_merger` task will define the paired-tree action algebra from the +2D requirements and may migrate `sf_merger` once both use cases are known. + +## Implementation phases + +Each phase should be one reviewable commit unless the tests and implementation +are clearer as two commits. Do not begin a later phase while the current phase +has failing tests. + +### Phase 0 — Capture current compatibility + +No production behaviour changes. + +1. Add golden SF fixtures created by the current code: + - one `terrain.index` using `flat`; + - one using `level_and_coordinate_directories`; + - across the fixtures, physical payload paths for a root, child, and deeper + descendant, without placing physical payloads at ancestor and descendant + keys in the same index; and + - index entries containing `Leaf` and `Virtual`, but no `Inner`. +2. Add one golden DAG dataset whose index selects `.bin`, whose payload + contains a valid serialized `dag::ClusterBatch`, and whose index contains + `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering + format, include non-trivial group metadata, and prove that the same file can + be opened through the read-only `NodeMetadata` view. Lock down that + `Group::child_errors` is not serialized. +3. Test that all fixtures open, resolve the expected IDs and extensions, and + traverse the expected sparse nodes. +4. Add path round-trip tests for boundary IDs and both layouts. +5. Add storage tests for: + - matching-extension hard links; + - different-extension decode/re-encode; + - overwrite-enabled replacement; + - indexed and unindexed opens; and + - final index creation by directory scan. +6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, + `sf_index_browser`, `dag_builder`, and `dag_convert_debug`, including + `DagMetaStorage`, `IndexedDagMetaStorage`, and `dag::ThreadSafeStorage`. + +Exit criterion: the compatibility tests pass against the untouched +implementation and fail when any stable filename, layout ID, path encoding, +status value, index field order, or DAG payload serialization is deliberately +changed. + +### Phase 1 — Extract topology into `store` + +1. Move `NodeStatus` and `NodeStatusOrMissing` to `store`, preserving their + underlying values and serialization. +2. Introduce the hierarchy-traits concept and `octree::StoreTraits`. +3. Convert `IndexMap` into `store::Index`. +4. Convert traversal into `store::traverse`. +5. Add `raster_store::StoreTraits` for `radix::tile::Id`. +6. Run the same index-transition and DFS/BFS tests with both trait types, + including deterministic child order, invalid-key errors through + `std::expected`, maximum-depth children, and explicit traversal roots. +7. Provide the temporary forwarding `octree::IndexMap` wrapper and aliases so + downstream migration is separate; otherwise migrate downstream immediately. + +Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D +callers still build through aliases; no filesystem code has changed. + +### Phase 2 — Introduce path mappings and runtime codecs + +This phase introduces the extensionless layout and runtime codec pieces +together. It does not cut production storage over to them yet. The existing +storage, layout strategies, and static codecs remain temporarily as the +working compatibility path until Phase 3 can replace the complete +layout-plus-codec path construction in one step. + +1. Add `store::NodePath`, `store::PathMapping`, and + `store::Layout`. +2. Add the stateful `store::Codec` interface with `paths()`, `read()`, + and `write()`, plus `CodecError`. +3. Add the runtime `store::codec::ZppBits`, preserving the existing `.bin` + path and serialized payload bytes. +4. Consolidate the DAG serialization functions in + `src/dag_builder/serialization.h` without changing their serialized field + order, meshoptimizer encoding, or JPEG texture encoding. +5. Add the terrain codec and one glTF codec configured for binary `.glb` or + JSON `.gltf`. Keep the current extension-dispatching `octree::MeshCodec` + only as temporary production compatibility glue until the Phase 3 cutover. + Test that glTF writer exceptions become `CodecError` values. +6. Port the two existing 3D layouts to ordinary function pairs without + changing stable IDs. The mappings return `level-index` and + `level/x/y/z` without file endings. +7. Add explicit `from_id()` and `all()` lookup functions in the 3D adapter. + Keep the singleton strategy registry only for the old production storage + path until Phase 3. +8. Compose each new 3D mapping with each applicable runtime codec in tests and + prove that they resolve all Phase 0 fixtures to identical physical payload + paths. +9. Add focused codec tests using single-file, multi-file, read/write, and + write-only test codecs. Test stable path ordering, unsupported operations, + directory creation, conversion of domain errors to `CodecError`, and + concurrent reads and writes. Exercise reentrancy of every production codec + used by the parallel DAG builder. + +Exit criterion: the extensionless mappings and runtime codecs together resolve +all Phase 0 fixtures to their existing physical payload paths; generic +`Layout` contains no extension; the new codec tests pass; existing production +storage and all callers still build unchanged through the temporary legacy +path. + +### Phase 3 — Generalize storage and index lifecycle + +1. Cut the production path construction over to the Phase 2 + `store::Layout` and runtime codecs. Move the legacy preferred extension + out of layout state and retain it as the 3D format adapter's codec selector. +2. Port legacy layout discovery so it recognizes codec endings through the + supplied resolver, strips the accepted ending, and then calls + `node_path_to_key()`. +3. Move copy error, raw storage, logical storage, and indexed storage into + `store`. Port the existing cache API where useful for source compatibility, + but do not require cache behaviour tests. +4. Make `RawStorage` exclusively own a configured + `std::unique_ptr>`. `Storage` owns it transitively through + raw storage; remove the codec template parameter from every storage type. +5. Replace every embedded `octree::Id` with `Traits::Key`. +6. Make every raw file operation obtain its complete file list through + `Codec::paths()`. `has()` requires every listed file, and `remove()` removes + every listed file. +7. Keep domain-specific mesh codecs outside the shared module under + `mesh::codec`. +8. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and + typed format/open errors. Split generic index maintenance from 3D index + serialization and legacy folder discovery. Test that only a missing index + starts discovery; all other index errors are returned. +9. Keep the current 3D `terrain.index` DTO and open functions as compatibility + adapters over the shared storage. Retain its exact preferred extension as + `codec_selector`, resolve it through the payload-domain mesh or DAG + resolver, and retain the format metadata required by automatic saving. +10. Add DAG storage convenience functions that supply the writable batch and + read-only metadata resolvers, and mesh storage convenience functions that + select the configured terrain or glTF codec. Preserve the DAG metadata + storage aliases. Migrate `dag_builder`, `dag_convert_debug`, and mesh + storage consumers without exposing codec objects at application call sites. +11. Migrate the existing octree storage aliases and all other application + callers, including adapting `dag::ThreadSafeStorage` to the new key and + expected-returning APIs while preserving its shared/exclusive locking. +12. Preserve the current 3D destructor-save behaviour until all callers have + explicit index finalization. Test move construction, move assignment over + dirty state, moved-from destruction, and the DAG move/release path. +13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's + overwrite mode and repeat debug export. Replace process termination on a + rejected overwrite with `AlreadyExists` in the storage-level expected + error. Test that a rejected save returns `AlreadyExists` and that enabling + overwrite replaces the existing payload. +14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable + `ClusterBatch` `.bin`, and read-only `NodeMetadata` `.bin` dispatch, plus + explicit failure for an unknown preferred extension and metadata writes. +15. Instantiate the shared storage tests with `raster_store::StoreTraits` + using a test-only path mapping and codec. This proves the storage templates + contain no hidden `octree::Id` dependency without defining a stable RF + layout, codec, or disk format. +16. Delete the old strategy base class, strategy registry, concrete strategy + classes, static codec concept and codecs, and their temporary compatibility + glue once no call site uses them. + +Test that the Phase 0 DAG fixture opens through both new resolvers and that a +new deterministic `.bin` payload matches its golden bytes and remains readable +through the unchanged ZPP serialization functions. Test unknown layout IDs, +malformed index metadata, invalid hierarchy keys, and retained underlying +open/codec errors through `std::expected`. + +Exit criterion: all existing applications build and all Phase 0 fixtures pass +through the shared runtime codec and storage implementation. Phase 0 DAG +payload bytes and `.bin` paths remain unchanged. No extension-dispatching +mesh codec, layout inheritance, RTTI lookup, static registrar, owning strategy +pointer, or second storage implementation remains under `octree`. + +### Phase 4 — Harden node reuse and enforce SF topology + +1. Change `Storage::copy_from()` to compare input and output codec path lists + for the fixed dummy `NodePath`. +2. Hard-link all actual files when the lists match. A partially failed + multi-file operation returns an error without rolling back links already + created. +3. Decode with the input codec and encode with the output codec when lists + differ. +4. Test: + - one-file hard linking; + - multi-file hard linking; + - different path counts and endings; + - error propagation after a partially failed multi-file hard link; + - overwrite failure leaving an existing logical node unindexed even when + old or partial files remain; + - conversion between terrain and glTF; + - `copy_from()` overwrite rejection; + - `copy_from()` overwrite-enabled replacement; and + - missing-file, hard-link, decode, and encode error propagation. +5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the + offending key when it encounters `Inner`. Do not add other validation rules + in this refactor. +6. Apply the validator to SF-merger merge and cut inputs and the DAG builder's + SF input before processing. Apply it to SF-builder and SF-merger output + after the completed `terrain.index` has been written. If output validation + fails, retain the written index and payloads for diagnosis, propagate + `sf::InvalidTopology` to the command line, and report that the output is + invalid. Do not apply the validator when opening DAG datasets, through + generic octree/store adapters, or in the diagnostic `sf_index_browser`. + Extract an SF-builder finalization boundary into `sfbuilderlib` which writes + the index, validates it, and returns the typed result so output validation + and command-line propagation are directly testable. +7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. + Change its subtree and cut call chains to propagate validation and + `copy_from()` failures through `std::expected` to the application boundary. +8. Add integration tests proving: + - valid `Leaf`/`Virtual` SF merge behaviour is unchanged; + - an SF input containing `Inner` fails validation before merge dispatch; + - invalid SF output writes `terrain.index` for diagnosis and returns + `sf::InvalidTopology` to the application boundary; + - an unchanged SF subtree is hard-linked and a changed boundary node is + newly written; and + - an unchanged leaf in the SF cut path is hard-linked while a clipped leaf + is newly written. + +Exit criterion: one-node copying works through `Codec::paths()`, SF consumers +reject `Inner` with a typed error before processing, existing valid SF merge +and cut behaviour is preserved, invalid SF output remains inspectable with a +written `terrain.index`, and neither a shared subtree copier nor a paired-tree +walker has been introduced. + +### Phase 5 — Cleanup and documentation + +1. Remove temporary forwarding headers that no repository caller needs. +2. Remove obsolete files under `octree/disk` and the old generic + implementation under `octree/storage`. +3. Keep only 3D key, format, codec, and compatibility adapters under + `octree`. +4. Update includes, CMake source lists, and precompiled-header includes. +5. Update [architecture.md](architecture.md), + [storage-format.md](storage-format.md), [status-quo.md](status-quo.md), and + the before-refactor report with links to the implemented boundary. State + clearly that RF formats and tools remain future work and that architecture + requirements are not acceptance criteria for this refactor. Preserve the + before-refactor report as history; do not rewrite it as if it described the + new code. +6. Document the final public names, a legacy 3D opening example, and an + in-memory `radix::tile::Id` topology example. Do not document a persistent + RF format or opening API. + +Exit criterion: repository search finds no generic implementation tied to +`octree::Id`; all tests pass; the old layout strategy hierarchy is gone. + +### Phase 6 — Remove migration-only compatibility fixtures + +1. Delete the Phase 0 golden SF and DAG datasets and the tests whose only + purpose is opening or byte-comparing data written before the completed + refactor. +2. Keep the format round-trip, path, resolver, topology, storage, error, and + application integration tests which exercise the final implementation + without pre-refactor fixture files. + +Exit criterion: no pre-refactor dataset fixture remains, and the retained test +suite passes against data produced by the final implementation. + +## Test and verification plan + +Generic store tests should live in the existing `unittests_terrainlib` target. +Suggested files: + +```text +unittests/terrainlib/store_index.cpp +unittests/terrainlib/store_traverse.cpp +unittests/terrainlib/store_layout.cpp +unittests/terrainlib/store_codec.cpp +unittests/terrainlib/store_storage.cpp +unittests/terrainlib/store_compatibility.cpp # temporary through Phase 5 +unittests/terrainlib/sf_validate_index.cpp +``` + +The temporary DAG payload-compatibility fixture and resolver integration test +belong in `unittests_dagbuilder`, because `terrainlib` must not depend on +`dag::ClusterBatch` or its serializers. Phase 6 removes the fixture and its +byte-comparison test while retaining resolver tests built from current data. + +The validator's unit tests belong in `unittests_terrainlib`. Boundary tests +belong with their consumers: SF-builder output validation in +`unittests_sfbuilder`, merge and cut validation/error propagation in +`unittests_sfmerger`, and DAG-builder SF-input validation in +`unittests_dagbuilder`. `sf_index_browser` remains unvalidated by design. + +Cache migration has compile coverage only. Existing cache application call +sites must continue to build where the API remains meaningful, but no cache +behaviour or compatibility test is required. + +During implementation: + +1. Build in `$source_dir/build/$config_name`. +2. Run unit tests from that build directory. +3. Run the focused store tests after each edit. +4. Run the full `unittests_terrainlib` target at every phase boundary. +5. Run `unittests_dagbuilder` after the ZPP codec, DAG serialization header, or + DAG resolver changes. +6. Configure with `ALP_BUILD_SF_BUILDER`, `ALP_BUILD_SF_MERGER`, + `ALP_BUILD_SF_INDEX_BROWSER`, `ALP_BUILD_DAG_BUILDER`, and + `ALP_BUILD_DAG_CONVERT_DEBUG` enabled, then build `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` after their + storage aliases move. +7. Run `unittests_sfbuilder`, `unittests_sfmerger`, and + `unittests_dagbuilder`, plus any existing merger integration fixture, after + the Phase 4 validation changes. +8. Inspect `git diff --check` and the final worktree before each commit. + +No formatting-only pass or unrelated refactor belongs in these commits. + +## Expected migration map + +| Current code | Target | +|---|---| +| `octree/NodeStatus.h` | `store/NodeStatus.h` plus temporary alias | +| `octree/NodeStatusOrMissing.h` | `store/NodeStatusOrMissing.h` plus temporary alias | +| `octree/IndexMap.*` | `store/Index.h` | +| `octree/traverse.h` | `store/traverse.h` | +| complete node paths embedded in layouts | extensionless `store/NodePath.h` plus codec endings | +| `octree/disk/Layout.h` | `store/Layout.h` | +| `octree/disk/layout/Strategy.h` | `store/PathMapping.h` | +| `StrategyRegister.h` | explicit dimension-adapter lookup functions | +| `strategy/Flat.h` | `octree/store_layout/Flat.h` | +| `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | +| `octree/storage/cache/*` | `store/cache/*` where useful; compile compatibility only | +| `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | +| `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | +| `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | +| `octree/storage/codec/ReadOnlyCodec.h` metadata specialization | `dag::codec::MetadataView` | +| `octree/storage/RawStorage.h` | `store/RawStorage.h` | +| `octree/storage/Storage.h` | `store/Storage.h` | +| `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | +| `octree::StorageSettings::allow_overwrite` | `store::StorageSettings::allow_overwrite`, preserving default and enabled behaviour | +| `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | +| `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | +| DAG serializers in `dag_node.h`, `dag_id.h`, `metadata.h`, `encoded.h`, and `zpp_bits_glm.h` | `dag_builder/serialization.h` | +| `dag_builder/storage.h` aliases | Batch and read-only metadata storage aliases plus codec resolver convenience functions | +| `dag_builder/thread_safe_storage.h` | Adapted caller-side wrapper over shared storage | +| `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | +| `sf_merger::NodeWriter` auxiliary `.png` write | remains an unmanaged, application-local debug artifact | +| `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | +| SF merger `Inner` `UNREACHABLE()` path | `sf::validate_index()` returning `sf::InvalidTopology` | + +## Risks and controls + +| Risk | Control | +|---|---| +| Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | +| The refactor changes current DAG `.bin` bytes | Temporary golden DAG fixture, unchanged serializers, and explicit `.bin` resolvers | +| Octree format adapter gains DAG dependencies | Caller-supplied resolver owned by `dag_builder` | +| Unknown legacy extension silently selects the wrong codec | Return an explicit `UnsupportedCodec` error | +| Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | +| Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | +| `radix::tile::Id` root underflows | Traits intercept root parent lookup | +| Invalid `radix::tile::Id` values enter the shared index | Validate through `raster_store::StoreTraits` and test boundary zooms | +| Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | +| SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | +| A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | +| Multi-file hard linking fails partway through | Remove any existing index entry before modifying target files, stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; old or partial files may remain | +| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | +| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | +| Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | +| The refactor accidentally fixes the future RF disk format | Do not add a persistent 2D format adapter, layout, or codec | + +## Deferred raster-fundamentalis work + +This refactor is a prerequisite for, not an implementation of, the system +described in [architecture.md](architecture.md). That document and +[storage-format.md](storage-format.md) remain useful design input, but their RF +details are not acceptance criteria for this refactor and are not declared +final here. + +A later RF design phase must resolve and test at least: + +- the persistent index filename and schema, and how they use the existing + generic serialization envelope and versioning support; +- persistent tile keys, coordinate convention, layout IDs, and node paths; +- tile and source-attribution payload formats and codecs; +- snapshot construction, validation, publication, and crash expectations; +- hard-link preflight and unchanged-tile reuse; +- `rf_merger` policy, paired-tree walking, and `Inner` copy behaviour; and +- RF builders, converters, debugging outputs, and other tools. + +None of those decisions blocks completion of this refactor. +The removed ideas are retained, without implementation-plan status, in the +explicitly draft [rf_builder notes](rf_builder.md) and +[rf_merger notes](rf_merger.md). diff --git a/docs/raster-store/rf_builder.md b/docs/raster-store/rf_builder.md new file mode 100644 index 00000000..7006b981 --- /dev/null +++ b/docs/raster-store/rf_builder.md @@ -0,0 +1,202 @@ +# DRAFT — `rf_builder` + +Status: **draft archive of removed ideas**. + +This is not a current implementation plan, accepted format specification, or +statement that the choices below are correct. It is a lightly reformatted +archive of RF-builder and persistent-2D material removed from +[refactor-plan.md](refactor-plan.md). The details are retained so they are not +lost; they require a separate design pass after the shared-store refactor. + +## Proposed names and source boundary + +The removed proposal placed the 2D implementation in +`src/terrainlib/raster_store` and used the `raster_store` namespace: + +```text +src/terrainlib/raster_store/ +├── StoreTraits.h +├── IndexFile.h +├── Storage.h +├── codec/ +│ ├── Amort.h +│ └── Debug.h +└── store_layout/ + └── ZoomXYGoogle.h +``` + +The proposed boundary was: + +- `store` contains dimension- and payload-neutral mechanisms; +- `raster_store` contains the 2D format, key adapters, and raster codecs; and +- subdirectory names match their namespaces where a subnamespace is used. + +The proposed concrete index type was: + +```cpp +store::Index +``` + +## `raster_store::StoreTraits` + +The removed proposal adapted `radix::tile::Id` through +`raster_store::StoreTraits` and specified that it: + +- treats zoom zero as the only root; +- never calls `radix::tile::Id::parent()` at zoom zero, where it underflows; +- rejects coordinates outside `[0, 2^zoom)`; +- accepts zoom levels 0 through + `std::numeric_limits::digits`, inclusive; +- treats that maximum zoom as terminal because a child cannot be represented + by the `uint32_t` x/y coordinates; +- validates the maximum zoom without evaluating an overflowing + `uint32_t{1} << 32`; +- uses `radix::tile::Id::Hasher`; and +- uses the Google/XYZ convention, with the origin at the north-west, at the + persistent boundary. + +The shared code was expected to obtain roots, parents, children, validation, +and hashing through the traits without specializing on the key type. + +## Node paths and layout + +The removed example mapped a raster node to an extensionless `store::NodePath`: + +```text +raster-store ZXY 12/2200/1400 +``` + +The proposed lookup functions were: + +```cpp +raster_store::store_layout::zoom_x_y_google() +raster_store::store_layout::from_id(id) +``` + +The stable layout ID was `zoom/x/y_google`. The default mapping produced +`//` directly below the snapshot root; there was no fixed +`chunks/` directory. A codec, rather than the layout, added `.amort` or debug +file endings. + +## Payload and codec sketches + +The removed proposal used the shared runtime codec interface and placed +raster codecs in `raster_store::codec`: + +```cpp +template +struct raster_store::codec::Amort + : store::Codec> { .. }; + +template +struct raster_store::codec::Debug + : store::Codec> { .. }; +``` + +`Amort` was proposed as readable and writable. `Debug` was proposed as +write-only, with runtime options for data format, attribution format, JPEG +quality, or similar debugging choices. It was not to be template-composed +from separate image codec types. + +The multi-file debug example was: + +```text +Debug raster codec configured for JPEG data and PNG attribution + 12/2200/1400 + -> 12/2200/1400.data.jpg + -> 12/2200/1400.attribution.png +``` + +The proposed final tile path was `//.amort`. The `.amort` payload +and source-attribution-table serialization were explicitly not defined beyond +the separate storage-format notes. + +## Index and format-adapter sketch + +The removed proposal used a separately versioned `raster_store::v1` DTO stored +as `raster_store.index`. + +The proposed version-1 contents were: + +- a layout ID; +- sparse key/status entries; +- serialized `Leaf`, `Inner`, and `Virtual` values; +- `Missing` represented by absence; +- no derived aggregate metadata until a concrete query requires it; +- fixed-width `uint32_t` zoom/x/y fields instead of platform `unsigned`; +- entries ordered lexicographically by `(zoom, x, y)`; and +- duplicate keys rejected while reading. + +The proposed serialization envelope contained: + +- a fixed, file-type-specific 64-bit magic value generated during + implementation; +- a 32-bit version; +- zlib CRC-32 stored as `uint32_t` and computed over the compressed payload; +- a compression enum; and +- a zstd-compressed payload using zstd's best-compression setting. + +The proposal imported zstd through the project's CMake dependency facility and +used the existing `ZLIB::ZLIB` dependency for CRC-32. + +Index serialization was not a responsibility of `store::Index`. The proposed +2D format adapter supplied the index filename, index conversion, mapping +lookup, and default mapping. + +## Snapshot publication sketch + +The removed proposal required explicit finalization/publication for the 2D +snapshot API; a destructor was not to make an incomplete snapshot +authoritative. + +The publication details are now retained in +[architecture.md](architecture.md#publication). The removed proposal used a +sibling `.part` directory, wrote the index last, validated and +closed all files, and atomically renamed it to `` on the same +filesystem. The destination could not already exist. + +The removed text explicitly did not promise durability or safe recovery after +a power failure, operating-system crash, or storage failure, and did not add +`fsync()`, `fdatasync()`, `FlushFileBuffers()`, or equivalent synchronization. + +## Removed implementation and verification ideas + +The removed 2D-adapter phase contained these items: + +- add the checked persistent-key conversion around `radix::tile::Id`; +- add the `//` mapping with stable ID `zoom/x/y_google`; +- define the versioned `raster_store.index` DTO; +- implement the magic/version/checksum/compression envelope; +- add the 2D format adapter and storage aliases under `raster_store`; +- add `raster_store::codec::Amort` when final `.amort` + serialization is available; +- add the output-only `raster_store::codec::Debug`; +- use a test codec instead of making `.amort` claims if final serialization is + unavailable; and +- keep raster-specific processing outside the shared store. + +The removed verification list contained: + +- invalid and boundary tile IDs, including maximum zoom and rejected children; +- index serialization and validation; +- `Leaf`/`Inner` coexistence; +- sparse traversal and ancestor lookup; +- path round trips; +- snapshot hard-link reuse; +- AMORT-to-debug output conversion; +- explicit cross-filesystem/preflight failure; +- publication from `.part` to ``; and +- publication rejection when the destination already exists. + +## Removed risks and controls + +| Removed risk | Removed control | +|---|---| +| `radix::tile::Id` root underflows | Traits intercept root parent lookup | +| Invalid 2D coordinates become persistent | Validate on every disk/API boundary | +| Linked snapshots are modified in place | Immutable snapshot API and overwrite-disabled output | +| Hard-link failure appears late | 2D operation preflight and explicit errors | +| Generic index dictates both disk formats | Separate 3D and 2D format adapters | + +All material in this document remains provisional despite the concrete names +preserved above. diff --git a/docs/raster-store/rf_merger.md b/docs/raster-store/rf_merger.md new file mode 100644 index 00000000..3fc97d0a --- /dev/null +++ b/docs/raster-store/rf_merger.md @@ -0,0 +1,231 @@ +# DRAFT — `rf_merger` + +Status: **draft archive of removed ideas**. + +This is not a current implementation plan or accepted RF merge specification. +It is a lightly reformatted archive of subtree-copy and paired-walker material +removed from [refactor-plan.md](refactor-plan.md). The details are retained so +they are not lost; later discussion already established that RF semantics must +be designed before choosing these abstractions. + +## Proposed names and source boundary + +The removed proposal added these dimension-neutral files: + +```text +src/terrainlib/store/ +├── copy_subtree.h +└── merge/ + ├── Action.h + └── walk.h +``` + +The proposed migration map was: + +| Existing code | Removed target | +|---|---| +| `sf_merger::NodeWriter` subtree loop | `store/copy_subtree.h` | +| `sf_merger::Merger` recursion | `store/merge/walk.h` | + +The proposal intended shared mechanisms to contain no mesh or raster merge +policy. + +## One-node copy proposal + +The proposed one-node API added: + +```cpp +struct CopyOptions { + bool force_reencode = false; +}; +``` + +For one key, the proposed `Storage::copy_from()` behaviour was: + +1. Call input and output `Codec::paths()` with the same fixed dummy + `NodePath`, proposed as `__codec_probe__/node`. +2. Compare path lists exactly, including count, order, and filename endings. +3. When the lists match and `force_reencode` is false, call both codecs with + the actual source and target `NodePath` and hard-link every corresponding + file. +4. When lists differ or re-encoding is forced, read with the input codec and + write with the output codec. +5. Update the target index only after all links or the write complete. + +If a multi-file link failed partway through, the proposal removed target links +created by that call before returning `CopyError`. `CopyError` retained any +underlying `CodecError`. There was no silent file-copy fallback. + +Codec settings that did not change `paths()`, such as compression level or +JPEG quality, did not force re-encoding by default. Callers could pass +`force_reencode = true`. + +The removed hard-link rules were: + +- never modify an existing linked payload in place; +- matching codec path lists hard-link every file; +- different path lists decode with the input codec and encode with the output + codec; +- `force_reencode` selects decode/encode; +- hard-link failure is explicit; +- 2D snapshot tools preflight hard-link support before a long operation; and +- no silent file-copy fallback. + +## Shared subtree-copy proposal + +The removed API sketch was: + +```cpp +std::expected +store::copy_subtree( + const IndexedStorage& source, + Storage& target, + const Key& root, + CopyOptions options = {}); +``` + +The proposed operation: + +1. traversed an indexed source subtree; +2. skipped `Virtual` nodes; +3. called `copy_from()` for physical payloads in `Leaf` and `Inner` states; +4. continued traversal below `Inner`; and +5. returned copy failures rather than asserting or terminating. + +The operation was intended to be payload-neutral and know nothing about +meshes, rasters, masks, attribution, or their encodings. + +The removed target call chain was: + +```text +merge policy decides to keep a source subtree unchanged + -> store::copy_subtree() + -> store::Storage::copy_from() + -> hard-link every codec path, or decode/encode +``` + +## Removed `Inner` copying rationale + +The removed proposal used this topology model: + +| Status | Physical payload | Indexed descendants | +|---|---:|---:| +| `Leaf` | yes | no | +| `Inner` | yes | yes | +| `Virtual` | no | yes | + +It used this RF example: + +```text +zoom 10 physical tile -> Inner +└── zoom 11 physical tile -> Leaf +``` + +The proposal said subtree reuse must preserve both payloads: copy the parent +payload, continue below it, and allow insertion of the descendant to promote +the copied parent from `Leaf` to `Inner` through normal index transitions. + +The proposed status handling was: + +```cpp +switch (status) { +case NodeStatus::Virtual: + break; +case NodeStatus::Leaf: +case NodeStatus::Inner: + target.copy_from(id, source, options); + break; +} +``` + +This described unchanged-subtree copying only. It did not define how two RF +trees should be merged. + +## Paired-tree walker proposal + +The removed proposal extracted dimension-neutral recursion from +`sf_merger::Merger` and introduced these policy results: + +```cpp +store::merge::Recurse +store::merge::Ignore +store::merge::KeepLeft +store::merge::KeepRight +store::merge::Write +``` + +The walker owned recursion and unchanged-subtree reuse. The policy owned +selection and payload combination. + +The removed proposal required all 16 combinations of `Missing`, `Leaf`, +`Inner`, and `Virtual` to be handled or rejected with a typed error rather than +falling into `UNREACHABLE()`. + +It kept these concerns outside the shared walker: + +- `NodeLoader` ancestor mesh reconstruction; +- ECEF node bounds; +- mesh masks and clipping; +- mesh combination and texture atlas generation; and +- mesh validation and auxiliary texture writes. + +The proposal suggested that a future 2D merger could supply a raster policy. +Later discussion identified that the mutually exclusive action list cannot +express both an action for an `Inner` payload and recursion into descendants. +Ideas mentioned after that were a combined `WriteAndRecurse` result or +independent current-node and descendant decisions. None is selected. + +## Removed implementation and verification ideas + +The removed subtree/walker phase contained: + +- compare input and output codec path lists for a fixed dummy `NodePath`; +- hard-link all files when lists match, including partial-failure cleanup; +- decode with the input codec and encode with the output codec when lists + differ; +- add `CopyOptions::force_reencode`; +- add the shared unchanged-subtree copier; +- add the paired hierarchy walker and typed actions; +- cover all 16 status pairs with table-driven tests; +- adapt the 3D merger while keeping mesh policy in `sf_merger`; +- remove generic recursion and copy logic from `sf_merger::Merger` and + `NodeWriter`; and +- add a 3D integration test for an unchanged hard-linked subtree and a newly + written changed boundary node. + +The removed focused tests included: + +- one-file hard linking; +- multi-file hard linking; +- different path counts and endings; +- forced re-encoding with otherwise equal paths; +- conversion between terrain and glTF; +- conversion into a write-only codec; +- runtime failure for an unsupported codec operation; +- copies containing `Leaf`, `Virtual`, and `Inner`; and +- all 16 paired status combinations. + +The proposed generic test file was: + +```text +unittests/terrainlib/store_merge_walk.cpp +``` + +## Removed error and risk notes + +The removed error-propagation proposal passed copy failures through +`copy_subtree()` and the merge call chain to the application boundary with the +affected key and path. Codec, filesystem, unsupported-conversion, +malformed-dataset, and overwrite failures used `std::expected`; operational +failures were not assertions or intentional exceptions. + +| Removed risk | Removed control | +|---|---| +| `Inner` payloads are lost during subtree reuse | Copy every physical status and test mixed-depth fixtures | +| Multi-file hard linking fails partway through | Remove links created by the failed `copy_from()` before returning | +| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | +| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | +| Shared code accumulates mesh/raster policy | Dependency tests/review against the source boundary | + +All material in this document remains provisional despite the concrete names +preserved above. diff --git a/docs/raster-store/sampling-and-generation.md b/docs/raster-store/sampling-and-generation.md new file mode 100644 index 00000000..83309521 --- /dev/null +++ b/docs/raster-store/sampling-and-generation.md @@ -0,0 +1,249 @@ +# Sampling and pyramid generation + +This document defines generator-facing sampling terminology and invariants. +It deliberately separates output sampling from how an original source raster +was measured or produced. + +## Terminology + +### Vertex pixel + +A vertex pixel is a generated value located on a grid vertex. For a tile with +`N` intervals per side, vertex positions are: + +```text +x(i) = left + i × tile_width / N, i = 0 … N +``` + +The output has `N+1` pixels per side: + +```text +tile boundary tile boundary +●---------●---------●---------●---------● +``` + +Height maps used to form mesh vertices require this placement. Adjacent +rendering tiles contain overlapping copies of their shared edge and corner +vertex pixels. + +### Area pixel + +An area pixel is a generated value associated with one raster cell. For a tile +with `N` cells per side, effective cell-centre positions are: + +```text +x(i) = left + (i + 1/2) × tile_width / N, i = 0 … N-1 +``` + +The output has `N` pixels per side: + +```text +tile boundary tile boundary +│ × × × × │ +``` + +The term describes placement and support in the generated grid. It does not +assert that the input value was a physical area integral. + +## Source semantics versus output placement + +An input height raster may have been produced from LiDAR points through +gridding, interpolation, fitting, or averaging. An orthophoto may already have +passed through sensor integration, reconstruction, reprojection, and +resampling. Those histories do not decide where a delivery format requires +its output values. + +Generation is modelled as: + +```text +stored discrete raster + ↓ reconstruct its implied field +continuous or evaluable field + ↓ low-pass for target resolution +filtered field + ↓ evaluate on requested output grid +vertex pixels or area pixels +``` + +The source interpretation and reconstruction rule are layer/generator policy. +Vertex-pixel and area-pixel placement are output requirements. + +## Reduction by two + +### Vertex pixels + +At fine spacing `Δ`, fine vertex positions are `nΔ`. Coarse positions are +`2mΔ`, coinciding with every second fine location: + +```text +fine: ●---●---●---●---● +coarse: ●-------●-------● +``` + +Copying every second value would be unfiltered decimation and is unacceptable +because frequencies above the new Nyquist limit would alias. The generator +must low-pass first, using a kernel centred on each retained vertex position: + +```text +coarse[m] = Σ h[k] × fine[2m - k] +``` + +The spatial centre stays in place; the value generally changes because it is +sampled from the filtered signal. + +### Area pixels + +Fine area-pixel centres are `(n+1/2)Δ`. A coarse cell spans two fine cells and +has its centre at `(2m+1)Δ`, halfway between two fine centres: + +```text +fine cells: |---- × ----|---- × ----| +coarse cell: |---------- × ----------| +``` + +The simplest reduction is the average of each 2x2 fine block. If fine values +are exact equal-area averages, that produces the exact average over the union +of the four cells. A box filter is not an ideal anti-aliasing filter, however, +and may be insufficient for visual imagery or other signals. + +A higher-quality area-pixel reduction applies a low-pass filter with the +correct half-sample phase, centred on the coarse cell centre. Its support may +extend beyond the four cells geometrically covered by the coarse cell. + +## No duplicated height borders in the store + +The authoritative store does not persist overlapping rendering borders. +Pyramid generation constructs a vertex-pixel output only after reconstruction +and filtering. + +The implementation may obtain a requested `(N+1) × (N+1)` output window by +reading non-overlapping store chunks plus the filter halo required on every +side. The generated shared vertices must be computed from the same global +coordinates and source data for both neighbouring output tiles. + +The generator must not independently clamp its filter at each tile edge. +Clamping would make an internal tile boundary behave like a data boundary and +could produce seams. + +Two implementation strategies can satisfy the invariant: + +1. Evaluate shared global vertex coordinates deterministically from a common + window reader; or +2. Generate a metatile, filter it once, and split it into overlapping output + tiles. + +The first gives execution-order independence. The second may reduce repeated +I/O. They can coexist if tests establish identical results. + +## Filter halos and chunk boundaries + +Any nontrivial low-pass filter needs samples outside the exact output bounds. +The required halo is determined by the reconstruction and reduction filters, +not by a fixed one-pixel border flag. + +The store reader should expose a logical raster window over the quadtree. It +resolves: + +- physical chunks selected for the requested accuracy; +- ancestor fallback where finer data is absent; +- chunk and source-map decoding; and +- neighbouring data needed by the window. + +The generator determines the requested halo and applies boundary conditions +only at true dataset/world boundaries or NoData boundaries. + +## Mixed sources + +The raster store contains exactly one payload and one source ID for each +stored pixel. A generator filter may span pixels attributed to several +sources: + +```text +store pixels: A A A B B +filter support: [-------] +output value: blend of A and B payloads +``` + +This is allowed. A generated output pixel does not retain a source ID. The +generated tile records tile-level provenance, at minimum the set of source IDs +whose payload values contributed nonzero filter weight to any output pixel. + +Source IDs are categorical and are never averaged. Payload filtering and +provenance collection are parallel operations: + +```text +numeric payload samples → weighted filtered value +source IDs → contributing-source set +``` + +The exact handling of invalid/NoData samples requires a policy. A common +continuous-raster rule is to normalize by the total weight of valid samples, +but that must not be applied automatically to categorical data. + +## Layer-specific filtering + +Sampling placement alone does not determine a correct filter: + +| Semantic kind | Relevant considerations | +|---|---| +| Height | low-pass before decimation; terrain error and peak loss | +| Orthophoto | linear-light filtering; alpha premultiplication | +| Categorical | mode, coverage, or another categorical policy | +| Probability/coverage | conservative area averaging may be appropriate | +| Vector/normal | component filtering followed by normalization where needed | +| Mask/NoData | validity-aware weights and explicit coverage rules | + +The current `radix::raster::generate_mipmap` performs a component-wise 2x2 +box average. It may be a reference for simple area-pixel aggregation, but it +does not implement these policies or vertex-pixel filtering. + +## Coherent coarse-source selection + +The generator need not always filter the deepest available descendants. If a +physical chunk at the requested scale is sufficiently accurate, using that +single coherent source may be preferable to composing several finer sources. + +The selection process is conceptually: + +```text +choose physical representation(s) for the requested output and quality policy + ↓ +read a continuous window with fallback and required halo + ↓ +filter for the target resolution + ↓ +evaluate vertex pixels or area pixels +``` + +Source-selection/refinement policy precedes filtering. Filtering does not +change the authoritative hierarchy. + +## World and dataset boundaries + +The generator needs explicit rules for: + +- horizontal wrapping at the Web Mercator antimeridian; +- north/south limits of the Web Mercator world; +- areas with no physical ancestor or descendant; +- NoData holes inside otherwise covered chunks; and +- filters whose support crosses a layer's coverage boundary. + +These rules are not yet decided. Tests must distinguish true boundaries from +ordinary internal chunk and delivery-tile boundaries. + +## Required golden tests + +Before production filtering is implemented, synthetic fixtures should prove: + +1. A constant raster remains constant across chunks and pyramid levels. +2. An impulse or frequency sweep demonstrates the chosen anti-alias response. +3. Two adjacent area-pixel tiles match a single equivalent metatile result. +4. Two adjacent vertex-pixel tiles produce bit-identical shared edges. +5. Filtering is unchanged when a store window is split into different chunks. +6. A source boundary blends payloads but reports both tile-level sources. +7. A NoData boundary follows the configured validity rule. +8. A coherent physical parent can be chosen instead of finer descendants. +9. TMS and Slippy input IDs normalize to the same canonical spatial tile. + +The filter coefficients and acceptable numeric tolerances remain open design +decisions. The tests should lock them only after representative evaluation. diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md new file mode 100644 index 00000000..9d2b5701 --- /dev/null +++ b/docs/raster-store/status-quo.md @@ -0,0 +1,274 @@ +# Status quo and reuse assessment + +This document evaluates the current repository after commit `9cf9065` +(`Consolidate raster handling and use std::expected`). It distinguishes +reusable mechanisms from interfaces that encode assumptions unsuitable for +the raster store. + +## Summary + +The project already contains most low-level ingredients: + +- `radix::Raster` for contiguous typed raster memory; +- `radix::tile::Id` for quadtree addressing; +- Web Mercator grid calculations and GDAL reprojection in `tile_builder`; +- sparse topology, indexed traversal, codecs, layouts, and hard-link reuse in + the octree storage code; and +- the SF merger's snapshot-like reuse of unchanged subtrees. + +There is no existing component that should become the raster store unchanged. +The best path is to compose the Radix raster and tile primitives with a new +2D storage layer, while extracting or adapting selected octree-storage ideas. + +## Reuse matrix + +| Component | Assessment | Intended use | +|---|---|---| +| `radix::Raster` | Reuse directly | In-memory payloads and source maps | +| `radix::RasterMask` | Reuse directly | Temporary validity/selection masks | +| `radix::raster::transform` | Reuse directly | Typed pixel transformations | +| `radix::raster::generate_mipmap` | Do not use as general generator | Only a 2x2 component-wise box average | +| `radix::tile::Id` | Reuse after hardening or through an adapter | Persistent quadtree keys | +| `radix::quad_tree::Node` | Do not reuse for disk index | Dense in-memory ownership model | +| CTB `GlobalMercator` and grid bounds | Reuse initially | Tile bounds and Web Mercator resolution | +| `Dataset` and GDAL setup | Reuse/adapt | Input discovery and reprojection | +| `DatasetReader` | Adapt substantially | Windowed, typed, multi-band ingestion | +| `Tiler` / `ParallelTiler` | Reuse calculations, replace orchestration | Candidate chunk enumeration | +| `ParallelTileGenerator` | Do not use as store builder | Small-file writer with incompatible lifecycle | +| Octree `IndexMap` algorithm | Reuse design; generalize or port | Sparse physical/virtual topology | +| Octree `Storage_` / `RawStorage_` | Reuse design and selected code | Codec boundary, indexing, hard-link copy | +| Octree disk layouts | Do not use as-is | They encode `octree::Id` and 3D paths | +| SF merger visitors and geometry code | Do not reuse | Mesh- and ECEF-specific semantics | +| SF merger unchanged-subtree copy | Reuse design | Snapshot construction and hard links | +| `zpp_bits` serialization helpers | Reuse cautiously | Versioned metadata/index serialization | + +## Radix raster + +### What can be reused + +`extern/radix/src/radix/raster.h` now provides a shared, value-typed raster: + +- rectangular `glm::uvec2` dimensions; +- contiguous row-major `std::vector` storage; +- element and byte spans; +- typed pixel access; +- move construction from an existing vector; +- a contiguous byte-valued `RasterMask`; +- dimension-checked concatenation; and +- masked and unmasked transforms. + +These properties fit both store arrays: + +```cpp +radix::Raster payload; +radix::Raster source_map; +``` + +The class correctly remains independent of Web Mercator, tile IDs, source +metadata, compression, and file I/O. Those belong to higher layers. + +### What needs adaptation around it + +Large chunks and filtered generation will benefit from operations not +currently supplied by `Raster`: + +- non-owning raster views and subwindows; +- explicit row stride where external codecs require it; +- halo/window assembly across neighbouring chunks; +- checked construction that reports allocation/dimension errors without + relying on assertions; and +- streaming or block processing when a full set of input chunks would exceed + the memory budget. + +These should be introduced only when required. They are not reasons to embed +store concepts into `Raster`. + +### What cannot be reused for final filtering + +`radix::raster::generate_mipmap` requires a square, power-of-two raster and +reduces each 2x2 group by component-wise averaging. It does not provide: + +- a selectable reconstruction or low-pass filter; +- the phase difference between vertex pixels and area pixels; +- halo samples across tile boundaries; +- linear-light colour and premultiplied-alpha handling; +- NoData-aware normalization; +- categorical reduction; or +- source-contribution tracking. + +It is therefore a useful simple raster utility, not the raster-store pyramid +generator. + +## Radix tile addressing + +### What can be reused + +`radix::tile::Id` already contains the required 2D identity: + +- zoom level; +- `x/y` coordinates; +- parent and four-child relationships; +- TMS/Slippy conversion; and +- hashing and ordering support. + +### Required hardening + +Persistent storage needs stronger invariants than the current convenience +type supplies: + +- Calling `parent()` at zoom zero currently underflows. +- Construction does not reject coordinates outside `[0, 2^z)`. +- Shifting `1u << zoom_level` limits valid conversion at high zooms. +- The scheme participates in identity, so the same spatial tile in TMS and + Slippy form becomes two keys. +- There is no persistent serialization contract or format version. + +The store should choose one canonical scheme and normalize all IDs at its API +boundary. Whether the hardening belongs in Radix or in a checked store adapter +is open. + +`radix::quad_tree::Node` is not a replacement for the index. It owns a +fully allocated group of four children whenever refined, represents no +missing child within such a group, and has no persistence or physical/virtual +status. The store requires a sparse map keyed by tile ID. + +## Tile builder and GDAL path + +### Reusable foundations + +The current tile builder already demonstrates: + +- opening GDAL raster datasets; +- reading dataset bounds; +- selecting Web Mercator or geodetic CTB grids; +- transforming arbitrary source SRS data during reads; +- calculating tile bounds and resolutions; +- enumerating intersecting `radix::tile::Id` values; and +- parallel per-tile processing. + +`ctb::GlobalMercator`, `Tiler::tile_for`, and `ParallelTiler` are useful +references and may be reused initially for grid math. + +### Required changes + +`DatasetReader` currently reads one band into a float raster, uses cubic GDAL +warping, and constructs a warped VRT for a requested output rectangle. The +store needs typed and multi-band reads, explicit alpha/NoData handling, +controlled resampling, source metadata, and deterministic alignment with the +store chunk grid. + +`Tiler` models delivery-oriented dimensions through `Border::Yes/No` and a +south/east extra pixel. Store chunks have no rendering overlap, and generated +vertex pixels require an explicit global sampling/filtering model. The border +boolean should not define store geometry. + +`ParallelTileGenerator` writes individual image files for explicitly +enumerated tiles. It is not suitable as the snapshot builder because it lacks: + +- source-map generation; +- old-snapshot reuse; +- atomic chunk containers; +- index transactions and publication; +- resumability validation; and +- bounded enumeration/streaming for very large tile sets. + +Its parallel work pattern and progress reporting may still inform the new +builder. + +## Octree index and storage + +### Reusable design + +The octree index captures the required topology semantics: + +- `Leaf`: physical payload without indexed descendants; +- `Inner`: physical payload with indexed descendants; +- `Virtual`: no physical payload, but indexed descendants; and +- absence from the map: missing node and subtree. + +Adding a physical node creates virtual ancestors and promotes a physical +ancestor from leaf to inner. Removing nodes collapses unused virtual chains. +Traversal follows only present index entries. + +The storage stack also has useful separation between: + +- logical indexed storage; +- raw path-based storage; +- disk layout; +- payload codec; and +- optional cache. + +`RawStorage_::copy_from` demonstrates hard-link reuse when source and target +extensions match. The SF merger demonstrates copying unchanged physical +subtrees and writing a new output index. + +### Why it cannot be reused unchanged + +The entire stack uses concrete `octree::Id` types. `IndexMap`, cache APIs, +layouts, filesystem path parsing, traversal, storage, serialization, and +formatting all embed this type. The coordinate-directory layout is +`level/x/y/z`, and the default codec is a mesh codec. + +The existing index file also records no tree kind. Reinterpreting its +serialized `(level, index)` octree IDs as web tiles would be unsafe. + +A 2D implementation can either: + +1. generalize the hierarchy/storage stack over an ID and layout policy; or +2. create a raster-store-specific 2D implementation using the same algorithms. + +Generalization avoids duplicate infrastructure but has a larger blast radius +in mature octree code. A separate implementation is initially safer but risks +long-term duplication. This requires an explicit decision before coding. + +### Behaviours that should not be copied blindly + +- Hard-link failure currently has no copy fallback. +- Existing indexes are trusted rather than reconciled against filesystem + contents when opened. +- An unindexed output discovers files through a final recursive directory + scan. +- Some error paths use assertions or process termination. +- The binary index lacks an explicit magic/version/topology header suitable + for a new durable format. + +The raster store should retain the useful topology and codec boundaries while +specifying stronger snapshot, validation, and error-handling rules. + +## SF builder and merger + +The SF builder's ECEF octree placement, mesh construction, texture atlases, +mask clipping, and mesh visitors are not reusable for a Web Mercator raster +store. + +The reusable ideas are architectural: + +- transform source data into a canonical spatial system during build; +- preserve physical ancestors as fallback representations; +- use an index to avoid per-node filesystem probing; +- stop refinement when a coherent representation is sufficient; and +- hard-link unchanged chunks into a new output dataset. + +Those ideas should be reimplemented against Radix tile IDs and raster payloads +rather than adapted through mesh abstractions. + +## Recommended component boundary + +The current components suggest the following dependency direction: + +```text +Radix + Raster, RasterMask, tile::Id, geometry + ↓ +Terrain library + GDAL dataset access, Web Mercator grid math, filtering primitives + ↓ +Raster store + source catalog, chunk container, sparse index, snapshots + ↓ +Builders and generators + ingestion, source selection, texture pyramids, height/geometry pyramids +``` + +The raster store should consume `radix::Raster`; Radix should not depend on +the store's source catalog, file format, or snapshot lifecycle. diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md new file mode 100644 index 00000000..5767e0f8 --- /dev/null +++ b/docs/raster-store/storage-format.md @@ -0,0 +1,99 @@ +# Storage format + +This document specifies the logical format and required invariants for the following data stores: +- raster-fundamentalis (our authoritative raster store) +- tile-base (the tile-pyramid for our tile-server) +- the source-attribution table (for correct copyright attribution and data selection) + +## Source-attribution table +The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. +- stores a vector of structs (called Table), each struct describing one source +- the struct (called Entity) contains: + - the spatial resolution in pixel-width at the equator (EPSG:3857) + - the date of data acquisition + - the date of ingestion + - a copyright string + - a copyright link string + - a license string +- the index 0 is reserved for "no-data" +- the index must be checked to be smaller than 2^16-1, and we throw unsupported if it becomes larger (this is because of the storage format of tiles). +- the source attribution table is implemented in src/terrainlib/source_attribution.h, in the namespace source_attribution::* +- We need a `struct` declaration and a `using Table = .. ` (both versioned) +- the source-attributino table is stored in a file named source_attribution_table.ard (.alpine raster data) +- there is one per directory tree (it's valid for all tiles stored within the same directory tree, all siblings and children). +- given a tile (either rf or tb), the lookup of the source attribution table is first in the same directory and then in all parrent dirs, until a source_attribution_table.ard is found (or a failure is thrown). + +## alpine maps raster store format +this is the binary format used to store rf and tb tiles. both share the same basic tile format, but use it in different ways. +- The hierarchy is a Web Mercator (EPSG:3857) quadtree keyed by tile IDs (radix::tile::ID, https://docs.maptiler.com/google-maps-coordinates-tile-bounds-projection/). +- stored tiles have a resolution of 4096x4096 pixels +- Every stored pixel has one data value (can be vector type) and one source attribution index. +- the source attribution index is stored as uint16, and indexes into a global source attribution table (see above) +- data is stored in radix::Raster objects (one for source attribution index, one for the actual data), i.e. template struct raster_store::Tile { radix::Raster data; radix::Raster source_attribution; }; +- the file ending is .amort (AlpineMapsOrg raster tile), it is serialised used the principles outlined below. +- each snapshot stores its index as `raster_store.index`. +- the default `zoom/x/y_google` layout stores a tile as + `//.amort`, directly below the snapshot root. + +## raster-fundamentalis (rf) format +raster-fundamentalis is our authoritative raster-store, containing only the data and no overviews / downsampled version. +- unlike a tile pyramid, not every level is occupied (there is no downsampled versions of the data). +- Coarse physical tiles (e.g. zoom level 10) may coexist with more accurate descendants (e.g. zoom level 15). +- the raster-fundamentalis builder is implemented in src/rf-builder/*, it consumes raw gdal data. + +## Tile-base Format (tb) +tile-base is a hierarchy build from raster-fundamentalis, containing all data and its overviews / downsampled versions. it is used directly by the tile-server to generate tiles at the requested resolution and format. +- every level is occupied, and every level selects an adequate data source +- the tile-base builder is implemented in src/tb-builder, it consumes rf + +### tile-server +- should generate tiles of requested resolution and pixel type (vertex|area) on the fly +- requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending +- live in src/tile-server/* +- the delivery tile format is not yet defined + +## serialization / deserialization envelope and versioning +- `zpp::bits` serialises C++ objects to byte streams and deserialises them again. +- The envelope is generic and is not specific to the raster store. +- We serialise objects with `zpp::bits` in two levels. + - The first level is an aggregate with the following fields, in this order: + - `uint64 magic`, always `F5FBD3EF919428CA`, identifying this envelope format; + - `string class_name`, identifying the payload type; + - `uint32 class_version`, identifying the versioned payload type; + - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `Crc32c` or `None`; + - `string checksum`, empty when no external checksum is used, otherwise the CRC-32C value; + - `CompressionAlgorithm compression_algorithm`, default `ZstdBestCompressionWithChecksum`, alternatively `None`; + - `uint64 uncompressed_size`, the exact size of the uncompressed second level; + - `Bytes compressed_data`, containing the second level. + - The second level is a compressed byte vector. It is deserialised directly into the selected versioned payload class. +- The magic is shared by all payload types. `class_name` distinguishes payload types. An incompatible future envelope layout requires a new magic. +- data structs are stored in versioned namespaces, e.g.: + `raster_store::v1::Tile` +- outside the versioned namespace, there is a using declaration for the newest version +- outside the versioned namespace, there is a serialization wrapper function taking only the newest version +- outside the versioned namespace, there is a deserialization function, taking a byte stream, and returning the newest version (convert to the newest version, if the payload encodes an older version) +- Newer versions provide a static `from_previous` function, e.g. `v2::Tile::from_previous(v1::Tile)`. Static conversion functions keep the payload types aggregates, allowing `zpp::bits` to serialise them without per-type serialisation declarations. A conversion trail upgrades v1 to v2 and then v3. +- `Version` pairs version numbers with payload types. `PayloadSchema` defines the class name, supported versions, latest type, and conversion trail. +- The generic serialisation function receives the schema and version as template parameters. Deserialisation reads `class_version`, deserialises exactly that payload type, and follows the conversion trail to the latest type. It does not speculatively try other payload versions. +- we have clear fails if + - the classname or magic is wrong, or the version is unsupported + - if the checksum check fails + - if the compression algorithm is missing or unsupported. + - deserialization fails +- we fail by returning an unexpected in these cases +- Compression uses libzstd at its best compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. `ZstdBestCompressionWithChecksum` writes an embedded zstd frame checksum, which libzstd verifies while decompressing. +- `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. +- `Crc32c` is an external CRC-32C (Castagnoli) checksum of the complete uncompressed serialised payload. It uses the reflected polynomial `82F63B78`, an initial value of `FFFFFFFF`, and a final XOR of `FFFFFFFF`. The checksum string contains exactly eight lowercase hexadecimal characters. The empty payload has checksum `00000000`; the ASCII test vector `123456789` has checksum `e3069283`. CRC-32C detects accidental corruption but is not cryptographic. +- `checked_decompress` accepts a maximum decompressed size, defaulting to 1 GiB. It uses a size reported by the compression format when available, otherwise it uses the maximum as its allocation bound and shrinks the result to the produced size. +- Envelope deserialisation passes `uncompressed_size` as that maximum and requires the produced size to match it exactly. A mismatch is a decompression failure. A declared size above the caller's limit or the hard 1 GiB limit is a size-limit failure. +- `None` compression may be paired with `None` or `Crc32c`. `ZstdBestCompressionWithChecksum` may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with `ZstdBestCompressionWithChecksum`. +- `None` and `HandledByCompressionLib` require an empty checksum string. `Crc32c` requires its canonical eight-character checksum. Deserialisation verifies it after bounded decompression and before parsing the payload with `zpp::bits`; envelope deserialisation also requires the resulting size to exactly match `uncompressed_size`. +- Decompression never allocates or produces output larger than 1 GiB. + + +## to be defined +- semantic layer kind (height scalars, linear colour, gamma-encoded colour, categorical values), should be used for filtering +- human readable description? +- enumeration of layers etc? + +For now, these things will be defined in code, we will have one terrain-elevation, one surface-elevation and one ortho-photo store. later probably also a percentage store (for the snow layer) diff --git a/docs/raster-store/terminology.md b/docs/raster-store/terminology.md new file mode 100644 index 00000000..eb98976b --- /dev/null +++ b/docs/raster-store/terminology.md @@ -0,0 +1,32 @@ +# Terminology +** attribution raster ** +: A square data matrix (image), containing indices into the source-attribution table + +** source-attribution table ** +: A table of data sources, including meta data like resolution, dates etc. + +**raster-fundamentalis (rf)** +: The authoritative raster store + +**tile-base (tb)** +: basically rf with overviews, used to generate derived tiles + +**Vertex pixel** +: A generated value located on a grid vertex. Height tiles for mesh generation + require vertex pixels, including shared boundary positions. + +**Area pixel** +: A generated value associated with a raster cell. Ordinary texture outputs + use area pixels whose cell boundaries align with tile boundaries. + +The terms vertex pixel and area pixel describe generator outputs. They do not +assert how an original sensor or source raster produced its values. + + +**tile** +: A chunk of raster data with an tile ID. Rf is a store for tiles, tb is a store for tiles, and we generate derived tiles / output tiles for the client. + +**Derived / output tile** +: A filtered and encoded output tile generated from the tile base store. +It may have different dimensions, sampling placement, encoding, and +provenance granularity from a store chunk. diff --git a/docs/raster-store/todo.md b/docs/raster-store/todo.md new file mode 100644 index 00000000..fe9bf14a --- /dev/null +++ b/docs/raster-store/todo.md @@ -0,0 +1,5 @@ +# Raster store TODO + +- Consider extending `sf::validate_index()` beyond rejecting `Inner` after + additional SF invariants and their required error reporting are defined. + This is not part of the shared-store refactor. diff --git a/docs/tile-downloader/download.sh b/docs/tile-downloader/download.sh index b375ad75..dcd168f8 100755 --- a/docs/tile-downloader/download.sh +++ b/docs/tile-downloader/download.sh @@ -3,7 +3,7 @@ build_path="/home/madam/Documents/work/tuw/alpinemaps/build-terrain-builder-Desktop_Qt_6_2_3_GCC_64bit-Release/src" while read p; do - read zoom row col <<<${p//[^0-9]/ } - echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --row ${row} --col ${col} --verbosity 0&" -# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --row ${row} --col ${col} --verbosity 0& + read zoom x y <<<${p//[^0-9]/ } + echo -e "nice -10 \$build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --verbosity 0&" +# nice -10 $build_path/tile-downloader --provider basemap --zoom ${zoom} --x ${x} --y ${y} --verbosity 0& done build_level( const auto save_result = ctx.output_storage.save(target, *result); if (save_result) { if (debug_storage) { - debug_storage->save(target, clustering_to_mesh(result->clustering)); + const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); + if (!debug_save_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save debug mesh for node {}: {}", target, debug_save_result.error()); + } } saved_ids.push_back(target); } else { diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h index 769e567f..2ccee0e9 100644 --- a/src/dag_builder/encoded.h +++ b/src/dag_builder/encoded.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -165,14 +165,14 @@ auto serialize(Archive &archive, Clustering &clustering) { } } -inline tl::expected +inline std::expected save_clustering(const Clustering &clustering, const std::filesystem::path &path, const bool make_dirs = true) { return ::io::write_to_path(clustering, path, make_dirs); } -inline tl::expected +inline std::expected load_clustering(const std::filesystem::path &path) { return ::io::read_from_path(path); } diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index d640d37f..acce9d48 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -44,11 +44,15 @@ int main(int argc, char **argv) { } dag::build_levels(input_storage, output_storage, options, args.level_range); - output_storage.save_index(); + const auto index_result = output_storage.save_index(); + if (!index_result.has_value()) { + LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); + return EXIT_FAILURE; + } return EXIT_SUCCESS; } catch (const std::exception &e) { LOG_ERROR("{}", e.what()); return EXIT_FAILURE; } -} \ No newline at end of file +} diff --git a/src/dag_builder/simplify.h b/src/dag_builder/simplify.h index 1ab64f74..e641994b 100644 --- a/src/dag_builder/simplify.h +++ b/src/dag_builder/simplify.h @@ -264,7 +264,7 @@ inline Clustering simplify( uvs[new_index] = original_cluster.uvs[original_index]; } } - + // Make error absolute and combine with the input cluster's error const double absolute_error = result.relative_error * max_extents; const double combined_error = detail::combine_error(options.error_mode, original_cluster.absolute_error, absolute_error); diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index 66424881..9b5e6120 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "octree/Id.h" @@ -33,7 +33,7 @@ class ThreadSafeStorage { return std::move(this->_storage); } - tl::expected load(const octree::Id &id) const { + std::expected load(const octree::Id &id) const { std::shared_lock lock(this->_mutex); return this->_storage.load(id); } @@ -47,7 +47,7 @@ class ThreadSafeStorage { return this->_storage.base_path(); } - tl::expected save(const octree::Id &id, const value_type &value) const { + std::expected save(const octree::Id &id, const value_type &value) const { std::unique_lock lock(this->_mutex); return this->_storage.save(id, value); } diff --git a/src/dag_builder/utils.h b/src/dag_builder/utils.h index 1a01504c..81ca5ec7 100644 --- a/src/dag_builder/utils.h +++ b/src/dag_builder/utils.h @@ -147,7 +147,7 @@ inline mesh::Simple manifold_clustering_to_mesh(const Clustering &clustering, co mesh.triangles.push_back(local_triangle + base_vertex); } } - + DEBUG_ASSERT(mesh::is_manifold(mesh)); if (debug_texture) { diff --git a/src/mesh_convert/main.cpp b/src/mesh_convert/main.cpp index ef258319..a396ce50 100644 --- a/src/mesh_convert/main.cpp +++ b/src/mesh_convert/main.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io.h" @@ -11,7 +11,7 @@ void run(const cli::Args& args) { LOG_INFO("Loading input mesh..."); - const tl::expected load_result = mesh::io::load_from_path(args.input_path); + const std::expected load_result = mesh::io::load_from_path(args.input_path); if (!load_result.has_value()) { LOG_ERROR("Failed to load mesh: {}", load_result.error().description()); return; @@ -30,7 +30,7 @@ void run(const cli::Args& args) { } LOG_INFO("Writing output mesh..."); - const tl::expected save_result = mesh::io::save_to_path(mesh, args.output_path); + const std::expected save_result = mesh::io::save_to_path(mesh, args.output_path); if (!save_result.has_value()) { LOG_ERROR("Failed to save mesh: {}", save_result.error().description()); return; diff --git a/src/sf_builder/CMakeLists.txt b/src/sf_builder/CMakeLists.txt index c50cae1c..cfdabb70 100644 --- a/src/sf_builder/CMakeLists.txt +++ b/src/sf_builder/CMakeLists.txt @@ -3,7 +3,7 @@ add_library(sfbuilderlib mesh_builder.cpp ) target_include_directories(sfbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog tl_expected) +target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog) add_executable(sf-builder main.cpp) target_link_libraries(sf-builder PRIVATE sfbuilderlib CLI11::CLI11) diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index b285134d..e7216346 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -70,7 +70,6 @@ radix::geometry::Aabb3d parse_bounds_from_values(const std::vector &data radix::geometry::Aabb3d parse_bounds_from_tile( const std::vector &data, - radix::tile::Scheme scheme, const OGRSpatialReference &srs) { // Determine the correct Grid type based on SRS @@ -87,7 +86,7 @@ radix::geometry::Aabb3d parse_bounds_from_tile( const uint32_t zoom_level = data[0]; const glm::uvec2 tile_coords(data[1], data[2]); - const radix::tile::Id target_tile(zoom_level, tile_coords, scheme); + const radix::tile::Id target_tile(zoom_level, tile_coords); return extend_bounds_to_3d(grid->srsBounds(target_tile, false)); } @@ -118,7 +117,6 @@ radix::geometry::Aabb3d parse_target_bounds( const std::vector &bounds_data, const std::vector &node_data, const std::vector &tile_data, - radix::tile::Scheme tile_scheme, OGRSpatialReference &srs) { if (!bounds_data.empty()) { @@ -126,7 +124,7 @@ radix::geometry::Aabb3d parse_target_bounds( } if (!tile_data.empty()) { - return parse_bounds_from_tile(tile_data, tile_scheme, srs); + return parse_bounds_from_tile(tile_data, srs); } if (!node_data.empty()) { @@ -223,7 +221,6 @@ int run(std::span args) { std::vector target_tile_data; std::vector target_node_data; std::string target_srs_input; - radix::tile::Scheme target_tile_scheme; auto *target = single->add_option_group("target"); target->add_option("--bounds", target_bounds_data, "Target bounds for the reference mesh as \"{xmin} {width} {ymin} {height} [{zmin} {depth}]\"") @@ -237,15 +234,6 @@ int run(std::span args) { single->add_option("--output", output_path, "Output path were the mesh is written to (.terrain, .gltf or .glb)") ->required(); - std::map scheme_str_map{ - {"slippymap", radix::tile::Scheme::SlippyMap}, - {"google", radix::tile::Scheme::SlippyMap}, - {"tms", radix::tile::Scheme::Tms}}; - single->add_option("--scheme", target_tile_scheme, "Tile scheme") - ->default_val(radix::tile::Scheme::SlippyMap) - ->needs("--tile") - ->transform(CLI::CheckedTransformer(scheme_str_map, CLI::ignore_case)); - single->add_option("--srs", target_srs_input, "EPSG code of the srs of the target bounds or id"); single->callback([&]() { if (target_srs_input.empty()) { @@ -290,14 +278,14 @@ int run(std::span args) { std::unique_ptr tile_provider; if (texture_base_path.has_value()) { - BasemapSchemeTilePathProvider basemap_provider(texture_base_path.value()); + GoogleMapboxTilePathProvider basemap_provider(texture_base_path.value()); if (min_texture_level.has_value() || max_texture_level.has_value()) { - tile_provider = std::make_unique>( + tile_provider = std::make_unique>( std::move(basemap_provider), min_texture_level, max_texture_level); } else { - tile_provider = std::make_unique(std::move(basemap_provider)); + tile_provider = std::make_unique(std::move(basemap_provider)); } } @@ -307,7 +295,6 @@ int run(std::span args) { target_bounds_data, target_node_data, target_tile_data, - target_tile_scheme, target_srs); terrainbuilder::build_and_save_patch( diff --git a/src/sf_builder/mesh_builder.cpp b/src/sf_builder/mesh_builder.cpp index dc2b19b2..be440a94 100644 --- a/src/sf_builder/mesh_builder.cpp +++ b/src/sf_builder/mesh_builder.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -15,7 +16,7 @@ #include "mesh/SimpleMesh.h" #include "mesh/cleanup.h" #include "mesh_builder.h" -#include "raster.h" +#include #include "raw_dataset_reader.h" #include "srs.h" #include "mesh/clip.h" @@ -51,7 +52,7 @@ glm::dvec2 apply_transform(std::array transform, const glm::tvec2 return result; } -glm::dvec3 convert_pixel_to_vertex(const float height, const raster::Coords pixel_coords, const RawDatasetReader& reader, const PixelBounds& pixel_bounds) { +glm::dvec3 convert_pixel_to_vertex(const float height, const glm::uvec2 pixel_coords, const RawDatasetReader& reader, const PixelBounds& pixel_bounds) { const glm::dvec2 point_offset_in_raster(0.5); // Convert pixel coordinates into a point in the dataset's srs. const glm::dvec2 coords_raster_relative = glm::dvec2(pixel_coords) + point_offset_in_raster; const glm::dvec2 coords_raster_absolute = coords_raster_relative + glm::dvec2(pixel_bounds.min); @@ -59,9 +60,9 @@ glm::dvec3 convert_pixel_to_vertex(const float height, const raster::Coords pixe return coords_source; } -SimpleMesh meshify(const raster::Raster& source_points, const raster::Mask& mask) { +SimpleMesh meshify(const radix::Raster& source_points, const radix::RasterMask& mask) { // Compact the vertex grid into a list of valid ones. - const size_t valid_vertex_count = std::reduce(mask.begin(), mask.end(), 0); + const size_t valid_vertex_count = std::reduce(mask.begin(), mask.end(), size_t(0)); // Check if we even have any valid vertices. Can happen if all of the region is padding. if (valid_vertex_count == 0) { return SimpleMesh(); @@ -70,11 +71,15 @@ SimpleMesh meshify(const raster::Raster& source_points, const raster std::vector positions; positions.reserve(valid_vertex_count); - const raster::Raster vertex_index_map = raster::transform(source_points, mask, [&](const glm::dvec3 &point) -> size_t { + auto vertex_index_map_result = radix::raster::transform(source_points, mask, [&](const glm::dvec3& point) -> size_t { const size_t index = positions.size(); positions.push_back(point); return index; }); + DEBUG_ASSERT(vertex_index_map_result.has_value()); + if (!vertex_index_map_result.has_value()) + return {}; + const auto vertex_index_map = std::move(*vertex_index_map_result); DEBUG_ASSERT(positions.size() == valid_vertex_count); // Allocate triangle vector @@ -82,13 +87,13 @@ SimpleMesh meshify(const raster::Raster& source_points, const raster std::vector triangles; triangles.reserve(max_triangle_count); - for (size_t y = 0; y < source_points.height() - 1; y++) { - for (size_t x = 0; x < source_points.width() - 1; x++) { - const std::array quad { - raster::Coords{x, y}, - raster::Coords{x + 1, y}, - raster::Coords{x + 1, y + 1}, - raster::Coords{x, y + 1}}; + for (unsigned y = 0; y < source_points.height() - 1; y++) { + for (unsigned x = 0; x < source_points.width() - 1; x++) { + const std::array quad { + glm::uvec2{x, y}, + glm::uvec2{x + 1, y}, + glm::uvec2{x + 1, y + 1}, + glm::uvec2{x, y + 1}}; for (uint32_t i = 0; i < 4; i++) { const auto& v0 = quad[i]; @@ -143,7 +148,7 @@ radix::geometry::Aabb3d extend_bounds_to_3d(radix::geometry::Aabb2d bounds2d) { } } -tl::expected build_reference_mesh_tile( +std::expected build_reference_mesh_tile( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &tile_srs, const radix::tile::SrsBounds &tile_bounds, @@ -151,7 +156,7 @@ tl::expected build_reference_mesh_tile( return build_reference_mesh_patch(dataset, mesh_srs, tile_srs, extend_bounds_to_3d(tile_bounds), texture_srs, texture_bounds); } -tl::expected build_reference_mesh_patch( +std::expected build_reference_mesh_patch( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &clip_srs, const radix::geometry::Aabb3d &clip_bounds, @@ -172,35 +177,39 @@ tl::expected build_reference_mesh_patch( radix::geometry::Aabb2i pixel_bounds = reader.transform_srs_bounds_to_pixel_bounds(target_bounds_in_source_srs); add_border_to_aabb(pixel_bounds, Border(1)); LOG_TRACE("Reading pixels [({}, {})-({}, {})] from dataset", pixel_bounds.min.x, pixel_bounds.min.y, pixel_bounds.max.x, pixel_bounds.max.y); - const std::optional read_result = reader.read_data_in_pixel_bounds_clamped(pixel_bounds); - if (!read_result.has_value() || read_result->size() == 0) { - return tl::unexpected(BuildMeshError::OutOfBounds); + auto read_result = reader.read_data_in_pixel_bounds_clamped(pixel_bounds); + if (!read_result.has_value() || read_result->buffer().empty()) { + return std::unexpected(BuildMeshError::OutOfBounds); } - const raster::HeightMap height_map = read_result.value(); + const radix::Raster height_map = std::move(*read_result); LOG_TRACE("Finding valid pixels"); const float no_data_value = reader.get_no_data_value(); - const raster::Mask valid_mask = raster::transform(height_map, [=](const float height) { + const radix::RasterMask valid_mask = radix::raster::transform(height_map, [=](const float height) { return height != no_data_value; }); LOG_TRACE("Transforming pixels to vertices"); - const raster::Raster source_points = raster::transform(height_map, valid_mask, [&](const float height, const raster::Coords& coords) { + auto source_points_result = radix::raster::transform(height_map, valid_mask, [&](const float height, const glm::uvec2& coords) { return convert_pixel_to_vertex(height, coords, reader, pixel_bounds); }); + DEBUG_ASSERT(source_points_result.has_value()); + if (!source_points_result.has_value()) + return std::unexpected(BuildMeshError::EmptyRegion); + const auto source_points = std::move(*source_points_result); LOG_TRACE("Generating triangles"); SimpleMesh mesh_in_source_srs = meshify(source_points, valid_mask); // Check if we even have any valid vertices. Can happen if all of the region is padding. if (mesh_in_source_srs.vertex_count() == 0 || mesh_in_source_srs.face_count() == 0) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } // Fast check if all vertices will be clipped const radix::geometry::Aabb3d actual_source_bounds = calculate_bounds(mesh_in_source_srs); const radix::geometry::Aabb3d approx_clip_bounds = srs::encompassing_bounds_transfer(source_srs, clip_srs, actual_source_bounds); if (!radix::geometry::intersect(approx_clip_bounds, clip_bounds)) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } LOG_TRACE("Clipping mesh based on target bounds"); @@ -208,7 +217,7 @@ tl::expected build_reference_mesh_patch( SimpleMesh clipped_mesh = mesh::clip_on_bounds(mesh_in_clip_srs, clip_bounds); // Check if there are any vertices left if (clipped_mesh.vertex_count() == 0 || clipped_mesh.face_count() == 0) { - return tl::unexpected(BuildMeshError::EmptyRegion); + return std::unexpected(BuildMeshError::EmptyRegion); } // TODO: move this to another function? diff --git a/src/sf_builder/mesh_builder.h b/src/sf_builder/mesh_builder.h index acf3e4b8..2979042d 100644 --- a/src/sf_builder/mesh_builder.h +++ b/src/sf_builder/mesh_builder.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include "Dataset.h" #include "srs.h" @@ -17,7 +17,7 @@ enum class BuildMeshError { std::ostream &operator<<(std::ostream &os, BuildMeshError error); /// Builds a mesh from the given height dataset. -tl::expected build_reference_mesh_patch( +std::expected build_reference_mesh_patch( Dataset &dataset, const OGRSpatialReference &mesh_srs, const OGRSpatialReference &clip_srs, const radix::geometry::Aabb3d &clip_bounds, diff --git a/src/sf_builder/raster.h b/src/sf_builder/raster.h deleted file mode 100644 index 5b30794e..00000000 --- a/src/sf_builder/raster.h +++ /dev/null @@ -1,148 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -namespace raster { - -using Index = size_t; -using Coords = glm::vec<2, size_t>; - -template -class Raster { -public: - Raster() = default; - Raster(Index width, Index height) - : _width(width), _height(height), _data(width * height) { - } - - [[nodiscard]] Index width() const { - return this->_width; - } - [[nodiscard]] Index height() const { - return this->_height; - } - [[nodiscard]] decltype(auto) pixel(const Coords &coords) { - DEBUG_ASSERT(coords.x < this->_width); - DEBUG_ASSERT(coords.y < this->_height); - const Index pixel_index = this->index(coords); - if constexpr (std::is_same_v) { - return static_cast(this->_data[pixel_index]); - } else { - return static_cast(this->_data[pixel_index]); - } - } - [[nodiscard]] decltype(auto) pixel(const Coords &coords) const { - DEBUG_ASSERT(coords.x < this->_width); - DEBUG_ASSERT(coords.y < this->_height); - const Index pixel_index = this->index(coords); - if constexpr (std::is_same_v) { - return static_cast(this->_data[pixel_index]); - } else { - return static_cast(this->_data[pixel_index]); - } - } - - [[nodiscard]] Index index(const Coords& coords) const { - return coords.y * this->_width + coords.x; - } - - [[nodiscard]] T *data() { - return this->_data.data(); - } - [[nodiscard]] const T *data() const { - return this->_data.data(); - } - - [[nodiscard]] Index size() const { - return this->_data.size(); - } - [[nodiscard]] auto begin() { - return this->_data.begin(); - } - [[nodiscard]] auto end() { - return this->_data.end(); - } - [[nodiscard]] auto begin() const { - return this->_data.begin(); - } - [[nodiscard]] auto end() const { - return this->_data.end(); - } - -private: - unsigned _width = 0; - unsigned _height = 0; - std::vector _data; -}; - -using Mask = Raster; -using HeightMap = Raster; - -template -concept TransformFn = requires(F f, In in) { - { f(in) }; -}; - -template -concept TransformFnWithCoords = requires(F f, In in, Coords coord) { - { f(in, coord) }; -}; - -template F> -[[nodiscard]] auto transform(const Raster &input, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)))); - Raster output(input.width(), input.height()); - std::transform(input.begin(), input.end(), output.begin(), std::forward(f)); - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)), Coords(0, 0))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - output.pixel(coords) = f(input.pixel(coords), coords); - } - } - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, const Mask &mask, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - if (mask.pixel(coords)) { - output.pixel(coords) = f(input.pixel(coords)); - } - } - } - return output; -} - -template F> -[[nodiscard]] auto transform(const Raster &input, const Mask &mask, F &&f) { - using Out = decltype(f(input.pixel(Coords(0, 0)), Coords(0, 0))); - Raster output(input.width(), input.height()); - for (Index y = 0; y < input.height(); ++y) { - for (Index x = 0; x < input.width(); ++x) { - Coords coords(x, y); - if (mask.pixel(coords)) { - output.pixel(coords) = f(input.pixel(coords), coords); - } - } - } - return output; -} -} diff --git a/src/sf_builder/raw_dataset_reader.h b/src/sf_builder/raw_dataset_reader.h index be91a44d..096526be 100644 --- a/src/sf_builder/raw_dataset_reader.h +++ b/src/sf_builder/raw_dataset_reader.h @@ -14,7 +14,7 @@ #include #include "log.h" -#include "raster.h" +#include namespace terrainbuilder { @@ -53,7 +53,7 @@ class RawDatasetReader { } // TODO: support reading other data types - std::optional read_data_in_pixel_bounds(const radix::geometry::Aabb2i& bounds) { + std::optional> read_data_in_pixel_bounds(const radix::geometry::Aabb2i& bounds) { DEBUG_ASSERT(glm::all(glm::greaterThanEqual(bounds.min, glm::ivec2(0)))); DEBUG_ASSERT(glm::all(glm::greaterThanEqual(bounds.max, glm::ivec2(0)))); DEBUG_ASSERT(glm::all(glm::lessThan(bounds.min, glm::ivec2(this->dataset_size())))); @@ -63,7 +63,7 @@ class RawDatasetReader { GDALRasterBand *height_band = this->dataset->GetRasterBand(1); // non-owning pointer // Initialize the HeightData for reading - raster::HeightMap height_data(bounds.width(), bounds.height()); + radix::Raster height_data(glm::uvec2(bounds.width(), bounds.height())); if (bounds.width() == 0 || bounds.height() == 0) { LOG_WARN("Target dataset bounds are empty"); return height_data; @@ -72,7 +72,7 @@ class RawDatasetReader { // Read data from the heights band into heights_data const int32_t read_result = height_band->RasterIO( GF_Read, bounds.min.x, bounds.min.y, bounds.width(), bounds.height(), - static_cast(height_data.data()), bounds.width(), bounds.height(), GDT_Float32, 0, 0); + static_cast(height_data.buffer().data()), bounds.width(), bounds.height(), GDT_Float32, 0, 0); if (read_result != CE_None) { const char * message = CPLGetLastErrorMsg(); @@ -82,7 +82,7 @@ class RawDatasetReader { return height_data; } - std::optional read_data_in_pixel_bounds_clamped(radix::geometry::Aabb2i &bounds) { + std::optional> read_data_in_pixel_bounds_clamped(radix::geometry::Aabb2i &bounds) { const auto original_bounds = bounds; const glm::ivec2 max_in_bounds = glm::ivec2(this->dataset_size()) - glm::ivec2(1); @@ -97,13 +97,13 @@ class RawDatasetReader { if (bounds.width() == 0 || bounds.height() == 0) { LOG_WARN("Target dataset bounds are empty (clamped)"); - return raster::HeightMap(0, 0); + return radix::Raster(); } return this->read_data_in_pixel_bounds(bounds); } - std::optional read_data_in_srs_bounds(const radix::tile::SrsBounds &bounds) { + std::optional> read_data_in_srs_bounds(const radix::tile::SrsBounds &bounds) { // Transform the SrsBounds to pixel space radix::geometry::Aabb2i pixel_bounds = this->transform_srs_bounds_to_pixel_bounds(bounds); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index ddeb01e6..083ab6ba 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include @@ -71,7 +70,7 @@ std::optional build_patch( std::chrono::high_resolution_clock::time_point start; start = std::chrono::high_resolution_clock::now(); LOG_INFO("Building mesh..."); - tl::expected mesh_result = build_reference_mesh_patch( + std::expected mesh_result = build_reference_mesh_patch( dataset, mesh_srs, target_bounds_srs, target_bounds, @@ -279,6 +278,7 @@ void build_all_patches( logger->set_level(new_level); } + tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; if (!overwrite_existing && storage.has(node)) { @@ -303,16 +303,33 @@ void build_all_patches( if (mesh_result.has_value()) { const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); - storage.save(node, mesh); + const auto save_result = storage.save(node, mesh); + if (!save_result.has_value()) { + LOG_ERROR("Failed to save mesh for node {}: {}", node, save_result.error()); + progress.task_finished(); + context.cancel_group_execution(); + return; + } } progress.task_finished(); - }); + }, context); // Restore original level logger->set_level(original_level); + if (context.is_group_execution_cancelled()) { + progress_thread.request_stop(); + } progress_thread.join(); - storage.save_or_create_index(); + + if (context.is_group_execution_cancelled()) { + LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); + } + + const auto index_result = storage.save_or_create_index(); + if (!index_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", storage.base_path(), index_result.error()); + } } } diff --git a/src/sf_builder/texture_assembler.h b/src/sf_builder/texture_assembler.h index bff6c018..8de959f1 100644 --- a/src/sf_builder/texture_assembler.h +++ b/src/sf_builder/texture_assembler.h @@ -196,7 +196,6 @@ inline TargetImageRegion calculate_target_image_region( const radix::tile::Id root_tile, const glm::uvec2 tile_image_pixel_size, const uint32_t max_zoom_level) { - tile = tile.to(radix::tile::Scheme::SlippyMap); const size_t relative_zoom_level = tile.zoom_level - root_tile.zoom_level; const glm::uvec2 tile_size_factor = glm::uvec2(std::pow(2, max_zoom_level - tile.zoom_level)); const glm::uvec2 tile_size = tile_image_pixel_size * tile_size_factor; @@ -469,7 +468,7 @@ inline std::optional assemble_texture_from_tiles( // Start by transforming the input bounds into the srs the tiles are in. const radix::tile::SrsBounds encompassing_bounds = srs::encompassing_bounds_transfer(target_srs, grid.getSRS(), target_bounds); // Then we find the smallest tile (id) that encompasses these bounds. - radix::tile::Id smallest_encompassing_tile = grid.findSmallestEncompassingTile(encompassing_bounds).value().to(radix::tile::Scheme::SlippyMap); + radix::tile::Id smallest_encompassing_tile = grid.findSmallestEncompassingTile(encompassing_bounds).value(); LOG_TRACE("Smallest encompassing tile for texture bounds is {}", radix::tile::to_string(smallest_encompassing_tile)); if (max_zoom.has_value() && smallest_encompassing_tile.zoom_level > max_zoom.value()) { diff --git a/src/sf_builder/tile_provider.h b/src/sf_builder/tile_provider.h index da90c0e2..c709536d 100644 --- a/src/sf_builder/tile_provider.h +++ b/src/sf_builder/tile_provider.h @@ -6,6 +6,8 @@ #include #include +#include "tile_path.h" + class TileProvider { public: virtual ~TileProvider() = default; @@ -52,16 +54,13 @@ class StaticTileProvider : public TileProvider { public: std::unordered_map tiles; - StaticTileProvider(const std::unordered_map& tiles) { - // TODO: remove this once the == operator of tile::Id is updated. - for (const auto& tile : tiles) { - const radix::tile::Id tile_id = tile.first.to(radix::tile::Scheme::SlippyMap); - this->tiles[tile_id] = tile.second; - } + StaticTileProvider(const std::unordered_map& tiles) + : tiles(tiles) + { } virtual std::optional get_tile(const radix::tile::Id tile_id) const override { - const auto tile = this->tiles.find(tile_id.to(radix::tile::Scheme::SlippyMap)); + const auto tile = this->tiles.find(tile_id); if (tile != this->tiles.end()) { return tile->second; } else { @@ -70,7 +69,7 @@ class StaticTileProvider : public TileProvider { } virtual bool has_tile(const radix::tile::Id tile_id) const override { - return this->tiles.find(tile_id.to(radix::tile::Scheme::SlippyMap)) != this->tiles.cend(); + return this->tiles.find(tile_id) != this->tiles.cend(); } }; @@ -139,13 +138,13 @@ class ZoomRangeTileProvider final : public TileProvider { uint32_t _max_zoom; }; -class BasemapSchemeTilePathProvider : public TilePathProvider { +class GoogleMapboxTilePathProvider : public TilePathProvider { public: - BasemapSchemeTilePathProvider(std::filesystem::path base_path) + GoogleMapboxTilePathProvider(std::filesystem::path base_path) : base_path(base_path) {} std::optional get_tile_path(const radix::tile::Id tile_id) const override { - return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.y) / (std::to_string(tile_id.coords.x) + ".jpeg"); + return google_tile_path(base_path, tile_id, ".jpeg"); } private: diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 6f8e4b40..286f473b 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -114,7 +114,10 @@ inline void cut_dataset( const bool keep_inside) { Context ctx(input, output, octree::Space::earth(), keep_inside); cut_node(ctx, octree::Id::root(), mask); - output.save_or_create_index(); + const auto index_result = output.save_or_create_index(); + if (!index_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output.base_path(), index_result.error()); + } } inline void cut_dataset( diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index fa6cd8af..8ceb8900 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -91,7 +92,9 @@ namespace { enum class LoadErrorKind { UnsupportedFormat, FileNotFound, - EmptySource + EmptySource, + InvalidGeometry, + UnsupportedSpatialReference }; class LoadError { @@ -118,6 +121,10 @@ class LoadError { return "file not found"; case LoadErrorKind::EmptySource: return "empty input source"; + case LoadErrorKind::InvalidGeometry: + return "invalid geometry"; + case LoadErrorKind::UnsupportedSpatialReference: + return "unsupported spatial reference"; default: return "unknown error"; } @@ -132,6 +139,58 @@ class LoadError { }; namespace { +constexpr double simplification_tolerance_metres = 0.1; + +std::optional simplification_tolerance(const OGRSpatialReference &srs) { + if (srs.IsProjected()) { + const double metres_per_unit = srs.GetLinearUnits(); + if (metres_per_unit > 0) { + return simplification_tolerance_metres / metres_per_unit; + } + } + + if (srs.IsGeographic()) { + OGRErr error = OGRERR_NONE; + const double semi_major_axis_metres = srs.GetSemiMajor(&error); + const double radians_per_unit = srs.GetAngularUnits(); + if (error == OGRERR_NONE && semi_major_axis_metres > 0 && radians_per_unit > 0) { + return simplification_tolerance_metres / semi_major_axis_metres / radians_per_unit; + } + } + + return std::nullopt; +} + +uint64_t point_count(const OGRGeometry &geometry) { + const OGRwkbGeometryType geometry_type = wkbFlatten(geometry.getGeometryType()); + if (geometry_type == wkbPolygon) { + const OGRPolygon *polygon = geometry.toPolygon(); + uint64_t count = polygon->getExteriorRing()->getNumPoints(); + for (int i = 0; i < polygon->getNumInteriorRings(); ++i) { + count += polygon->getInteriorRing(i)->getNumPoints(); + } + return count; + } + + if (geometry_type == wkbMultiPolygon || geometry_type == wkbGeometryCollection) { + const OGRGeometryCollection *collection = geometry.toGeometryCollection(); + uint64_t count = 0; + for (int i = 0; i < collection->getNumGeometries(); ++i) { + count += point_count(*collection->getGeometryRef(i)); + } + return count; + } + + return 0; +} + +std::unique_ptr simplify_geometry( + const OGRGeometry &geometry, + const double tolerance +) { + return std::unique_ptr(geometry.SimplifyPreserveTopology(tolerance)); +} + std::optional convert_ring(const OGRLinearRing &ring, bool is_outer) { uint32_t num_points = ring.getNumPoints(); if (ring.get_IsClosed()) { @@ -272,21 +331,9 @@ glm::dvec2 calculate_radius_range(const std::span meshes) { } // namespace -inline tl::expected load_referenced_from_dataset(Dataset& mask_dataset) { +inline std::expected load_referenced_from_dataset(Dataset& mask_dataset) { GDALDataset *dataset = mask_dataset.gdalDataset(); - MultipolygonWithHoles2 polygons; - for (auto &&feature_layer_pair : dataset->GetFeatures()) { - OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); - DEBUG_ASSERT(geometry); - process_geometry(*geometry, polygons); - } - - if (polygons.is_empty()) { - LOG_ERROR("No valid polygons found in mask dataset '{}'", mask_dataset.name()); - return tl::unexpected(LoadErrorKind::EmptySource); - } - OGRSpatialReference srs; // TODO: remove this try catch try { @@ -297,6 +344,35 @@ inline tl::expected load_referenced_from_datas // srs.SetAxisMappingStrategy(OAMS_AUTHORITY_COMPLIANT); } + const std::optional tolerance = simplification_tolerance(srs); + if (!tolerance) { + LOG_ERROR("Cannot express the {} m mask simplification tolerance in the source SRS", + simplification_tolerance_metres); + return std::unexpected(LoadErrorKind::UnsupportedSpatialReference); + } + + MultipolygonWithHoles2 polygons; + for (auto &&feature_layer_pair : dataset->GetFeatures()) { + OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); + DEBUG_ASSERT(geometry); + + const uint64_t original_point_count = point_count(*geometry); + std::unique_ptr simplified = simplify_geometry(*geometry, *tolerance); + if (!simplified || simplified->IsEmpty() || !simplified->IsValid()) { + LOG_ERROR("Failed to simplify mask geometry while preserving its topology"); + return std::unexpected(LoadErrorKind::InvalidGeometry); + } + + LOG_DEBUG("Simplified mask with {} m tolerance from {} to {} points", + simplification_tolerance_metres, original_point_count, point_count(*simplified)); + process_geometry(*simplified, polygons); + } + + if (polygons.is_empty()) { + LOG_ERROR("No valid polygons found in mask dataset '{}'", mask_dataset.name()); + return std::unexpected(LoadErrorKind::EmptySource); + } + return ReferencedPolygonMask{.polygons = std::move(polygons), .srs = std::move(srs)}; } @@ -472,25 +548,25 @@ inline MeshMask extrude( return extrude(mask, padded_radius_range); } -inline tl::expected load_referenced_from_path(const std::filesystem::path &path) { +inline std::expected load_referenced_from_path(const std::filesystem::path &path) { if (!std::filesystem::exists(path)) { LOG_ERROR("Mask file does not exist: {}", path); - return tl::unexpected(LoadErrorKind::FileNotFound); + return std::unexpected(LoadErrorKind::FileNotFound); } auto ds_opt = Dataset::open_vector(path); if (!ds_opt.has_value()) { LOG_ERROR("Failed to load mask datset: {}", path); - return tl::unexpected(LoadErrorKind::FileNotFound); + return std::unexpected(LoadErrorKind::FileNotFound); } Dataset dataset = std::move(ds_opt.value()); return load_referenced_from_dataset(dataset); } -inline tl::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { +inline std::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { auto ref_mask_res = load_referenced_from_path(path); if (!ref_mask_res.has_value()) { - return tl::unexpected(ref_mask_res.error()); + return std::unexpected(ref_mask_res.error()); } ReferencedPolygonMask ref_polygon_mask = std::move(ref_mask_res.value()); SpherePolygonMask sphere_polygon_mask = project_onto_sphere(std::move(ref_polygon_mask), radius_range.x); diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index ea3a1b18..180dbb59 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -177,5 +177,8 @@ inline void merge_datasets( merger.merge_root(); } - output_dataset.save_or_create_index(); + const auto index_result = output_dataset.save_or_create_index(); + if (!index_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output_dataset.base_path(), index_result.error()); + } } diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index fcfa9b3a..a51a7821 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(terrainlib ctb/types.hpp io/bytes.cpp + io/compression.cpp io/utils.cpp mesh/connectivity/adjacency.cpp @@ -53,6 +54,7 @@ add_library(terrainlib init.cpp log.cpp ProgressIndicator.cpp + tile_path.h ) target_compile_features(terrainlib PUBLIC cxx_std_23) @@ -74,9 +76,9 @@ target_link_libraries(terrainlib PUBLIC spdlog fmt zpp_bits + zstd::libzstd_static cgltf TBB::tbb - tl_expected opencv_core opencv_imgproc opencv_imgcodecs diff --git a/src/terrainlib/ProgressIndicator.cpp b/src/terrainlib/ProgressIndicator.cpp index 565f7ee8..a9f49272 100644 --- a/src/terrainlib/ProgressIndicator.cpp +++ b/src/terrainlib/ProgressIndicator.cpp @@ -39,6 +39,7 @@ void ProgressIndicator::task_finished() { if (m_step > m_n_steps) { throw std::runtime_error("Too many steps reported."); } + m_monitor_condition.notify_all(); } std::jthread ProgressIndicator::start_monitoring() const { @@ -50,19 +51,30 @@ std::jthread ProgressIndicator::start_monitoring() const { }; const auto t0 = std::chrono::steady_clock::now(); - std::jthread thread([=, this]() { - const auto delta_t = 500ms; - auto v_t_minus_1 = this->m_step.load(); - do { - auto v_t = this->m_step.load(); - const auto delta_v = v_t - v_t_minus_1; - v_t_minus_1 = v_t; - print(delta_v, delta_t); - std::this_thread::sleep_for(delta_t); - } while (this->m_step < this->m_n_steps); - const auto t1 = std::chrono::steady_clock::now(); - print(this->m_n_steps, std::chrono::duration_cast(t1 - t0)); - std::cout << std::endl; + std::jthread thread([=, this](std::stop_token stop_token) noexcept { + try { + const auto delta_t = 500ms; + auto v_t_minus_1 = this->m_step.load(); + while (!stop_token.stop_requested() && this->m_step < this->m_n_steps) { + const auto v_t = this->m_step.load(); + const auto delta_v = v_t - v_t_minus_1; + v_t_minus_1 = v_t; + print(delta_v, delta_t); + + std::unique_lock lock(this->m_monitor_mutex); + this->m_monitor_condition.wait_for( + lock, stop_token, delta_t, [this]() { return this->m_step >= this->m_n_steps; }); + } + + if (!stop_token.stop_requested()) { + const auto t1 = std::chrono::steady_clock::now(); + print(this->m_n_steps, std::chrono::duration_cast(t1 - t0)); + std::cout << std::endl; + } + } catch (...) { + // Console output and formatting must not terminate the process from + // inside the monitoring thread. + } }); return thread; } diff --git a/src/terrainlib/ProgressIndicator.h b/src/terrainlib/ProgressIndicator.h index fcf34eb0..ca16e36e 100644 --- a/src/terrainlib/ProgressIndicator.h +++ b/src/terrainlib/ProgressIndicator.h @@ -21,12 +21,16 @@ #include #include +#include +#include #include #include class ProgressIndicator { const size_t m_n_steps; std::atomic m_step = 0; + mutable std::condition_variable_any m_monitor_condition; + mutable std::mutex m_monitor_mutex; public: ProgressIndicator(size_t n_steps); diff --git a/src/terrainlib/ctb/GlobalGeodetic.hpp b/src/terrainlib/ctb/GlobalGeodetic.hpp index 5ddeecc5..917690b9 100644 --- a/src/terrainlib/ctb/GlobalGeodetic.hpp +++ b/src/terrainlib/ctb/GlobalGeodetic.hpp @@ -28,10 +28,11 @@ namespace ctb { /** - * @brief An implementation of the TMS Global Geodetic Profile + * @brief An implementation of the global Geodetic grid profile * * This class models the [Tile Mapping Service Global Geodetic * Profile](http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-geodetic). + * Its public tile identifiers use Google/Mapbox/XYZ coordinates. */ class GlobalGeodetic : public Grid { public: @@ -42,7 +43,7 @@ class GlobalGeodetic : public Grid { cSRS, 4326, // global geodetic has 2 root tiles: https://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-geodetic - std::vector{radix::tile::Id{0, {0, 0}, radix::tile::Scheme::Tms}, radix::tile::Id{0, {1, 0}, radix::tile::Scheme::Tms}}, + std::vector{radix::tile::Id{0, {0, 0}}, radix::tile::Id{0, {1, 0}}}, 2) { } diff --git a/src/terrainlib/ctb/GlobalMercator.hpp b/src/terrainlib/ctb/GlobalMercator.hpp index 2da275a4..9de4af42 100644 --- a/src/terrainlib/ctb/GlobalMercator.hpp +++ b/src/terrainlib/ctb/GlobalMercator.hpp @@ -29,10 +29,11 @@ class GlobalMercator; } /** - * @brief An implementation of the TMS Global Mercator Profile + * @brief An implementation of the global Mercator grid profile * * This class models the [Tile Mapping Service Global Mercator * Profile](http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification#global-mercator). + * Its public tile identifiers use Google/Mapbox/XYZ coordinates. */ class ctb::GlobalMercator : public Grid { public: @@ -41,7 +42,7 @@ class ctb::GlobalMercator : public Grid { radix::tile::SrsBounds{{-cOriginShift, -cOriginShift}, {cOriginShift, cOriginShift}}, cSRS, 3857, - std::vector{radix::tile::Id{0, {0, 0}, radix::tile::Scheme::Tms}}, + std::vector{radix::tile::Id{0, {0, 0}}}, 2) { } diff --git a/src/terrainlib/ctb/Grid.hpp b/src/terrainlib/ctb/Grid.hpp index a4338843..8d81488c 100644 --- a/src/terrainlib/ctb/Grid.hpp +++ b/src/terrainlib/ctb/Grid.hpp @@ -51,15 +51,8 @@ class Grid; * The code here generalises the logic in the `gdal2tiles.py` script available * with the GDAL library. * - * Warning: The y directino is dangerous. Sometimes the positve y axis points north, sometimes not. - * - GlobalMercator has y positive pointing north - * - GlobalGeodetic as well. - * - https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/175.75/56.27 - * - google webmercator has tile coordinates where y=0 is the northern most tile. - * - tms webmercator has tile coordinates with y=0 being southern most. - * - * Effectively, ctb::Grid is always positive pointing north. Support for google webmercator / - * slippyMap is done in Tile.h + * Tile identifiers use Google/Mapbox/XYZ coordinates: the origin is north-west and y grows south. + * CRS and pixel coordinates inside Grid retain their conventional positive-north orientation. */ class ctb::Grid { public: @@ -143,23 +136,25 @@ class ctb::Grid { /// Get the tile coordinate in which a location falls at a specific zoom level [[nodiscard]] inline radix::tile::Id crsToTile(const CRSPoint &coord, i_zoom zoom) const { const PixelPoint pixel = crsToPixels(coord, zoom); - TilePoint tile = pixelsToTile(pixel); + const TilePoint tile = pixelsToTile(pixel); + const auto tile_count = i_tile(1u << zoom); - return {zoom, tile, radix::tile::Scheme::Tms}; + return {zoom, {tile.x, tile_count - tile.y - 1}}; } /// Get the CRS bounds of a particular tile /// border_se should be true if a border should be included on the south eastern corner /// e.g., for the cesium raster terrain format (https://github.com/CesiumGS/cesium/wiki/heightmap-1%2E0) [[nodiscard]] inline radix::tile::SrsBounds srsBounds(const radix::tile::Id &tile_id, bool border_se) const { - const auto tms_tile_id = tile_id.to(radix::tile::Scheme::Tms); + const auto tile_count = i_tile(1u << tile_id.zoom_level); + const auto grid_y = tile_count - tile_id.coords.y - 1; // get the pixels coordinates representing the tile bounds - const PixelPoint pxMinLeft(tms_tile_id.coords.x * mGridSize, tms_tile_id.coords.y * mGridSize); - const PixelPoint pxMaxRight((tms_tile_id.coords.x + 1) * mGridSize + border_se, (tms_tile_id.coords.y + 1) * mGridSize + border_se); + const PixelPoint pxMinLeft(tile_id.coords.x * mGridSize, grid_y * mGridSize); + const PixelPoint pxMaxRight((tile_id.coords.x + 1) * mGridSize + border_se, (grid_y + 1) * mGridSize + border_se); // convert pixels to native coordinates - const CRSPoint minLeft = pixelsToCrs(pxMinLeft, tms_tile_id.zoom_level); - const CRSPoint maxRight = pixelsToCrs(pxMaxRight, tms_tile_id.zoom_level); + const CRSPoint minLeft = pixelsToCrs(pxMinLeft, tile_id.zoom_level); + const CRSPoint maxRight = pixelsToCrs(pxMaxRight, tile_id.zoom_level); return { minLeft, maxRight }; } diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index 499b0ee9..1412ce33 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -6,7 +6,7 @@ namespace io { -tl::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { @@ -16,31 +16,31 @@ tl::expected write_bytes_to_path(const std::span byt std::ofstream file(path, std::ios::binary); if (!file.is_open()) { LOG_DEBUG("Failed to open file for writing {}", path); - return tl::unexpected(Error::OpenFile); + return std::unexpected(Error::OpenFile); } file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); if (!file.good()) { LOG_ERROR("Failed to write bytes to file {}", path); - return tl::unexpected(Error::WriteBytes); + return std::unexpected(Error::WriteBytes); } return {}; } -tl::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { +std::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { LOG_TRACE("Reading bytes from path {}", path); std::ifstream file(path, std::ios::binary | std::ios::ate); if (!file.is_open()) { LOG_DEBUG("Failed to open file for reading {}", path); - return tl::unexpected(Error::OpenFile); + return std::unexpected(Error::OpenFile); } const std::streamsize size = file.tellg(); if (size < 0) { LOG_ERROR("Failed to determine size for file {}", path); - return tl::unexpected(Error::DetermineSize); + return std::unexpected(Error::DetermineSize); } std::vector buffer(static_cast(size)); @@ -49,7 +49,7 @@ tl::expected, Error> read_bytes_from_path(const std::filesy if (!file.good()) { LOG_ERROR("Failed to read bytes from file {}", path); - return tl::unexpected(Error::ReadBytes); + return std::unexpected(Error::ReadBytes); } return buffer; diff --git a/src/terrainlib/io/bytes.h b/src/terrainlib/io/bytes.h index 5c156cf4..be3852dc 100644 --- a/src/terrainlib/io/bytes.h +++ b/src/terrainlib/io/bytes.h @@ -4,13 +4,13 @@ #include #include -#include +#include #include "io/Error.h" namespace io { -tl::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); -tl::expected, Error> read_bytes_from_path(const std::filesystem::path &path); +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); +std::expected, Error> read_bytes_from_path(const std::filesystem::path &path); } diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp new file mode 100644 index 00000000..ba18e638 --- /dev/null +++ b/src/terrainlib/io/compression.cpp @@ -0,0 +1,221 @@ +#define ZSTD_STATIC_LINKING_ONLY +#include +#include + +#include "io/compression.h" + +#include +#include +#include +#include +#include + +namespace io::envelope { +namespace { + +using CompressionContext = std::unique_ptr; + +std::string crc32c_checksum(const Bytes &data) +{ + static constexpr auto table = [] { + constexpr std::uint32_t polynomial = 0x82f63b78u; + + std::array values{}; + for (std::size_t index = 0; index < values.size(); ++index) { + auto crc = static_cast(index); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + } + values[index] = crc; + } + return values; + }(); + + std::uint32_t crc = 0xffffffffu; + for (const auto value : data) { + const auto index = (crc ^ std::to_integer(value)) & 0xffu; + crc = table[index] ^ (crc >> 8u); + } + crc = ~crc; + + constexpr char hex_digits[] = "0123456789abcdef"; + std::string checksum(8, '0'); + for (std::size_t index = 0; index < checksum.size(); ++index) { + const auto shift = static_cast((checksum.size() - index - 1) * 4); + checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; + } + return checksum; +} + +std::expected validate_algorithms( + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + switch (checksum_algorithm) { + case ChecksumAlgorithm::None: + case ChecksumAlgorithm::HandledByCompressionLib: + case ChecksumAlgorithm::Crc32c: + break; + default: + return std::unexpected(Error{ErrorCode::UnsupportedChecksumAlgorithm}); + } + + switch (compression_algorithm) { + case CompressionAlgorithm::None: + case CompressionAlgorithm::ZstdBestCompressionWithChecksum: + break; + default: + return std::unexpected(Error{ErrorCode::UnsupportedCompressionAlgorithm}); + } + + const bool no_compression = compression_algorithm == CompressionAlgorithm::None + && (checksum_algorithm == ChecksumAlgorithm::None + || checksum_algorithm == ChecksumAlgorithm::Crc32c); + const bool zstd_with_checksum = + compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib + || checksum_algorithm == ChecksumAlgorithm::Crc32c); + if (!no_compression && !zstd_with_checksum) { + return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + } + + return {}; +} + +} // namespace + +std::expected compress_with_checksum( + const Bytes &uncompressed_data, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(validation.error()); + } + if (uncompressed_data.size() > default_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c + ? crc32c_checksum(uncompressed_data) + : std::string{}; + + if (compression_algorithm == CompressionAlgorithm::None) { + return CompressedData{uncompressed_data, checksum}; + } + + CompressionContext context{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + if (!context) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, ZSTD_maxCLevel())) + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); + Bytes compressed_data(capacity); + const std::size_t compressed_size = ZSTD_compress2( + context.get(), + compressed_data.data(), + compressed_data.size(), + uncompressed_data.data(), + uncompressed_data.size()); + if (ZSTD_isError(compressed_size)) { + return std::unexpected(Error{ErrorCode::CompressionFailed}); + } + + compressed_data.resize(compressed_size); + return CompressedData{std::move(compressed_data), checksum}; +} + +std::expected checked_decompress( + const Bytes &compressed_data, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm, + const std::string_view checksum, + const std::size_t max_decompressed_size) +{ + if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(validation.error()); + } + if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { + return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + } + if (checksum_algorithm == ChecksumAlgorithm::Crc32c + && (checksum.size() != 8 + || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { + return (character >= '0' && character <= '9') + || (character >= 'a' && character <= 'f'); + }))) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + + const std::size_t effective_max_decompressed_size = + std::min(max_decompressed_size, default_max_decompressed_size); + + Bytes uncompressed_data; + if (compression_algorithm == CompressionAlgorithm::None) { + if (compressed_data.size() > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + uncompressed_data = compressed_data; + } else { + ZSTD_frameHeader frame_header{}; + const std::size_t frame_header_result = ZSTD_getFrameHeader( + &frame_header, compressed_data.data(), compressed_data.size()); + if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (frame_header.checksumFlag == 0) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + + const unsigned long long frame_content_size = + ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); + if (frame_content_size == ZSTD_CONTENTSIZE_ERROR + || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && frame_content_size > std::numeric_limits::max())) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; + if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + const std::size_t allocation_size = content_size_is_known + ? static_cast(frame_content_size) + : effective_max_decompressed_size; + uncompressed_data.resize(allocation_size); + const std::size_t decompressed_size = ZSTD_decompress( + uncompressed_data.data(), + uncompressed_data.size(), + compressed_data.data(), + compressed_data.size()); + if (ZSTD_isError(decompressed_size)) { + if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + if (!content_size_is_known + && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + if (content_size_is_known && decompressed_size != uncompressed_data.size()) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + uncompressed_data.resize(decompressed_size); + } + + if (checksum_algorithm == ChecksumAlgorithm::Crc32c + && crc32c_checksum(uncompressed_data) != checksum) { + return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + } + return uncompressed_data; +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h new file mode 100644 index 00000000..104f9edd --- /dev/null +++ b/src/terrainlib/io/compression.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace io::envelope { + +using Bytes = std::vector; + +inline constexpr std::size_t default_max_decompressed_size = std::size_t{1} << 30; + +enum class ChecksumAlgorithm : std::uint8_t { + None, + HandledByCompressionLib, + Crc32c, +}; + +enum class CompressionAlgorithm : std::uint8_t { + None, + ZstdBestCompressionWithChecksum, +}; + +enum class ErrorCode : std::uint8_t { + SerializationFailed, + DeserializationFailed, + InvalidMagic, + WrongClassName, + UnsupportedClassVersion, + UnsupportedChecksumAlgorithm, + UnsupportedCompressionAlgorithm, + InvalidAlgorithmCombination, + ChecksumMismatch, + CompressionFailed, + DecompressionFailed, + SizeLimitExceeded, +}; + +struct Error { + ErrorCode code; + std::errc serialization_error{}; + + constexpr bool operator==(const Error &) const = default; +}; + +struct CompressedData { + Bytes compressed_data; + std::string checksum; +}; + +std::expected compress_with_checksum( + const Bytes &uncompressed_data, + CompressionAlgorithm compression_algorithm, + ChecksumAlgorithm checksum_algorithm); + +std::expected checked_decompress( + const Bytes &compressed_data, + CompressionAlgorithm compression_algorithm, + ChecksumAlgorithm checksum_algorithm, + std::string_view checksum, + std::size_t max_decompressed_size = default_max_decompressed_size); + +} // namespace io::envelope diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h new file mode 100644 index 00000000..4b36065b --- /dev/null +++ b/src/terrainlib/io/envelope.h @@ -0,0 +1,141 @@ +#pragma once + +#include "io/compression.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace io::envelope { + +inline constexpr std::uint64_t magic = 0xF5FBD3EF919428CAULL; + +struct Envelope { + std::uint64_t magic; + std::string class_name; + std::uint32_t class_version; + ChecksumAlgorithm checksum_algorithm; + std::string checksum; + CompressionAlgorithm compression_algorithm; + std::uint64_t uncompressed_size; + Bytes compressed_data; +}; + +template +struct FixedString { + char value[Size]; + + constexpr FixedString(const char (&text)[Size]) + { + std::copy_n(text, Size, value); + } + + constexpr auto operator<=>(const FixedString &) const = default; +}; + +template +struct Version { + static constexpr std::uint32_t number = Number; + using payload_type = VersionedPayloadType; +}; + +namespace detail { + +template +struct FindVersion; + +template +struct FindVersion { + using type = std::conditional_t::type>; +}; + +template +struct FindVersion { + using type = void; +}; + +template +consteval bool versions_are_strictly_increasing() +{ + constexpr std::uint32_t numbers[] = {Versions::number...}; + for (std::size_t index = 1; index < sizeof...(Versions); ++index) { + if (numbers[index - 1] >= numbers[index]) { + return false; + } + } + return true; +} + +template +concept ConvertsFromPrevious = requires(From previous) { + { To::from_previous(std::move(previous)) } -> std::same_as; +}; + +template +consteval bool conversions_are_valid(std::index_sequence) +{ + return (ConvertsFromPrevious< + typename std::tuple_element_t::payload_type, + typename std::tuple_element_t::payload_type> + && ...); +} + +} // namespace detail + +template +struct PayloadSchema { + static_assert(sizeof...(Versions) > 0, "a payload schema requires at least one version"); + static_assert(detail::versions_are_strictly_increasing(), + "payload versions must be strictly increasing"); + + using version_tuple = std::tuple; + static constexpr std::size_t version_count = sizeof...(Versions); + + template + using version_at = std::tuple_element_t; + + using latest_version_descriptor = version_at; + using latest_type = typename latest_version_descriptor::payload_type; + + static constexpr std::string_view class_name{ClassName.value, sizeof(ClassName.value) - 1}; + static constexpr std::uint32_t latest_version = latest_version_descriptor::number; + + template + static constexpr bool supports_version = + !std::is_void_v::type>; + + static constexpr bool supports_version_number(const std::uint32_t number) + { + return ((number == Versions::number) || ...); + } + + template + using payload_type = typename detail::FindVersion::type::payload_type; + + static_assert(version_count == 1 + || detail::conversions_are_valid( + std::make_index_sequence{}), + "each payload version must provide from_previous for the preceding version"); +}; + +template +std::expected serialize( + const typename Schema::template payload_type &payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdBestCompressionWithChecksum, + ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); + +template +std::expected deserialize( + std::span bytes, + std::size_t max_decompressed_size = default_max_decompressed_size); + +} // namespace io::envelope + +#include "io/envelope.inl" diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl new file mode 100644 index 00000000..c9055894 --- /dev/null +++ b/src/terrainlib/io/envelope.inl @@ -0,0 +1,148 @@ +#pragma once + +#include + +#include + +namespace io::envelope { +namespace detail { + +template +std::expected serialize_to_bytes(const Value &value) +{ + Bytes bytes; + zpp::bits::out output(bytes, zpp::bits::alloc_limit()); + const zpp::bits::errc result = output(value); + if (zpp::bits::failure(result)) { + return std::unexpected(Error{ErrorCode::SerializationFailed, result.code}); + } + return bytes; +} + +template +std::expected deserialize_from_bytes(const std::span bytes) +{ + Value value{}; + zpp::bits::in input(bytes, zpp::bits::alloc_limit()); + const zpp::bits::errc result = input(value); + if (zpp::bits::failure(result) || input.position() != bytes.size()) { + return std::unexpected(Error{ + ErrorCode::DeserializationFailed, + zpp::bits::failure(result) ? result.code : std::errc::bad_message, + }); + } + return value; +} + +template +typename Schema::latest_type convert_to_latest(Current current) +{ + if constexpr (Index + 1 == Schema::version_count) { + return current; + } else { + using Next = typename Schema::template version_at::payload_type; + return convert_to_latest(Next::from_previous(std::move(current))); + } +} + +template +std::expected deserialize_version( + const std::uint32_t class_version, + const std::span payload_bytes) +{ + using CurrentVersion = typename Schema::template version_at; + if (class_version == CurrentVersion::number) { + auto payload = deserialize_from_bytes(payload_bytes); + if (!payload) { + return std::unexpected(payload.error()); + } + return convert_to_latest(std::move(*payload)); + } + + if constexpr (Index + 1 < Schema::version_count) { + return deserialize_version(class_version, payload_bytes); + } else { + return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + } +} + +} // namespace detail + +template +std::expected serialize( + const typename Schema::template payload_type &payload, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + static_assert(Schema::template supports_version, + "the requested payload version is not part of the schema"); + + auto payload_bytes = detail::serialize_to_bytes(payload); + if (!payload_bytes) { + return std::unexpected(payload_bytes.error()); + } + + auto compressed = compress_with_checksum(*payload_bytes, compression_algorithm, checksum_algorithm); + if (!compressed) { + return std::unexpected(compressed.error()); + } + + const Envelope envelope{ + .magic = magic, + .class_name = std::string{Schema::class_name}, + .class_version = VersionNumber, + .checksum_algorithm = checksum_algorithm, + .checksum = std::move(compressed->checksum), + .compression_algorithm = compression_algorithm, + .uncompressed_size = payload_bytes->size(), + .compressed_data = std::move(compressed->compressed_data), + }; + return detail::serialize_to_bytes(envelope); +} + +template +std::expected deserialize( + const std::span bytes, + const std::size_t max_decompressed_size) +{ + auto envelope = detail::deserialize_from_bytes(bytes); + if (!envelope) { + return std::unexpected(envelope.error()); + } + if (envelope->magic != magic) { + return std::unexpected(Error{ErrorCode::InvalidMagic}); + } + if (envelope->class_name != Schema::class_name) { + return std::unexpected(Error{ErrorCode::WrongClassName}); + } + if (!Schema::supports_version_number(envelope->class_version)) { + return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + } + + const std::size_t effective_max_decompressed_size = + std::min(max_decompressed_size, default_max_decompressed_size); + if (envelope->uncompressed_size > std::numeric_limits::max() + || envelope->uncompressed_size > effective_max_decompressed_size) { + return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + } + + auto payload_bytes = checked_decompress( + envelope->compressed_data, + envelope->compression_algorithm, + envelope->checksum_algorithm, + envelope->checksum, + static_cast(envelope->uncompressed_size)); + if (!payload_bytes) { + if (payload_bytes.error().code == ErrorCode::SizeLimitExceeded) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + return std::unexpected(payload_bytes.error()); + } + if (payload_bytes->size() != envelope->uncompressed_size) { + return std::unexpected(Error{ErrorCode::DecompressionFailed}); + } + + return detail::deserialize_version(envelope->class_version, *payload_bytes); +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/serialize.h b/src/terrainlib/io/serialize.h index 55919e23..ffda2d1f 100644 --- a/src/terrainlib/io/serialize.h +++ b/src/terrainlib/io/serialize.h @@ -4,21 +4,21 @@ #include #include -#include +#include #include "io/Error.h" namespace io { template -tl::expected, Error> write_to_bytes(const T &value); +std::expected, Error> write_to_bytes(const T &value); template -tl::expected read_from_bytes(const std::span bytes); +std::expected read_from_bytes(const std::span bytes); template -tl::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); +std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); template -tl::expected read_from_path(const std::filesystem::path &path); +std::expected read_from_path(const std::filesystem::path &path); } diff --git a/src/terrainlib/io/serialize.inl b/src/terrainlib/io/serialize.inl index 90d7080b..80324d03 100644 --- a/src/terrainlib/io/serialize.inl +++ b/src/terrainlib/io/serialize.inl @@ -9,7 +9,7 @@ namespace io { template -tl::expected, Error> write_to_bytes(const T &value) { +std::expected, Error> write_to_bytes(const T &value) { std::vector data; zpp::bits::out out(data); const auto result = out(value); @@ -21,11 +21,11 @@ tl::expected, Error> write_to_bytes(const T &value) { case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. case std::errc::message_size: // message size is beyond the user defined allocation limits. case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return tl::unexpected(Error::OutOfMemory); + return std::unexpected(Error::OutOfMemory); case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. case std::errc::bad_message: // attempt to read a variant of unrecognized type. case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return tl::unexpected(Error::Serialize); + return std::unexpected(Error::Serialize); case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. UNREACHABLE(); @@ -37,7 +37,7 @@ tl::expected, Error> write_to_bytes(const T &value) { } template -tl::expected read_from_bytes(const std::span bytes) { +std::expected read_from_bytes(const std::span bytes) { zpp::bits::in in(bytes); T value; const auto result = in(value); @@ -49,11 +49,11 @@ tl::expected read_from_bytes(const std::span bytes) { case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. case std::errc::message_size: // message size is beyond the user defined allocation limits. case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return tl::unexpected(Error::OutOfMemory); + return std::unexpected(Error::OutOfMemory); case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. case std::errc::bad_message: // attempt to read a variant of unrecognized type. case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return tl::unexpected(Error::Deserialize); + return std::unexpected(Error::Deserialize); case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. UNREACHABLE(); @@ -65,20 +65,20 @@ tl::expected read_from_bytes(const std::span bytes) { } template -tl::expected read_from_path(const std::filesystem::path &path) { +std::expected read_from_path(const std::filesystem::path &path) { const auto result = read_bytes_from_path(path); if (!result.has_value()) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const std::vector bytes = result.value(); return read_from_bytes(bytes); } template -tl::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { +std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { const auto result = write_to_bytes(value); if (!result.has_value()) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const std::vector bytes = result.value(); diff --git a/src/terrainlib/mesh/encode.h b/src/terrainlib/mesh/encode.h index 302fd016..eca66508 100644 --- a/src/terrainlib/mesh/encode.h +++ b/src/terrainlib/mesh/encode.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "log.h" #include "mesh/EncodedMesh.h" @@ -81,7 +81,7 @@ inline std::ostream& operator<<(std::ostream& os, const EncodeError& err) { } template -tl::expected encode(const Simple_& mesh, const EncodeOptions options = {}) { +std::expected encode(const Simple_& mesh, const EncodeOptions options = {}) { using Mesh = Simple_; const size_t vertex_count = mesh.vertex_count(); @@ -96,7 +96,7 @@ tl::expected encode(const Simple_& mesh, const position_buf.resize(meshopt_encodeVertexBufferBound(vertex_count, position_size)); const size_t pos_written = meshopt_encodeVertexBuffer(position_buf.data(), position_buf.size(), mesh.positions.data(), vertex_count, position_size); if (pos_written == 0) { - return tl::unexpected(EncodeError::PositionEncode); + return std::unexpected(EncodeError::PositionEncode); } position_buf.resize(pos_written); } @@ -107,7 +107,7 @@ tl::expected encode(const Simple_& mesh, const uv_buf.resize(meshopt_encodeVertexBufferBound(vertex_count, uv_size)); const size_t uv_written = meshopt_encodeVertexBuffer(uv_buf.data(), uv_buf.size(), mesh.uvs.data(), vertex_count, uv_size); if (uv_written == 0) { - return tl::unexpected(EncodeError::UvEncode); + return std::unexpected(EncodeError::UvEncode); } uv_buf.resize(uv_written); } @@ -121,7 +121,7 @@ tl::expected encode(const Simple_& mesh, const reinterpret_cast(mesh.triangles.data()), index_count); if (index_written == 0) { - return tl::unexpected(EncodeError::TriangleEncode); + return std::unexpected(EncodeError::TriangleEncode); } index_buf.resize(index_written); } @@ -133,7 +133,7 @@ tl::expected encode(const Simple_& mesh, const texture_buf = mesh::io::write_texture_to_encoded_buffer(mesh.texture.value(), options.texture_format); } catch (const cv::Exception &e) { LOG_ERROR("Failed while encoding texture {}", e.what()); - return tl::unexpected(EncodeError::TextureEncode); + return std::unexpected(EncodeError::TextureEncode); } } @@ -189,13 +189,13 @@ inline std::ostream &operator<<(std::ostream &os, const DecodeError &err) { template -tl::expected, DecodeError> decode(const Encoded &encoded, const DecodeOptions = {}) { +std::expected, DecodeError> decode(const Encoded &encoded, const DecodeOptions = {}) { const uint32_t expected_component_type = component_type_id(); const Encoded::Header& header = encoded.header; if (header.version != 1 || header.n_dims != n_dims || header.component_type != expected_component_type) { - return tl::unexpected(DecodeError::IncompatibleData); + return std::unexpected(DecodeError::IncompatibleData); } using Mesh = Simple_; @@ -208,7 +208,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.positions.resize(vertex_count); result = meshopt_decodeVertexBuffer(mesh.positions.data(), vertex_count, position_size, encoded.positions.data(), encoded.positions.size()); if (result != 0) { - return tl::unexpected(DecodeError::PositionDecode); + return std::unexpected(DecodeError::PositionDecode); } // Decode uvs @@ -217,7 +217,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.uvs.resize(vertex_count); result = meshopt_decodeVertexBuffer(mesh.uvs.data(), vertex_count, uv_size, encoded.uvs.data(), encoded.uvs.size()); if (result != 0) { - return tl::unexpected(DecodeError::UvDecode); + return std::unexpected(DecodeError::UvDecode); } } @@ -228,7 +228,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.triangles.resize(face_count); result = meshopt_decodeIndexBuffer(mesh.triangles.data(), index_count, index_size, encoded.triangles.data(), encoded.triangles.size()); if (result != 0) { - return tl::unexpected(DecodeError::TriangleDecode); + return std::unexpected(DecodeError::TriangleDecode); } // Decode texture @@ -237,7 +237,7 @@ tl::expected, DecodeError> decode(const Encoded &encoded, con mesh.texture = mesh::io::read_texture_from_encoded_bytes(encoded.texture); } catch (const cv::Exception &e) { LOG_ERROR("Failed while decoding texture {}", e.what()); - return tl::unexpected(DecodeError::TextureDecode); + return std::unexpected(DecodeError::TextureDecode); } } diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 6d56c2c1..467e0537 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -6,7 +6,7 @@ namespace mesh::io { -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions& options) { const std::filesystem::path extension = path.extension(); @@ -15,11 +15,11 @@ tl::expected load_from_path( } else if (extension == ".terrain") { return terrain::load_from_path(path, options); } else { - return tl::unexpected(LoadMeshErrorKind::UnsupportedFormat); + return std::unexpected(LoadMeshErrorKind::UnsupportedFormat); } } -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options) { @@ -33,7 +33,7 @@ tl::expected save_to_path( } else if (extension == ".terrain") { return terrain::save_to_path(mesh, path, options); } else { - return tl::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); } } diff --git a/src/terrainlib/mesh/io.h b/src/terrainlib/mesh/io.h index 81fb076e..ea474c73 100644 --- a/src/terrainlib/mesh/io.h +++ b/src/terrainlib/mesh/io.h @@ -2,7 +2,7 @@ #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/options.h" @@ -10,11 +10,11 @@ namespace mesh::io { -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions& options = {}); -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options = {}); diff --git a/src/terrainlib/mesh/io/gltf.cpp b/src/terrainlib/mesh/io/gltf.cpp index 36160963..318716a6 100644 --- a/src/terrainlib/mesh/io/gltf.cpp +++ b/src/terrainlib/mesh/io/gltf.cpp @@ -200,7 +200,7 @@ std::optional load_texture_from_material(const cgltf_material &material #define GET_OR_INVALID_FORMAT(var, opt) \ do { \ if (!(opt).has_value()) { \ - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); \ + return std::unexpected(LoadMeshErrorKind::InvalidFormat); \ } else { \ var = opt.value(); \ } \ @@ -265,56 +265,56 @@ static std::string image_ext_to_mime(std::string_view extension) { } } -tl::expected load_raw_from_path(const std::filesystem::path &path) { +std::expected load_raw_from_path(const std::filesystem::path &path) { cgltf_options options = {}; cgltf_data *data = NULL; const std::string path_str = path.string(); const char *path_ptr = path_str.c_str(); cgltf_result result = cgltf_parse_file(&options, path_ptr, &data); if (result != cgltf_result::cgltf_result_success) { - return tl::unexpected(result); + return std::unexpected(result); } result = cgltf_load_buffers(&options, data, path_ptr); if (result != cgltf_result::cgltf_result_success) { cgltf_free(data); - return tl::unexpected(result); + return std::unexpected(result); } result = cgltf_validate(data); if (result != cgltf_result_success) { cgltf_free(data); - return tl::unexpected(result); + return std::unexpected(result); } return RawMesh(data, cgltf_free); } -tl::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { +std::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { LOG_TRACE("Loading mesh from gltf data"); const cgltf_data &data = *raw; const auto mesh_opt = get_single_element("mesh", data.meshes_count, data.meshes); if (!mesh_opt.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const cgltf_mesh &mesh = mesh_opt.value(); const auto mesh_primitive_opt = get_single_element("mesh primitive", mesh.primitives_count, mesh.primitives); if (!mesh_primitive_opt.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const cgltf_primitive &mesh_primitive = mesh_primitive_opt.value(); if (mesh_primitive.type != cgltf_primitive_type::cgltf_primitive_type_triangles) { LOG_ERROR("mesh has invalid primitive type"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } // indices if (mesh_primitive.indices == nullptr) { LOG_ERROR("mesh primitive has no indices"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } cgltf_accessor &index_accessor = *mesh_primitive.indices; std::vector indices; @@ -330,13 +330,13 @@ tl::expected load_from_raw(const RawMesh &raw, const cgltf_attribute *position_attr = find_attribute_with_type(mesh_primitive.attributes, mesh_primitive.attributes_count, cgltf_attribute_type_position); if (position_attr == nullptr) { LOG_ERROR("mesh has no position attribute"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } cgltf_accessor &position_accessor = *position_attr->data; if (position_accessor.type != cgltf_type_vec3) { LOG_WARN("mesh positions are not vec3"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } std::vector positions; positions.resize(position_accessor.count); @@ -351,7 +351,7 @@ tl::expected load_from_raw(const RawMesh &raw, const cgltf_accessor &uv_accessor = *uv_attr->data; if (uv_accessor.type != cgltf_type_vec2) { LOG_WARN("mesh uvss are not vec2"); - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } uvs.resize(uv_accessor.count); cgltf_accessor_unpack_floats(&uv_accessor, reinterpret_cast(uvs.data()), uvs.size() * 2); @@ -381,7 +381,7 @@ tl::expected load_from_raw(const RawMesh &raw, const } /// Saves the mesh as a .gltf or .glb file at the given path. -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &terrain_mesh, const std::filesystem::path &path, const SaveOptions& options) { @@ -719,10 +719,10 @@ tl::expected save_to_path( return {}; } -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { - tl::expected raw_mesh = load_raw_from_path(path); +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { + std::expected raw_mesh = load_raw_from_path(path); if (!raw_mesh) { - return tl::unexpected(map_cgltf_error(raw_mesh.error())); + return std::unexpected(map_cgltf_error(raw_mesh.error())); } return load_from_raw(*raw_mesh, options); } diff --git a/src/terrainlib/mesh/io/gltf.h b/src/terrainlib/mesh/io/gltf.h index 60757382..20333e93 100644 --- a/src/terrainlib/mesh/io/gltf.h +++ b/src/terrainlib/mesh/io/gltf.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/error.h" @@ -15,19 +15,19 @@ namespace mesh::io::gltf { using RawMesh = std::unique_ptr; -tl::expected load_from_path( +std::expected load_from_path( const std::filesystem::path &path, const LoadOptions &options = {}); -tl::expected load_from_raw( +std::expected load_from_raw( const RawMesh &mesh, const LoadOptions &options = {}); -tl::expected save_to_path( +std::expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -// tl::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); +// std::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); -tl::expected load_raw_from_path(const std::filesystem::path &path); +std::expected load_raw_from_path(const std::filesystem::path &path); } // namespace mesh::io::gltf diff --git a/src/terrainlib/mesh/io/terrain.cpp b/src/terrainlib/mesh/io/terrain.cpp index 1068a991..d443d6f8 100644 --- a/src/terrainlib/mesh/io/terrain.cpp +++ b/src/terrainlib/mesh/io/terrain.cpp @@ -34,24 +34,24 @@ SaveMeshError save_error_from_io_error(::io::Error error) { } } -tl::expected write_bytes_to_path( +std::expected write_bytes_to_path( const std::span bytes, const std::filesystem::path &path) { const auto result = ::io::write_bytes_to_path(bytes, path); if (!result.has_value()) { - return tl::unexpected(save_error_from_io_error(result.error())); + return std::unexpected(save_error_from_io_error(result.error())); } return {}; } -tl::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { +std::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { const auto result = ::io::read_bytes_from_path(path); if (!result.has_value()) { - return tl::unexpected(load_error_from_io_error(result.error())); + return std::unexpected(load_error_from_io_error(result.error())); } return result.value(); } -tl::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { +std::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { LOG_TRACE("Serializing mesh to buffer"); // TODO: this ignores the texture format in SaveOptions @@ -66,7 +66,7 @@ tl::expected, SaveMeshError> save_encoded_to_buffer(const m case std::errc::no_buffer_space: case std::errc::message_size: case std::errc::result_out_of_range: - return tl::unexpected(SaveMeshErrorKind::OutOfMemory); + return std::unexpected(SaveMeshErrorKind::OutOfMemory); break; default: UNREACHABLE(); @@ -77,7 +77,7 @@ tl::expected, SaveMeshError> save_encoded_to_buffer(const m return data; } -tl::expected load_encoded_from_buffer(const std::span bytes) { +std::expected load_encoded_from_buffer(const std::span bytes) { LOG_TRACE("Deserializing mesh from buffer"); zpp::bits::in in(bytes); @@ -90,12 +90,12 @@ tl::expected load_encoded_from_buffer(const std::s switch (result) { case std::errc::no_buffer_space: case std::errc::message_size: - return tl::unexpected(LoadMeshErrorKind::OutOfMemory); + return std::unexpected(LoadMeshErrorKind::OutOfMemory); case std::errc::value_too_large: case std::errc::bad_message: case std::errc::protocol_error: case std::errc::result_out_of_range: - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); case std::errc::not_supported: case std::errc::invalid_argument: UNREACHABLE(); @@ -109,61 +109,61 @@ tl::expected load_encoded_from_buffer(const std::s } } -tl::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { +std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { const auto encode_result = mesh::encode(mesh, mesh::EncodeOptions{ .texture_format = options.texture_format}); if (!encode_result.has_value()) { - return tl::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); } const mesh::Encoded encoded = encode_result.value(); const auto deser_result = save_encoded_to_buffer(encoded); if (!deser_result.has_value()) { - return tl::unexpected(deser_result.error()); + return std::unexpected(deser_result.error()); } const std::vector buffer = deser_result.value(); return buffer; } -tl::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { +std::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { const auto deser_result = load_encoded_from_buffer(bytes); if (!deser_result.has_value()) { - return tl::unexpected(deser_result.error()); + return std::unexpected(deser_result.error()); } const mesh::Encoded encoded = deser_result.value(); const auto decode_result = mesh::decode(encoded); if (!decode_result.has_value()) { - return tl::unexpected(LoadMeshErrorKind::InvalidFormat); + return std::unexpected(LoadMeshErrorKind::InvalidFormat); } const mesh::Simple mesh = decode_result.value(); return mesh; } -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { const auto bytes_result = read_bytes_from_path(path); if (!bytes_result.has_value()) { - return tl::unexpected(bytes_result.error()); + return std::unexpected(bytes_result.error()); } const std::vector bytes = bytes_result.value(); return load_from_buffer(bytes); } -tl::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { +std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { LOG_TRACE("Saving mesh as high precision terrain"); const auto result = save_to_buffer(mesh, options); if (!result.has_value()) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const std::vector bytes = result.value(); const auto write_result = write_bytes_to_path(bytes, path); if (!write_result.has_value()) { - return tl::unexpected(write_result.error()); + return std::unexpected(write_result.error()); } return {}; diff --git a/src/terrainlib/mesh/io/terrain.h b/src/terrainlib/mesh/io/terrain.h index 9c505c14..fd097a24 100644 --- a/src/terrainlib/mesh/io/terrain.h +++ b/src/terrainlib/mesh/io/terrain.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include "mesh/SimpleMesh.h" #include "mesh/io/error.h" @@ -11,10 +11,10 @@ namespace mesh::io::terrain { -tl::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); -tl::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); +std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); +std::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); -tl::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -tl::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); +std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); +std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); } // namespace mesh::io::terrain diff --git a/src/terrainlib/octree/storage/IndexedStorage.h b/src/terrainlib/octree/storage/IndexedStorage.h index 70076cd3..3ba94e8f 100644 --- a/src/terrainlib/octree/storage/IndexedStorage.h +++ b/src/terrainlib/octree/storage/IndexedStorage.h @@ -45,7 +45,7 @@ class IndexedStorage_ : public Storage_ { void update_index() noexcept { Storage_::update_index(); } - tl::expected save_index() const noexcept { + std::expected save_index() const noexcept { if (!this->is_index_dirty()) { return {}; } diff --git a/src/terrainlib/octree/storage/RawStorage.h b/src/terrainlib/octree/storage/RawStorage.h index 679efb38..39482250 100644 --- a/src/terrainlib/octree/storage/RawStorage.h +++ b/src/terrainlib/octree/storage/RawStorage.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include "octree/Id.h" #include "octree/disk/Layout.h" @@ -29,23 +29,23 @@ class RawStorage_ { RawStorage_(RawStorage_ &&) = default; RawStorage_ &operator=(RawStorage_ &&) = default; - tl::expected load(const Id &id) const noexcept { + std::expected load(const Id &id) const noexcept { const auto path = this->path_for(id); return Codec::load_from_path(path); } - tl::expected save(const Id &id, const T &node) const noexcept { + std::expected save(const Id &id, const T &node) const noexcept { const auto path = this->path_for(id); return Codec::save_to_path(node, path); } - tl::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { + std::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { return target.copy_from(id, *this); } - tl::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { + std::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { if (!source.has(id)) { - return tl::unexpected(CopyErrorKind::FileNotFound); + return std::unexpected(CopyErrorKind::FileNotFound); } const auto source_path = source.path_for(id); @@ -54,13 +54,13 @@ class RawStorage_ { // TODO: should this error instead? const auto load_result = source.load(id); if (!load_result.has_value()) { - return tl::unexpected(CopyErrorKind::Read); + return std::unexpected(CopyErrorKind::Read); } const value_type node = load_result.value(); const auto save_result = this->save(id, node); if (!save_result.has_value()) { - return tl::unexpected(CopyErrorKind::Write); + return std::unexpected(CopyErrorKind::Write); } return {}; } @@ -68,18 +68,18 @@ class RawStorage_ { std::error_code ec; if (std::filesystem::remove(target_path, ec)) { if (ec) { - return tl::unexpected(CopyErrorKind::RemoveOld); + return std::unexpected(CopyErrorKind::RemoveOld); } } std::filesystem::create_directories(target_path.parent_path(), ec); if (ec) { - return tl::unexpected(CopyErrorKind::CreateDirectories); + return std::unexpected(CopyErrorKind::CreateDirectories); } std::filesystem::create_hard_link(source_path, target_path, ec); if (ec) { - return tl::unexpected(CopyErrorKind::CreateLink); + return std::unexpected(CopyErrorKind::CreateLink); } return {}; diff --git a/src/terrainlib/octree/storage/Storage.h b/src/terrainlib/octree/storage/Storage.h index 7fbfc9eb..782a9c03 100644 --- a/src/terrainlib/octree/storage/Storage.h +++ b/src/terrainlib/octree/storage/Storage.h @@ -7,7 +7,7 @@ #include #include -#include +#include #include "mesh/io.h" #include "octree/Id.h" @@ -143,13 +143,13 @@ class Storage_ { } } - tl::expected load(const Id &id) const noexcept { + std::expected load(const Id &id) const noexcept { if (const auto value_opt = this->_cache.get(id)) { return value_opt.value(); } if (!this->_index.contains(id, true)) { - return tl::unexpected(Codec::file_not_found()); + return std::unexpected(Codec::file_not_found()); } const auto result = this->_inner.load(id); @@ -159,7 +159,7 @@ class Storage_ { return result; } - tl::expected save(const Id &id, const value_type &value) noexcept { + std::expected save(const Id &id, const value_type &value) noexcept { if (this->check_overwrite(id)) { LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); } @@ -172,9 +172,9 @@ class Storage_ { return result; } - tl::expected copy_from(const Id &id, const Storage_ &source) noexcept { + std::expected copy_from(const Id &id, const Storage_ &source) noexcept { if (!source._index.contains(id, true)) { - return tl::unexpected(CopyErrorKind::FileNotFound); + return std::unexpected(CopyErrorKind::FileNotFound); } if (this->check_overwrite(id)) { @@ -189,7 +189,7 @@ class Storage_ { return result; } - tl::expected copy_to(const Id &id, Storage_ &target) const noexcept { + std::expected copy_to(const Id &id, Storage_ &target) const noexcept { return target.copy_from(id, *this); } @@ -252,7 +252,7 @@ class Storage_ { } } - tl::expected save_or_create_index() noexcept { + std::expected save_or_create_index() noexcept { if (this->is_indexed() && !this->_index.dirty) { return {}; } diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h index fcf4f20f..1c705e05 100644 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ b/src/terrainlib/octree/storage/cache/LruCache.h @@ -11,6 +11,9 @@ namespace octree::cache { // TODO: UNTESTED +// Do not use this cache for NodeLoader's ancestor lookup until cached ancestors +// are clipped to the requested node bounds. NodeLoader currently returns a +// cached ancestor unchanged, producing incorrect geometry for child requests. template class Lru_ : public ICache { public: diff --git a/src/terrainlib/octree/storage/codec/Codec.h b/src/terrainlib/octree/storage/codec/Codec.h index b4c3bc47..f31b6f45 100644 --- a/src/terrainlib/octree/storage/codec/Codec.h +++ b/src/terrainlib/octree/storage/codec/Codec.h @@ -4,7 +4,7 @@ #include #include -#include +#include template concept CodecFor = @@ -16,10 +16,10 @@ concept CodecFor = requires std::same_as; { Codec::load_from_path(path) } noexcept - -> std::same_as>; + -> std::same_as>; { Codec::save_to_path(value, path) } noexcept - -> std::same_as>; + -> std::same_as>; { Codec::file_not_found() } noexcept -> std::same_as; diff --git a/src/terrainlib/octree/storage/codec/MeshCodec.h b/src/terrainlib/octree/storage/codec/MeshCodec.h index 2dc65fb6..7a15400a 100644 --- a/src/terrainlib/octree/storage/codec/MeshCodec.h +++ b/src/terrainlib/octree/storage/codec/MeshCodec.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "mesh/io.h" @@ -15,11 +15,11 @@ struct MeshCodec { using load_error = mesh::io::LoadMeshError; using save_error = mesh::io::SaveMeshError; - static tl::expected load_from_path(const std::filesystem::path& path) noexcept { + static std::expected load_from_path(const std::filesystem::path& path) noexcept { return mesh::io::load_from_path(path); } - static tl::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { + static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { return mesh::io::save_to_path(value, path); } diff --git a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h index 2d397cb2..3ea68ba9 100644 --- a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h +++ b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h @@ -1,9 +1,8 @@ #pragma once +#include #include -#include - #include "log.h" namespace octree { @@ -12,7 +11,7 @@ namespace octree { // read-only (e.g. a view that aliases another storage's files). template struct ReadOnlyCodec : Codec { - static tl::expected save_to_path( + static std::expected save_to_path( const typename Codec::value_type &, const std::filesystem::path &) noexcept { LOG_ERROR_AND_EXIT("Attempted to write through a read-only codec"); } diff --git a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h b/src/terrainlib/octree/storage/codec/ZppBitsCodec.h index dd70a27b..dc4acd5a 100644 --- a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h +++ b/src/terrainlib/octree/storage/codec/ZppBitsCodec.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "Codec.h" #include "io/serialize.h" @@ -18,11 +18,11 @@ struct ZppBitsCodec { using load_error = io::Error; using save_error = io::Error; - static tl::expected load_from_path(const std::filesystem::path& path) noexcept { + static std::expected load_from_path(const std::filesystem::path& path) noexcept { return io::read_from_path(path); } - static tl::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { + static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { return io::write_to_path(value, path); } diff --git a/src/terrainlib/octree/storage/helpers.cpp b/src/terrainlib/octree/storage/helpers.cpp index 7366be55..ed817515 100644 --- a/src/terrainlib/octree/storage/helpers.cpp +++ b/src/terrainlib/octree/storage/helpers.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include "octree/storage/helpers.h" #include "io/serialize.h" @@ -90,7 +90,7 @@ std::optional guess_layout_strategy( return std::nullopt; } -tl::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { +std::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { const auto index_path = layout.base_path() / disk::v1::index_file_name(); LOG_TRACE("Saving octree storage index to {}", index_path); @@ -102,7 +102,7 @@ tl::expected save_index_map(const IndexMap& index, const disk:: const auto result = io::write_to_path(index_file, index_path); if (!result.has_value()) { LOG_ERROR("Failed to save octree storage index to {}", index_path); - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } return {}; } diff --git a/src/terrainlib/octree/storage/helpers.h b/src/terrainlib/octree/storage/helpers.h index a7dff810..288096cf 100644 --- a/src/terrainlib/octree/storage/helpers.h +++ b/src/terrainlib/octree/storage/helpers.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "io/Error.h" #include "octree/IndexMap.h" @@ -22,7 +22,7 @@ struct LayoutWithoutBase { std::optional guess_layout_strategy( const std::filesystem::path &base_path, size_t max_files_to_check = 100); -tl::expected save_index_map(const IndexMap &index, const disk::Layout &layout); +std::expected save_index_map(const IndexMap &index, const disk::Layout &layout); void update_index_map(IndexMap &index, const disk::Layout &layout); } diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index ec212e88..b21ede08 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include "io/Error.h" #include "octree/disk/layout/strategy/Default.h" @@ -21,7 +21,7 @@ struct OpenOptions { }; template Codec = DefaultCodecFor> -tl::expected, io::Error> open_index(const std::filesystem::path &index_path); +std::expected, io::Error> open_index(const std::filesystem::path &index_path); template Codec = DefaultCodecFor> Storage_ open_folder( const std::filesystem::path &base_path, diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl index e0c9b0d1..c60c3699 100644 --- a/src/terrainlib/octree/storage/open.inl +++ b/src/terrainlib/octree/storage/open.inl @@ -4,7 +4,7 @@ #include #include -#include +#include #include "io/serialize.h" #include "log.h" @@ -19,13 +19,13 @@ namespace octree { template Codec> -tl::expected, io::Error> open_index(const std::filesystem::path &index_path) { +std::expected, io::Error> open_index(const std::filesystem::path &index_path) { LOG_TRACE("Opening storage index {}", index_path); const auto result = io::read_from_path(index_path); if (!result.has_value()) { LOG_TRACE("Failed to open storage index due to {}", result.error()); - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } auto index_file = result.value(); LOG_TRACE("Successfully read storage index with {} entries.", index_file.map.size()); @@ -93,7 +93,10 @@ Storage_ open_folder( IndexMap map; helpers::update_index_map(map, layout); if (!map.empty()) { - helpers::save_index_map(map, layout); + const auto save_result = helpers::save_index_map(map, layout); + if (!save_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to create storage index in {}: {}", base_path, save_result.error()); + } } return Storage_(RawStorage_(std::move(layout)), std::move(map)); } diff --git a/src/terrainlib/pch.h b/src/terrainlib/pch.h index dce9cf57..dfbd471c 100644 --- a/src/terrainlib/pch.h +++ b/src/terrainlib/pch.h @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include // Internal headers diff --git a/src/terrainlib/srs.h b/src/terrainlib/srs.h index af8308f5..a5e5ce5c 100644 --- a/src/terrainlib/srs.h +++ b/src/terrainlib/srs.h @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include namespace srs { @@ -352,19 +352,19 @@ inline radix::geometry::Aabb3d encompassing_bounds_transfer( return encompassing_bounds_transfer(transform.get(), source_bounds, intermediate_points_edges, intermediate_points_faces); } -inline tl::expected from_epsg(const uint32_t epsg) { +inline std::expected from_epsg(const uint32_t epsg) { OGRSpatialReference srs; if (srs.importFromEPSG(epsg) != OGRERR_NONE) { - return tl::unexpected(fmt::format("Failed to import spatial reference from EPSG code: {}", epsg)); + return std::unexpected(fmt::format("Failed to import spatial reference from EPSG code: {}", epsg)); } srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; } -inline tl::expected from_user_input(const std::string &user_input) { +inline std::expected from_user_input(const std::string &user_input) { OGRSpatialReference srs; if (srs.SetFromUserInput(user_input.c_str()) != OGRERR_NONE) { - return tl::unexpected(fmt::format("Failed to set spatial reference from user input: {}", user_input)); + return std::unexpected(fmt::format("Failed to set spatial reference from user input: {}", user_input)); } srs.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); return srs; diff --git a/src/terrainlib/tile_path.h b/src/terrainlib/tile_path.h new file mode 100644 index 00000000..8f866d80 --- /dev/null +++ b/src/terrainlib/tile_path.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +#include + +[[nodiscard]] inline std::filesystem::path google_tile_path( + const std::filesystem::path& base_path, + const radix::tile::Id& tile_id, + const std::string& extension) +{ + return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.x) / (std::to_string(tile_id.coords.y) + extension); +} diff --git a/src/terrainlib/uv/unwrap.cpp b/src/terrainlib/uv/unwrap.cpp index 1802ae91..9186c5de 100644 --- a/src/terrainlib/uv/unwrap.cpp +++ b/src/terrainlib/uv/unwrap.cpp @@ -130,7 +130,7 @@ struct CgalMap { double aspect = 1.0; }; -tl::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm algorithm, Border border) { +std::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm algorithm, Border border) { const cgal::HalfedgeDescriptor bhd = CGAL::Polygon_mesh_processing::longest_border(mesh).first; DEBUG_ASSERT(bhd != boost::graph_traits::null_halfedge()); @@ -179,7 +179,7 @@ tl::expected parameterize_mesh(cgal::Mesh &mesh, Algorithm } if (result != CGAL::Surface_mesh_parameterization::OK) { - return tl::unexpected(UnwrapError(result)); + return std::unexpected(UnwrapError(result)); } const auto vertex_count = CGAL::num_vertices(mesh); @@ -208,7 +208,7 @@ std::vector decode_uv_map(const UvMap &map, size_t vertex_count) { } } -tl::expected unwrap( +std::expected unwrap( const std::span triangles, const std::span positions, Algorithm algorithm, @@ -220,7 +220,7 @@ tl::expected unwrap( cgal::Mesh cgal_mesh = convert::to_cgal_mesh(mesh); auto result = parameterize_mesh(cgal_mesh, algorithm, border); if (!result) { - return tl::unexpected(result.error()); + return std::unexpected(result.error()); } const CgalMap &cgal_map = result.value(); Uvs uvs = decode_uv_map(cgal_map.uvs, mesh.vertex_count()); diff --git a/src/terrainlib/uv/unwrap.h b/src/terrainlib/uv/unwrap.h index f47967fd..98786483 100644 --- a/src/terrainlib/uv/unwrap.h +++ b/src/terrainlib/uv/unwrap.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -57,13 +57,13 @@ struct Map { inline constexpr Algorithm DEFAULT_ALGORITHM = Algorithm::TutteBarycentricMapping; inline constexpr Border DEFAULT_BORDER = Border::Circle; -tl::expected unwrap( +std::expected unwrap( const std::span triangles, const std::span positions, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER); -inline tl::expected unwrap( +inline std::expected unwrap( const std::vector& triangles, const std::vector& positions, Algorithm algorithm = DEFAULT_ALGORITHM, @@ -75,7 +75,7 @@ inline tl::expected unwrap( border); } -inline tl::expected unwrap( +inline std::expected unwrap( const mesh::View &mesh, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER) { @@ -86,7 +86,7 @@ inline tl::expected unwrap( border); } -inline tl::expected unwrap( +inline std::expected unwrap( const mesh::Simple &mesh, Algorithm algorithm = DEFAULT_ALGORITHM, Border border = DEFAULT_BORDER) { diff --git a/src/tile_builder/CMakeLists.txt b/src/tile_builder/CMakeLists.txt index d4464342..576df41c 100644 --- a/src/tile_builder/CMakeLists.txt +++ b/src/tile_builder/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(tilebuilderlib alpine_raster.cpp DatasetReader.cpp - Image.cpp + image_writer.cpp ParallelTileGenerator.cpp ParallelTiler.cpp TileHeightsGenerator.cpp diff --git a/src/tile_builder/DatasetReader.cpp b/src/tile_builder/DatasetReader.cpp index bad96340..a472ae54 100644 --- a/src/tile_builder/DatasetReader.cpp +++ b/src/tile_builder/DatasetReader.cpp @@ -29,7 +29,7 @@ #include "Dataset.h" #include "Exception.h" -#include "Image.h" +#include #include "ctb/types.hpp" #include "log.h" @@ -185,12 +185,12 @@ DatasetReader::DatasetReader(const std::shared_ptr& dataset, const OGRS throw Exception(fmt::format("Dataset does not contain band number {} (there are {} bands).", band, dataset->n_bands())); } -HeightData DatasetReader::read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { return readFrom(m_dataset, bounds, width, height); } -HeightData DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { #ifdef ALP_ENABLE_OVERVIEW_READING auto transformer_args = make_image_transform_args(*this, m_dataset.get(), bounds, width, height); @@ -202,7 +202,7 @@ HeightData DatasetReader::readWithOverviews(const radix::tile::SrsBounds& bounds #endif } -HeightData DatasetReader::readFrom(const std::shared_ptr& source_dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const +radix::Raster DatasetReader::readFrom(const std::shared_ptr& source_dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const { // if we have performance problems with the warping, it'd still be possible to approximate the warping operation with a linear transform (mostly when zoomed in / on higher zoom levels). // CTB does this in GDALTiler.cpp around line 375 ("// Decide if we are doing an approximate or exact transformation"). @@ -212,9 +212,9 @@ HeightData DatasetReader::readFrom(const std::shared_ptr& source_datase auto warped_dataset = Dataset(static_cast(GDALCreateWarpedVRT(source_dataset->gdalDataset(), int(width), int(height), adfGeoTransform.data(), warp_options.first.get()))); auto* heights_band = warped_dataset.gdalDataset()->GetRasterBand(1); // non-owning pointer - auto heights_data = HeightData(width, height); + auto heights_data = radix::Raster({ width, height }); if (heights_band->RasterIO(GF_Read, 0, 0, int(width), int(height), - static_cast(heights_data.data()), int(width), int(height), GDT_Float32, 0, 0) + static_cast(heights_data.buffer().data()), int(width), int(height), GDT_Float32, 0, 0) != CE_None) throw Exception("couldn't read data"); diff --git a/src/tile_builder/DatasetReader.h b/src/tile_builder/DatasetReader.h index e75b1ff8..a60400c6 100644 --- a/src/tile_builder/DatasetReader.h +++ b/src/tile_builder/DatasetReader.h @@ -23,7 +23,7 @@ #include #include -#include "Image.h" +#include #include class Dataset; @@ -33,8 +33,8 @@ class DatasetReader { public: DatasetReader(const std::shared_ptr& dataset, const OGRSpatialReference& targetSRS, unsigned band, bool warn_on_missing_overviews = true); - HeightData read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; - HeightData readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster read(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster readWithOverviews(const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; unsigned dataset_band() const { return m_band; } bool isReprojecting() const { return m_requires_reprojection; } @@ -42,7 +42,7 @@ class DatasetReader { std::string target_srs_wkt() const { return m_target_srs_wkt; } protected: - HeightData readFrom(const std::shared_ptr& dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; + radix::Raster readFrom(const std::shared_ptr& dataset, const radix::tile::SrsBounds& bounds, unsigned width, unsigned height) const; private: std::shared_ptr m_dataset; diff --git a/src/tile_builder/Image.cpp b/src/tile_builder/Image.cpp deleted file mode 100644 index 3b33465b..00000000 --- a/src/tile_builder/Image.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include -#include "Image.h" -#include - -void image::saveImageAsPng(const Image& inputImage, const std::string& path) -{ - const int width = static_cast(inputImage.width()); - const int height = static_cast(inputImage.height()); - - cv::Mat image(height, width, CV_8UC3); - - for (int row = 0; row < height; ++row) { - for (int col = 0; col < width; ++col) { - const glm::u8vec3& pixel = inputImage.pixel(height - row - 1, col); // Flip vertically - image.at(row, col) = cv::Vec3b(pixel.z, pixel.y, pixel.x); // RGB to BGR - } - } - - cv::imwrite(path, image); -} diff --git a/src/tile_builder/Image.h b/src/tile_builder/Image.h deleted file mode 100644 index 5785329d..00000000 --- a/src/tile_builder/Image.h +++ /dev/null @@ -1,105 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#ifndef IMAGE_H -#define IMAGE_H - -#include -#include -#include -#include -#include -#include - -#include - -namespace tntn { -template -class Raster; -} - -template -class Image { -public: - Image() = default; - Image(unsigned width, unsigned height) - : m_width(width) - , m_height(height) - , m_data(size_t(m_width * m_height)) - { - } - - [[nodiscard]] unsigned width() const { return m_width; } - [[nodiscard]] unsigned height() const { return m_height; } - [[nodiscard]] T pixel(unsigned row, unsigned column) const - { - assert(column < m_width); - assert(row < m_height); - assert(m_data.size() == size_t(m_width * m_height)); - return m_data[row * m_width + column]; - } - - [[nodiscard]] float* data() { return m_data.data(); } - - [[nodiscard]] auto size() const { return m_data.size(); } - [[nodiscard]] auto begin() { return m_data.begin(); } - [[nodiscard]] auto end() { return m_data.end(); } - [[nodiscard]] auto begin() const { return m_data.begin(); } - [[nodiscard]] auto end() const { return m_data.end(); } - -private: - unsigned m_width = 0; - unsigned m_height = 0; - std::vector m_data; - - friend class tntn::Raster; -}; - -using HeightData = Image; -using RgbImage = Image; -using uchar = unsigned char; - -namespace image { -void saveImageAsPng(const Image& image, const std::string& path); - -template -[[nodiscard]] auto transformImage(const Image& i, Fun conversion_fun) -> Image -{ - using T2 = decltype(conversion_fun(*i.begin())); - Image i2(i.width(), i.height()); - std::transform(i.begin(), i.end(), i2.begin(), conversion_fun); - return i2; -} - -template -void debugOut(const Image& image, const std::string& path) -{ - auto [min, max] = std::ranges::minmax(image); - - // [min=min, ..] is required for cpp correctness. min/max from the capture are not variables, we need to copy them: - // https://stackoverflow.com/questions/50799719/reference-to-local-binding-declared-in-enclosing-function?noredirect=1&lq=1 - saveImageAsPng(transformImage(image, [min = min, max = max](auto v) { - const auto c = uchar(255.F * (float(v) - float(min)) / float(max - min)); - return glm::u8vec3(c, c, c); - }), - path); -} -} - -#endif // HEIGHTDATA_H diff --git a/src/tile_builder/ParallelTileGenerator.cpp b/src/tile_builder/ParallelTileGenerator.cpp index e0d7e731..6a52a497 100644 --- a/src/tile_builder/ParallelTileGenerator.cpp +++ b/src/tile_builder/ParallelTileGenerator.cpp @@ -41,7 +41,7 @@ ParallelTileGenerator::ParallelTileGenerator(const std::string& input_data_path, } ParallelTileGenerator ParallelTileGenerator::make(const std::string& input_data_path, - ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, + ctb::Grid::Srs srs, std::unique_ptr tile_writer, const std::string& output_data_path, unsigned grid_resolution) @@ -51,7 +51,7 @@ ParallelTileGenerator ParallelTileGenerator::make(const std::string& input_data_ if (srs == ctb::Grid::Srs::SphericalMercator) grid = ctb::GlobalMercator(grid_resolution); const auto border = tile_writer->formatRequiresBorder(); - return { input_data_path, grid, ParallelTiler(grid, dataset->bounds(grid.getSRS()), border, tiling_scheme), std::move(tile_writer), output_data_path }; + return { input_data_path, grid, ParallelTiler(grid, dataset->bounds(grid.getSRS()), border), std::move(tile_writer), output_data_path }; } const ParallelTiler& ParallelTileGenerator::tiler() const @@ -64,7 +64,7 @@ const ctb::Grid& ParallelTileGenerator::grid() const return m_grid; } -void ParallelTileGenerator::write(const radix::tile::Descriptor& tile, const HeightData& heights) const +void ParallelTileGenerator::write(const radix::tile::Descriptor& tile, const radix::Raster& heights) const { const auto dir_path = fmt::format("{}/{}/{}", m_output_data_path, tile.id.zoom_level, tile.id.coords.x); const auto file_path = fmt::format("{}/{}.{}", dir_path, tile.id.coords.y, m_tile_writer->formatFileEnding()); diff --git a/src/tile_builder/ParallelTileGenerator.h b/src/tile_builder/ParallelTileGenerator.h index f95a412f..cd6f6240 100644 --- a/src/tile_builder/ParallelTileGenerator.h +++ b/src/tile_builder/ParallelTileGenerator.h @@ -22,7 +22,7 @@ #include #include -#include "Image.h" +#include #include "ParallelTiler.h" #include "ctb/Grid.hpp" #include "ctb/types.hpp" @@ -41,7 +41,7 @@ class ParallelTileGenerator { public: ParallelTileGenerator(const std::string& input_data_path, const ctb::Grid& grid, const ParallelTiler& tiler, std::unique_ptr tile_writer, const std::string& output_data_path); [[nodiscard]] static ParallelTileGenerator make(const std::string& input_data_path, - ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, + ctb::Grid::Srs srs, std::unique_ptr tile_writer, const std::string& output_data_path, unsigned grid_resolution = 256); @@ -49,7 +49,7 @@ class ParallelTileGenerator { void setWarnOnMissingOverviews(bool flag) { m_warn_on_missing_overviews = flag; } [[nodiscard]] const ParallelTiler& tiler() const; [[nodiscard]] const ctb::Grid& grid() const; - void write(const radix::tile::Descriptor& tile, const HeightData& heights) const; + void write(const radix::tile::Descriptor& tile, const radix::Raster& heights) const; void process(const std::pair& zoom_range, bool progress_bar_on_console = false, bool generate_world_wide_tiles = false) const; }; @@ -68,7 +68,7 @@ class ParallelTileWriterInterface { virtual ~ParallelTileWriterInterface() = default; ParallelTileWriterInterface& operator=(const ParallelTileWriterInterface&) = default; ParallelTileWriterInterface& operator=(ParallelTileWriterInterface&&) = default; - virtual void write(const std::string& file_path, const radix::tile::Descriptor& tile, const HeightData& heights) const = 0; + virtual void write(const std::string& file_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const = 0; [[nodiscard]] radix::tile::Border formatRequiresBorder() const; [[nodiscard]] const std::string& formatFileEnding() const; }; diff --git a/src/tile_builder/ParallelTiler.cpp b/src/tile_builder/ParallelTiler.cpp index 3f1c2958..fe6b2330 100644 --- a/src/tile_builder/ParallelTiler.cpp +++ b/src/tile_builder/ParallelTiler.cpp @@ -22,32 +22,31 @@ #include "Exception.h" #include -ParallelTiler::ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) : Tiler(grid, bounds, border, scheme) +ParallelTiler::ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) : Tiler(grid, bounds, border) { } radix::tile::Id ParallelTiler::southWestTile(unsigned zoom_level) const { - return grid().crsToTile(bounds().min, zoom_level).to(scheme()); + return grid().crsToTile(bounds().min, zoom_level); } radix::tile::Id ParallelTiler::northEastTile(unsigned zoom_level) const { const auto epsilon = grid().resolution(zoom_level) / 100; - return grid().crsToTile(bounds().max - epsilon, zoom_level).to(scheme()); + return grid().crsToTile(bounds().max - epsilon, zoom_level); } std::vector ParallelTiler::generateTiles(unsigned zoom_level) const { - // in the tms scheme south west corresponds to the smaller numbers. hence we can iterate from sw to ne - const auto sw = southWestTile(zoom_level).to(radix::tile::Scheme::Tms).coords; - const auto ne = northEastTile(zoom_level).to(radix::tile::Scheme::Tms).coords; + const auto sw = southWestTile(zoom_level).coords; + const auto ne = northEastTile(zoom_level).coords; std::vector tiles; - tiles.reserve((ne.y - sw.y + 1) * (ne.x - sw.x + 1)); - for (auto ty = sw.y; ty <= ne.y; ++ty) { + tiles.reserve((sw.y - ne.y + 1) * (ne.x - sw.x + 1)); + for (auto ty = ne.y; ty <= sw.y; ++ty) { for (auto tx = sw.x; tx <= ne.x; ++tx) { - const auto tile_id = radix::tile::Id { zoom_level, { tx, ty }, radix::tile::Scheme::Tms }.to(scheme()); + const auto tile_id = radix::tile::Id { zoom_level, { tx, ty } }; tiles.emplace_back(tile_for(tile_id)); if (tiles.size() >= 1'000'000'000) // think about creating an on the fly tile generator. storing so many tiles takes a lot of memory. diff --git a/src/tile_builder/ParallelTiler.h b/src/tile_builder/ParallelTiler.h index 7d01f0de..62cd07ac 100644 --- a/src/tile_builder/ParallelTiler.h +++ b/src/tile_builder/ParallelTiler.h @@ -23,7 +23,7 @@ class ParallelTiler : public Tiler { public: - ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + ParallelTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); [[nodiscard]] std::vector generateTiles(unsigned zoom_level) const; [[nodiscard]] std::vector generateTiles(const std::pair& zoom_range) const; diff --git a/src/tile_builder/TileHeightsGenerator.cpp b/src/tile_builder/TileHeightsGenerator.cpp index 6c816dde..0e535fab 100644 --- a/src/tile_builder/TileHeightsGenerator.cpp +++ b/src/tile_builder/TileHeightsGenerator.cpp @@ -28,10 +28,9 @@ #include "depth_first_tile_traverser.h" #include -TileHeightsGenerator::TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Scheme scheme, radix::tile::Border border, std::filesystem::path output_path) +TileHeightsGenerator::TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Border border, std::filesystem::path output_path) : m_input_data_path(std::move(input_data_path)) , m_srs(srs) - , m_scheme(scheme) , m_border(border) , m_output_path(std::move(output_path)) { @@ -51,7 +50,7 @@ void TileHeightsGenerator::run(unsigned max_zoom_level) const grid = ctb::GlobalMercator(64); const auto bounds = dataset->bounds(grid.getSRS()); const auto tile_reader = DatasetReader(dataset, grid.getSRS(), 1, false); - const auto tiler = TopDownTiler(grid, bounds, m_border, m_scheme); + const auto tiler = TopDownTiler(grid, bounds, m_border); auto tile_heights = radix::TileHeights(); const auto read_function = [&](const radix::tile::Descriptor& tile) -> MinMaxData { @@ -74,10 +73,10 @@ void TileHeightsGenerator::run(unsigned max_zoom_level) const }; - traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 }, m_scheme }, max_zoom_level); + traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 } }, max_zoom_level); if (m_srs == ctb::Grid::Srs::WGS84) { // two root tiles - traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 1, 0 }, m_scheme }, max_zoom_level); + traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 1, 0 } }, max_zoom_level); } tile_heights.write_to(m_output_path); diff --git a/src/tile_builder/TileHeightsGenerator.h b/src/tile_builder/TileHeightsGenerator.h index 0d64af56..703a1227 100644 --- a/src/tile_builder/TileHeightsGenerator.h +++ b/src/tile_builder/TileHeightsGenerator.h @@ -29,11 +29,9 @@ class TileHeightsGenerator { std::string m_input_data_path; ctb::Grid::Srs m_srs; - radix::tile::Scheme m_scheme; radix::tile::Border m_border; std::filesystem::path m_output_path; public: - TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Scheme scheme, radix::tile::Border border, std::filesystem::path output_path); + TileHeightsGenerator(std::string input_data_path, ctb::Grid::Srs srs, radix::tile::Border border, std::filesystem::path output_path); void run(unsigned max_zoom_level) const; }; - diff --git a/src/tile_builder/Tiler.cpp b/src/tile_builder/Tiler.cpp index 00153c6c..fafa3ba7 100644 --- a/src/tile_builder/Tiler.cpp +++ b/src/tile_builder/Tiler.cpp @@ -20,11 +20,10 @@ #include -Tiler::Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) +Tiler::Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) : m_grid(std::move(grid)) , m_bounds(bounds) , m_border_south_east(border) - , m_scheme(scheme) { } @@ -55,11 +54,6 @@ radix::tile::Descriptor Tiler::tile_for(const radix::tile::Id& tile_id) const return {tile_id, srs_bounds, grid().getEpsgCode(), grid_size(), tile_size()}; } -radix::tile::Scheme Tiler::scheme() const -{ - return m_scheme; -} - const radix::tile::SrsBounds& Tiler::bounds() const { return m_bounds; @@ -69,4 +63,3 @@ void Tiler::setBounds(const radix::tile::SrsBounds& newBounds) { m_bounds = newBounds; } - diff --git a/src/tile_builder/Tiler.h b/src/tile_builder/Tiler.h index d477ce3b..70b5049e 100644 --- a/src/tile_builder/Tiler.h +++ b/src/tile_builder/Tiler.h @@ -25,9 +25,8 @@ class Tiler { public: - Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + Tiler(ctb::Grid grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); - [[nodiscard]] radix::tile::Scheme scheme() const; [[nodiscard]] const radix::tile::SrsBounds& bounds() const; void setBounds(const radix::tile::SrsBounds& newBounds); [[nodiscard]] radix::tile::Descriptor tile_for(const radix::tile::Id& tile_id) const; @@ -43,6 +42,4 @@ class Tiler const ctb::Grid m_grid; radix::tile::SrsBounds m_bounds; const radix::tile::Border m_border_south_east; - const radix::tile::Scheme m_scheme; }; - diff --git a/src/tile_builder/TopDownTiler.cpp b/src/tile_builder/TopDownTiler.cpp index 9962962e..ca5ead02 100644 --- a/src/tile_builder/TopDownTiler.cpp +++ b/src/tile_builder/TopDownTiler.cpp @@ -18,15 +18,14 @@ #include "TopDownTiler.h" -TopDownTiler::TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme) - : Tiler(grid, bounds, border, scheme) +TopDownTiler::TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border) + : Tiler(grid, bounds, border) { } std::vector TopDownTiler::generateTiles(const radix::tile::Id& parent_id) const { - assert(parent_id.scheme == scheme()); - const auto tile_ids = parent_id.to(scheme()).children(); + const auto tile_ids = parent_id.children(); std::vector tiles; for (const auto& tile_id : tile_ids) { radix::tile::Descriptor t = tile_for(tile_id); diff --git a/src/tile_builder/TopDownTiler.h b/src/tile_builder/TopDownTiler.h index a2e065d1..f078ca89 100644 --- a/src/tile_builder/TopDownTiler.h +++ b/src/tile_builder/TopDownTiler.h @@ -24,7 +24,7 @@ class TopDownTiler : public Tiler { public: - TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border, radix::tile::Scheme scheme); + TopDownTiler(const ctb::Grid& grid, const radix::tile::SrsBounds& bounds, radix::tile::Border border); [[nodiscard]] std::vector generateTiles(const radix::tile::Id& parent_id) const; }; diff --git a/src/tile_builder/algorithms/primitives.h b/src/tile_builder/algorithms/primitives.h deleted file mode 100644 index a6101364..00000000 --- a/src/tile_builder/algorithms/primitives.h +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include - -#ifndef ALGORITHMS_PRIMITIVES_H -#define ALGORITHMS_PRIMITIVES_H - -namespace primitives { - -// two times the area of the ccw triangle with vertices a, b, and c. -// negative, if the triangle is cw -template -inline T triAreaX2(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - // doesn't work for unsigned types. if you need that, come up with something :) - static_assert(std::is_signed_v); - return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); -} - -enum class Winding : char { - CW = -1, - Undefined = 0, - CCW = 1 -}; - -template -inline Winding winding(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - if constexpr (is_integral) { - using Signed = typename std::conditional::type, T>::type; - auto v = (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) - (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - if (v == 0) - return Winding::Undefined; - if (v < 0) - return Winding::CW; - return Winding::CCW; - } else { - auto v = (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x); - if (std::abs(v) < 0.0000000000000001) - return Winding::Undefined; - if (v < 0) - return Winding::CW; - return Winding::CCW; - } -} - -template -inline bool ccw(const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - if constexpr (is_integral) { - using Signed = typename std::conditional::type, T>::type; - if constexpr (include_border) - return (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) >= (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - return (Signed(b.x) - Signed(a.x)) * (Signed(c.y) - Signed(a.y)) > (Signed(b.y) - Signed(a.y)) * (Signed(c.x) - Signed(a.x)); - } - if constexpr (include_border) - return (b.x - a.x) * (c.y - a.y) >= (b.y - a.y) * (c.x - a.x); - return (b.x - a.x) * (c.y - a.y) > (b.y - a.y) * (c.x - a.x); -} - -template -inline bool rightOf(const glm::tvec2& x, const glm::tvec2& org, const glm::tvec2& dest) -{ - return ccw(x, dest, org); -} - -template -inline bool leftOf(const glm::tvec2& x, const glm::tvec2& org, const glm::tvec2& dest) -{ - return ccw(x, org, dest); -} - -// https://www.scratchapixel.com/lessons/3d-basic-rendering/rasterization-practical-implementation/rasterization-stage -// the bottom edge of the raster is not included! -template -inline bool inside(const glm::tvec2& x, const glm::tvec2& a, const glm::tvec2& b, const glm::tvec2& c) -{ - constexpr bool is_integral = std::is_integral_v; - using Signed = typename std::conditional::type, T>::type; - using sVec = glm::tvec2; - // return leftOf(x, a, b) && leftOf(x, b, c) && leftOf(x, c, a); - const auto ccw = [](Winding w) { return w == Winding::CCW; }; - const auto undef = [](Winding w) { return w == Winding::Undefined; }; - const auto topleftedge = [](const sVec& edge) { return (edge.y == 0 && edge.x < 0) || edge.y < 0; }; - - const auto w_ab = winding(x, a, b); - const auto w_bc = winding(x, b, c); - const auto w_ca = winding(x, c, a); - const auto e_ab = sVec(b) - sVec(a); - const auto e_bc = sVec(c) - sVec(b); - const auto e_ca = sVec(a) - sVec(c); - - bool overlap = true; - overlap &= ccw(w_ab) || (undef(w_ab) && topleftedge(e_ab)); - overlap &= ccw(w_bc) || (undef(w_bc) && topleftedge(e_bc)); - overlap &= ccw(w_ca) || (undef(w_ca) && topleftedge(e_ca)); - return overlap; -} -} - -#endif diff --git a/src/tile_builder/algorithms/raster_triangle_scanline.h b/src/tile_builder/algorithms/raster_triangle_scanline.h deleted file mode 100644 index 7e22fdca..00000000 --- a/src/tile_builder/algorithms/raster_triangle_scanline.h +++ /dev/null @@ -1,101 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#ifndef ALGORITHMS_RASTER_TRIANGLE_SCANLINE_H -#define ALGORITHMS_RASTER_TRIANGLE_SCANLINE_H - -#include "primitives.h" -#include "tntn/Raster.h" -#include -#include -#include - -namespace raster { - -template -void triangle_scanline(const tntn::Raster& raster, const glm::uvec2& a, const glm::uvec2& b, const glm::uvec2& c, const Lambda& fun) -{ - assert(primitives::winding(a, b, c) == primitives::Winding::CCW); - - const auto min = glm::min(glm::min(a, b), c); - const auto max = glm::max(glm::max(a, b), c); - for (auto y = min.y; y <= max.y; ++y) { - for (auto x = min.x; x <= max.x; ++x) { - const auto coord = glm::uvec2(x, y); - if (primitives::inside(coord, a, b, c)) - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - if (min.y == 0) { - // bottom row of raster is not included in primitives::inside, so check for it and make an extrawurscht. - const auto walk_bottom = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).y == 0 && a.y == 0) { - const auto end_x = std::max(a.x, b.x); - for (auto x = std::min(a.x, b.x); x < end_x; ++x) { - const auto coord = glm::uvec2(x, 0); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - }; - walk_bottom(a, b); - walk_bottom(b, c); - walk_bottom(c, a); - } - const auto last_x = raster.get_width() - 1; - if (max.x == last_x) { - // similar with the rightmost column - const auto walk_right = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).x == 0 && a.x == last_x) { - const auto end_y = std::max(a.y, b.y); - for (auto y = std::min(a.y, b.y); y < end_y; ++y) { - const auto coord = glm::uvec2(last_x, y); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - } - }; - walk_right(a, b); - walk_right(b, c); - walk_right(c, a); - } - // if (max.x == last_x && min.y == 0) { - // const auto check_br = [&](const glm::uvec2& a, const glm::uvec2& b) { - // if ((b - a).y == 0 && a.y == 0 && b.x == last_x) { - // const auto coord = glm::uvec2(last_x, 0); - // fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - // } - // }; - // check_br(a, b); - // check_br(b, c); - // check_br(c, a); - // } - const auto last_y = raster.get_height() - 1; - if (max.x == last_x && max.y == last_y) { - const auto check_tr = [&](const glm::uvec2& a, const glm::uvec2& b) { - if ((b - a).x == 0 && b.y == last_y && b.x == last_x) { - const auto coord = glm::uvec2(last_x, last_y); - fun(coord, raster.value(coord.y, coord.x)); // raster is row / column - } - }; - check_tr(a, b); - check_tr(b, c); - check_tr(c, a); - } -} -} -#endif diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index b9ce5353..c5d8a6b0 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -25,18 +25,18 @@ #include #include -#include "Image.h" #include "ParallelTileGenerator.h" #include "ctb/Grid.hpp" +#include "image_writer.h" +#include #include -ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution) +ParallelTileGenerator alpine_raster::make_generator(const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, radix::tile::Border border, unsigned grid_resolution) { - return ParallelTileGenerator::make(input_data_path, srs, tiling_scheme, std::make_unique(border), output_data_path, grid_resolution); + return ParallelTileGenerator::make(input_data_path, srs, std::make_unique(border), output_data_path, grid_resolution); } -void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const HeightData& heights) const +void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const { - image::saveImageAsPng(image::transformImage(heights, radix::height_encoding::to_rgb), - file_path); + image::saveImageAsPng(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); } diff --git a/src/tile_builder/alpine_raster.h b/src/tile_builder/alpine_raster.h index 966d55fb..d82f05a6 100644 --- a/src/tile_builder/alpine_raster.h +++ b/src/tile_builder/alpine_raster.h @@ -25,7 +25,7 @@ #include #include -#include "Image.h" +#include #include "ParallelTileGenerator.h" #include #include "ctb/Grid.hpp" @@ -38,13 +38,12 @@ class TileWriter : public ParallelTileWriterInterface { : ParallelTileWriterInterface(border, "png") { } - void write(const std::string& base_path, const radix::tile::Descriptor& tile, const HeightData& heights) const override; + void write(const std::string& base_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const override; }; [[nodiscard]] ParallelTileGenerator make_generator( const std::string& input_data_path, const std::string& output_data_path, ctb::Grid::Srs srs, - radix::tile::Scheme tiling_scheme, radix::tile::Border border, unsigned grid_resolution = 256); }; diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp new file mode 100644 index 00000000..725cdfae --- /dev/null +++ b/src/tile_builder/image_writer.cpp @@ -0,0 +1,46 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * Copyright (C) 2025 Martin Braunsperger + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#include "image_writer.h" + +#include +#include + +void image::saveImageAsPng(const radix::Raster& input_image, const std::string& path) +{ + const int width = static_cast(input_image.width()); + const int height = static_cast(input_image.height()); + + cv::Mat image(height, width, CV_8UC3); + + for (int row = 0; row < height; ++row) { + for (int column = 0; column < width; ++column) { + const auto& pixel = input_image.pixel({ static_cast(column), static_cast(height - row - 1) }); + image.at(row, column) = cv::Vec3b(pixel.z, pixel.y, pixel.x); + } + } + + try { + if (!cv::imwrite(path, image)) + throw std::runtime_error("Failed to write PNG image to " + path); + } catch (const cv::Exception& error) { + throw std::runtime_error("Failed to write PNG image to " + path + ": " + error.what()); + } +} diff --git a/src/tile_builder/image_writer.h b/src/tile_builder/image_writer.h new file mode 100644 index 00000000..db8ff8f9 --- /dev/null +++ b/src/tile_builder/image_writer.h @@ -0,0 +1,51 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * Copyright (C) 2025 Martin Braunsperger + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace image { + +void saveImageAsPng(const radix::Raster& image, const std::string& path); + +template +void debugOut(const radix::Raster& image, const std::string& path) +{ + if (image.buffer().empty()) + throw std::invalid_argument("Can't write an empty raster to " + path); + + const auto [min, max] = std::ranges::minmax(image); + const auto range = float(max) - float(min); + saveImageAsPng(radix::raster::transform(image, [min, range](const auto value) { + const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); + return glm::u8vec3(intensity); + }), + path); +} + +} // namespace image diff --git a/src/tile_builder/main.cpp b/src/tile_builder/main.cpp index c2f0fe05..06ec1c93 100644 --- a/src/tile_builder/main.cpp +++ b/src/tile_builder/main.cpp @@ -12,28 +12,26 @@ int main() { // const std::string input_raster = "/home/madam/rajaton/raw/Oe_2020/OeRect_01m_gs_31287.img"; // const std::string output_path = "/home/madam/rajaton/tiles/atb_terrain/"; - //// const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, Tiler::Scheme::SlippyMap, Tiler::Border::No); + //// const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::No); //// generator.process({16, 16}); - // const auto generator = cesium_tin_terra::make_generator(input_raster, output_path, ctb::Grid::Srs::WGS84, Tiler::Scheme::Tms, Tiler::Border::Yes); // generator.process({0, 5}, true, true); // generator.process({6, 16}, true, false); const std::string input_raster = "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img"; // const std::string input_raster = "/home/madam/valtava/raw/vienna/innenstadt_gs_1m_mgi.tif"; const std::string output_path = "/home/madam/valtava/tiles/alpine_png2"; - // const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, Tiler::Scheme::SlippyMap, Tiler::Border::No); + // const auto generator = alpine_raster::make_generator("./test_tiles/", "/home/madam/valtava/raw/Oe_2020/OeRect_01m_gs_31287.img", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::No); // generator.process({16, 16}); - const auto generator = alpine_raster::make_generator(input_raster, output_path, ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, 64); + const auto generator = alpine_raster::make_generator(input_raster, output_path, ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, 64); // generator.process({0, 5}, true, true); generator.process({ 15, 16 }, true, false); - // const auto metadata = MetaDataGenerator::make(input_raster, ctb::Grid::Srs::WGS84, Tiler::Scheme::Tms); // const auto json = layer_json_writer::process(metadata); //// generate height data (min and max) for tiles up to level 13 // const auto base_path = std::filesystem::path(output_path); // constexpr auto file_name = "height_data.atb"; -// const auto generator = TileHeightsGenerator(input_raster, ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); +// const auto generator = TileHeightsGenerator(input_raster, ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, base_path / file_name); // generator.run(13); return 0; diff --git a/src/tile_downloader/HttpClient.h b/src/tile_downloader/HttpClient.h index 4db7e230..d5ef1ca4 100644 --- a/src/tile_downloader/HttpClient.h +++ b/src/tile_downloader/HttpClient.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include #include +#include #include #include @@ -18,7 +20,10 @@ using ProgressFn = std::function; class HttpClient { public: - HttpClient() : _curl(curl_easy_init()) { + using WriteFn = std::function &, const char *, size_t)>; + + explicit HttpClient(WriteFn write = {}) + : _curl(curl_easy_init()), _write(std::move(write)) { if (!_curl) { throw std::runtime_error("failed to init cURL"); } @@ -35,10 +40,12 @@ class HttpClient { HttpResponse get(const std::string &url, const ProgressFn &on_progress = {}) const { HttpResponse response; + WriteContext write_context{&response.body, &this->_write, {}}; + ProgressContext progress_context{&on_progress, {}}; curl_easy_setopt(this->_curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(this->_curl, CURLOPT_WRITEFUNCTION, write_cb); - curl_easy_setopt(this->_curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(this->_curl, CURLOPT_WRITEDATA, &write_context); curl_easy_setopt(this->_curl, CURLOPT_TIMEOUT, 5L); curl_easy_setopt(this->_curl, CURLOPT_CONNECTTIMEOUT, 5L); curl_easy_setopt(this->_curl, CURLOPT_FAILONERROR, 1L); @@ -46,12 +53,18 @@ class HttpClient { if (on_progress) { curl_easy_setopt(this->_curl, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(this->_curl, CURLOPT_XFERINFOFUNCTION, progress_cb); - curl_easy_setopt(this->_curl, CURLOPT_XFERINFODATA, &on_progress); + curl_easy_setopt(this->_curl, CURLOPT_XFERINFODATA, &progress_context); } else { curl_easy_setopt(this->_curl, CURLOPT_NOPROGRESS, 1L); } response.curl_code = curl_easy_perform(this->_curl); + if (write_context.exception) { + std::rethrow_exception(write_context.exception); + } + if (progress_context.exception) { + std::rethrow_exception(progress_context.exception); + } char *ct = nullptr; if (curl_easy_getinfo(this->_curl, CURLINFO_CONTENT_TYPE, &ct) == CURLE_OK && ct) { @@ -68,23 +81,50 @@ class HttpClient { } private: - CURL *_curl; + struct WriteContext { + std::vector *buffer; + const WriteFn *write; + std::exception_ptr exception; + }; - static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) { - auto &buf = *static_cast *>(userdata); - size_t total = size * nmemb; - buf.insert(buf.end(), static_cast(ptr), static_cast(ptr) + total); - return total; + struct ProgressContext { + const ProgressFn *progress; + std::exception_ptr exception; + }; + + CURL *_curl; + WriteFn _write; + + static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) noexcept { + auto &context = *static_cast(userdata); + const size_t total = size * nmemb; + try { + const auto *data = static_cast(ptr); + if (*context.write) { + (*context.write)(*context.buffer, data, total); + } else { + context.buffer->insert(context.buffer->end(), data, data + total); + } + return total; + } catch (...) { + context.exception = std::current_exception(); + return 0; + } } static int progress_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow, - curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) { - const auto &fn = *static_cast(clientp); - if (dltotal > 0) { - fn(double(dlnow) / double(dltotal)); - } else { - fn(-1.0); + curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) noexcept { + auto &context = *static_cast(clientp); + try { + if (dltotal > 0) { + (*context.progress)(double(dlnow) / double(dltotal)); + } else { + (*context.progress)(-1.0); + } + return 0; + } catch (...) { + context.exception = std::current_exception(); + return 1; } - return 0; } }; diff --git a/src/tile_downloader/TileDownloader.h b/src/tile_downloader/TileDownloader.h index 4678376c..e1f49bf4 100644 --- a/src/tile_downloader/TileDownloader.h +++ b/src/tile_downloader/TileDownloader.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -12,85 +11,81 @@ #include "HttpClient.h" #include "TileLogger.h" #include "TileUrlBuilder.h" +#include "tile_path.h" +#include "write_file.h" class TileDownloader { public: - TileDownloader(const TileUrlBuilder &url_builder, std::string output_pattern, - bool early_skip, std::optional max_zoom_level, unsigned root_zoom_level) + TileDownloader(const TileUrlBuilder &url_builder, std::filesystem::path output_directory, + std::optional max_zoom_level, unsigned root_zoom_level) : _url_builder(url_builder), - _output_pattern(std::move(output_pattern)), + _output_directory(std::move(output_directory)), _logger(root_zoom_level), - _early_skip(early_skip), _max_zoom_level(max_zoom_level) {} - void download_recursive(const radix::tile::Id &root_id) { - this->_logger.start(); - this->download_recursive_core(root_id); - this->_logger.finish(); + [[nodiscard]] bool download_recursive(const radix::tile::Id &root_id) { + auto progress_session = this->_logger.start(); + const bool complete = this->download_recursive_core(root_id); + progress_session.finish(); + return complete; } private: const TileUrlBuilder &_url_builder; - std::string _output_pattern; + std::filesystem::path _output_directory; HttpClient _http; TileLogger _logger; - bool _early_skip; std::optional _max_zoom_level; - void download_recursive_core(const radix::tile::Id &root_id) { + [[nodiscard]] bool download_recursive_core(const radix::tile::Id &root_id) { auto result = this->download_tile(root_id); this->_logger.report_error(root_id, result); + if (std::holds_alternative(result)) { + this->_logger.skipped(root_id); + return true; + } + + if (std::holds_alternative(result)) { + this->_logger.missing(root_id); + return true; + } + if (is_failure(result)) { this->_logger.missing(root_id); - return; + return false; } const auto children = root_id.children(); + bool children_complete = true; for (size_t i = 0; i < children.size(); i++) { - if (this->_early_skip && i + 1 < children.size()) { - if (this->tile_exists(children[i + 1])) { - this->_logger.skipped(children[i]); - continue; - } - } - if (this->_max_zoom_level.has_value() && children[i].zoom_level > *this->_max_zoom_level) { this->_logger.skipped(children[i]); continue; } - this->download_recursive_core(children[i]); + if (!this->download_recursive_core(children[i])) { + children_complete = false; + } + } + + if (!children_complete) { + return false; } + + mark_tile_children_complete(this->tile_path(root_id)); + return true; } static bool is_failure(const TileResult::Status &result) { - return std::holds_alternative(result) - || std::holds_alternative(result) + return std::holds_alternative(result) || std::holds_alternative(result) || std::holds_alternative(result) || std::holds_alternative(result); } - bool tile_exists(const radix::tile::Id &tile) const { - return std::filesystem::exists(this->tile_path(tile)); - } - - std::string tile_path(const radix::tile::Id &tile) const { - std::string path = this->_output_pattern; - replace_all(path, "{zoom}", std::to_string(tile.zoom_level)); - replace_all(path, "{x}", std::to_string(tile.coords.x)); - replace_all(path, "{y}", std::to_string(tile.coords.y)); - replace_all(path, "{ext}", "jpeg"); - return path; - } - - static void replace_all(std::string &s, std::string_view find, std::string_view replace) { - size_t pos = 0; - while ((pos = s.find(find, pos)) != std::string::npos) { - s.replace(pos, find.length(), replace); - pos += replace.length(); - } + std::filesystem::path tile_path(const radix::tile::Id &tile) const { + return google_tile_path(_output_directory, tile, ".jpeg"); } static void ensure_parent_dirs(const std::filesystem::path &path) { @@ -101,20 +96,15 @@ class TileDownloader { } } - static void write_file(const std::filesystem::path &path, const std::vector &data) { - std::ofstream out(path, std::ios::binary); - if (!out) { - throw std::runtime_error(fmt::format("failed to open \"{}\" for writing", path.string())); - } - out.write(data.data(), data.size()); - } - TileResult::Status download_tile(const radix::tile::Id &tile) { const auto path = std::filesystem::absolute(this->tile_path(tile)); if (std::filesystem::exists(path)) { return TileResult::Skipped{}; } + if (std::filesystem::exists(children_pending_tile_path(path))) { + return TileResult::ChildrenPending{}; + } ensure_parent_dirs(path); @@ -124,7 +114,7 @@ class TileDownloader { HttpResponse response = this->_http.get(url); if (response.curl_code == CURLE_OK && this->_http.is_image(response)) { - write_file(path, response.body); + write_file_children_pending(path, response.body); return TileResult::Downloaded{}; } diff --git a/src/tile_downloader/TileLogger.h b/src/tile_downloader/TileLogger.h index 3919e74c..c9d6c81b 100644 --- a/src/tile_downloader/TileLogger.h +++ b/src/tile_downloader/TileLogger.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ struct TileResult { struct Downloaded {}; + struct ChildrenPending {}; struct Skipped {}; struct Absent {}; struct HttpError { long status_code; }; @@ -26,18 +28,17 @@ struct TileResult { struct CurlError { CURLcode code; }; struct TimedOut {}; - using Status = std::variant; + using Status = std::variant; }; class TileLogger { public: + class Session; + explicit TileLogger(unsigned root_zoom_level) : _root_zoom_level(root_zoom_level), _progress(PROGRESS_RESOLUTION) {} - // Starts the progress display; pair with a call to finish(). - void start() { - this->_progress_thread = this->_progress.start_monitoring(); - } + [[nodiscard]] Session start(); // The tile turned out not to exist (or errored out), so its branch stops here. void missing(const radix::tile::Id &tile) { @@ -64,9 +65,20 @@ class TileLogger { }, status); } - // Fills in any steps still missing due to floating point rounding, so the - // progress indicator reaches its total, then joins the thread started by start(). - void finish() { +private: + static constexpr size_t PROGRESS_RESOLUTION = 10'000; + + unsigned _root_zoom_level; + ProgressIndicator _progress; + std::jthread _progress_thread; + size_t _steps_done = 0; + std::map _level_counts; + + void start_monitoring() { + this->_progress_thread = this->_progress.start_monitoring(); + } + + void finish_monitoring() { while (this->_steps_done < PROGRESS_RESOLUTION) { this->_progress.task_finished(); this->_steps_done++; @@ -76,14 +88,18 @@ class TileLogger { } } -private: - static constexpr size_t PROGRESS_RESOLUTION = 10'000; + void cancel_monitoring() noexcept { + if (!this->_progress_thread.joinable()) { + return; + } - unsigned _root_zoom_level; - ProgressIndicator _progress; - std::jthread _progress_thread; - size_t _steps_done = 0; - std::map _level_counts; + this->_progress_thread.request_stop(); + try { + this->_progress_thread.join(); + } catch (...) { + // Session cleanup must not replace the active exception. + } + } static std::string format(const radix::tile::Id &tile) { return fmt::format("Tile[Zoom={}, X={}, Y={}]", tile.zoom_level, tile.coords.x, tile.coords.y); @@ -114,3 +130,37 @@ class TileLogger { } } }; + +class TileLogger::Session { +public: + explicit Session(TileLogger &logger) + : _logger(&logger) { + this->_logger->start_monitoring(); + } + + Session(const Session &) = delete; + Session &operator=(const Session &) = delete; + + Session(Session &&other) noexcept + : _logger(std::exchange(other._logger, nullptr)) {} + + Session &operator=(Session &&) = delete; + + ~Session() { + if (this->_logger) { + this->_logger->cancel_monitoring(); + } + } + + void finish() { + this->_logger->finish_monitoring(); + this->_logger = nullptr; + } + +private: + TileLogger *_logger; +}; + +inline TileLogger::Session TileLogger::start() { + return Session(*this); +} diff --git a/src/tile_downloader/TileUrlBuilder.h b/src/tile_downloader/TileUrlBuilder.h index 328cf110..e28f3002 100644 --- a/src/tile_downloader/TileUrlBuilder.h +++ b/src/tile_downloader/TileUrlBuilder.h @@ -1,39 +1,85 @@ #pragma once +#include +#include #include +#include +#include -#include #include -class TileUrlBuilder { -public: - virtual ~TileUrlBuilder() = default; - virtual std::string build_url(const radix::tile::Id &tile_id) const = 0; +enum class TileDownloadProvider { + Basemap, + Gataki }; -class BasemapTileUrlBuilder : public TileUrlBuilder { -public: - BasemapTileUrlBuilder(std::string layer, std::string style) - : _layer(std::move(layer)), _style(std::move(style)) {} - - std::string build_url(const radix::tile::Id &tile_id) const override { - const auto t = tile_id.to(radix::tile::Scheme::SlippyMap); - return fmt::format( - "https://mapsneu.wien.gv.at/basemap/{}/{}/google3857/{}/{}/{}.jpeg", - this->_layer, this->_style, t.zoom_level, t.coords.y, t.coords.x); - } +enum class TileYDirection { + Down, + Up +}; -private: - std::string _layer; - std::string _style; +struct TileProviderConfig { + std::string url_pattern; + TileYDirection y_direction = TileYDirection::Down; }; -class GatakiTileUrlBuilder : public TileUrlBuilder { +[[nodiscard]] inline TileProviderConfig tile_provider_config(TileDownloadProvider provider) +{ + switch (provider) { + case TileDownloadProvider::Basemap: + return { + "https://mapsneu.wien.gv.at/basemap/bmaporthofoto30cm/normal/google3857/{zoom}/{y}/{x}.jpeg", + TileYDirection::Down + }; + case TileDownloadProvider::Gataki: + return { + "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg", + TileYDirection::Down + }; + } + + throw std::invalid_argument("unknown tile download provider"); +} + +class TileUrlBuilder { public: - std::string build_url(const radix::tile::Id &tile_id) const override { - const auto t = tile_id.to(radix::tile::Scheme::SlippyMap); - return fmt::format( - "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{}/{}/{}.jpeg", - t.zoom_level, t.coords.y, t.coords.x); + explicit TileUrlBuilder(TileProviderConfig config) + : _url_pattern(std::move(config.url_pattern)) + , _y_direction(config.y_direction) + { + if (_url_pattern.find("{zoom}") == std::string::npos + || _url_pattern.find("{x}") == std::string::npos + || _url_pattern.find("{y}") == std::string::npos) { + throw std::invalid_argument("tile URL pattern must contain {zoom}, {x}, and {y}"); + } + } + + [[nodiscard]] std::string build_url(const radix::tile::Id& tile_id) const + { + auto y = tile_id.coords.y; + if (_y_direction == TileYDirection::Up) { + if (tile_id.zoom_level >= std::numeric_limits::digits) + throw std::invalid_argument("tile zoom level is too large for legacy TMS coordinates"); + y = (1u << tile_id.zoom_level) - y - 1; + } + + auto url = _url_pattern; + replace_all(url, "{zoom}", std::to_string(tile_id.zoom_level)); + replace_all(url, "{x}", std::to_string(tile_id.coords.x)); + replace_all(url, "{y}", std::to_string(y)); + return url; } + +private: + static void replace_all(std::string& value, std::string_view placeholder, const std::string& replacement) + { + size_t position = 0; + while ((position = value.find(placeholder, position)) != std::string::npos) { + value.replace(position, placeholder.size(), replacement); + position += replacement.size(); + } + } + + std::string _url_pattern; + TileYDirection _y_direction; }; diff --git a/src/tile_downloader/cli.cpp b/src/tile_downloader/cli.cpp index b83d4c36..bd1ca8af 100644 --- a/src/tile_downloader/cli.cpp +++ b/src/tile_downloader/cli.cpp @@ -13,29 +13,33 @@ Args parse(int argc, const char *const *argv) { Args args; - app.add_option("--provider", args.provider, "Tile provider (basemap or gataki)") - ->required() - ->check(CLI::IsMember({"basemap", "gataki"}, CLI::ignore_case)); + const std::map provider_map{ + {"basemap", TileDownloadProvider::Basemap}, + {"gataki", TileDownloadProvider::Gataki}}; + auto* source_group = app.add_option_group("Tile source"); + source_group->add_option("--provider", args.provider, "Configured tile provider (basemap or gataki)") + ->transform(CLI::CheckedTransformer(provider_map, CLI::ignore_case)); + auto* url_option = source_group->add_option("--url", args.url_pattern, "Custom tile URL pattern containing {zoom}, {x}, and {y}"); + source_group->require_option(1); app.add_option("--zoom", args.zoom, "Root tile zoom level")->required(); - app.add_option("--x,--row", args.x, "Root tile x coordinate")->required(); - app.add_option("--y,--col", args.y, "Root tile y coordinate")->required(); - - const std::map scheme_map{ - {"slippymap", radix::tile::Scheme::SlippyMap}, - {"google", radix::tile::Scheme::SlippyMap}, - {"xyz", radix::tile::Scheme::SlippyMap}, - {"tms", radix::tile::Scheme::Tms}}; - args.scheme = radix::tile::Scheme::SlippyMap; - app.add_option("--scheme", args.scheme, "Tile scheme") - ->default_val(radix::tile::Scheme::SlippyMap) - ->transform(CLI::CheckedTransformer(scheme_map, CLI::ignore_case)); + app.add_option("--x,--col", args.x, "Root tile x/column in Google/Mapbox coordinates")->required(); + app.add_option("--y,--row", args.y, "Root tile y/row in Google/Mapbox coordinates")->required(); + + const std::map y_direction_map{ + {"down", TileYDirection::Down}, + {"up", TileYDirection::Up}}; + args.url_y_direction = TileYDirection::Down; + app.add_option("--url-y-direction", args.url_y_direction, "Custom URL y direction: down (Google/Mapbox) or up (legacy TMS)") + ->default_str("down") + ->needs(url_option) + ->transform(CLI::CheckedTransformer(y_direction_map, CLI::ignore_case)); args.srs = 3857; app.add_option("--srs", args.srs, "Spatial reference system EPSG code")->default_val(3857); - args.output = "tiles/{zoom}/{y}/{x}.{ext}"; - app.add_option("--output", args.output, "Output path template")->default_val(args.output); + args.output = "tiles"; + app.add_option("--output", args.output, "Output directory; files use Google/Mapbox zoom/x/y.jpeg layout")->default_val(args.output.string()); const std::map log_level_names{ {"off", spdlog::level::off}, @@ -50,18 +54,8 @@ Args parse(int argc, const char *const *argv) { ->transform(CLI::CheckedTransformer(log_level_names, CLI::ignore_case)) ->default_val(spdlog::level::info); - args.early_skip = true; - app.add_option("--early-skip", args.early_skip, "Resume optimization: skip completed subtrees") - ->default_val(true); - app.add_option("--max-zoom-level", args.max_zoom_level, "Maximum zoom level to descend to"); - args.layer = "bmaporthofoto30cm"; - app.add_option("--layer", args.layer, "Basemap layer name")->default_val(args.layer); - - args.style = "normal"; - app.add_option("--style", args.style, "Basemap style")->default_val(args.style); - try { app.parse(argc, argv); } catch (const CLI::ParseError &e) { diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index 03edd30f..8e2380f8 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -4,24 +4,23 @@ #include #include -#include #include +#include "TileUrlBuilder.h" + namespace cli { struct Args { - std::string provider; + std::optional provider; + std::optional url_pattern; unsigned int zoom; unsigned int x; unsigned int y; - radix::tile::Scheme scheme; + TileYDirection url_y_direction; unsigned int srs; - std::string output; + std::filesystem::path output; spdlog::level::level_enum log_level; - bool early_skip; std::optional max_zoom_level; - std::string layer; - std::string style; }; Args parse(int argc, const char *const *argv); diff --git a/src/tile_downloader/main.cpp b/src/tile_downloader/main.cpp index 09c69d64..17c25314 100644 --- a/src/tile_downloader/main.cpp +++ b/src/tile_downloader/main.cpp @@ -1,7 +1,3 @@ -#include -#include -#include - #include "TileDownloader.h" #include "TileUrlBuilder.h" #include "cli.h" @@ -15,20 +11,13 @@ int main(int argc, char *argv[]) { LOG_ERROR_AND_EXIT("unsupported srs EPSG \"{}\"", args.srs); } - std::unique_ptr url_builder; - std::string provider = args.provider; - std::transform(provider.begin(), provider.end(), provider.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (provider == "basemap") { - url_builder = std::make_unique(args.layer, args.style); - } else { - url_builder = std::make_unique(); - } - - const radix::tile::Id root_id = {args.zoom, {args.x, args.y}, args.scheme}; + const auto provider_config = args.provider.has_value() + ? tile_provider_config(*args.provider) + : TileProviderConfig { *args.url_pattern, args.url_y_direction }; + const TileUrlBuilder url_builder(provider_config); - TileDownloader downloader(*url_builder, args.output, args.early_skip, args.max_zoom_level, root_id.zoom_level); - downloader.download_recursive(root_id); + const radix::tile::Id root_id = {args.zoom, {args.x, args.y}}; - return 0; + TileDownloader downloader(url_builder, args.output, args.max_zoom_level, root_id.zoom_level); + return downloader.download_recursive(root_id) ? 0 : 1; } diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h new file mode 100644 index 00000000..ab2d07f2 --- /dev/null +++ b/src/tile_downloader/write_file.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace tile_downloader_detail { + +inline void write_file_checked_direct(const std::filesystem::path &path, const std::vector &data) +{ + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error(fmt::format("failed to open \"{}\" for writing", path.string())); + } + + output.write(data.data(), static_cast(data.size())); + if (!output) { + throw std::runtime_error(fmt::format("failed to write \"{}\"", path.string())); + } + + output.close(); + if (!output) { + throw std::runtime_error(fmt::format("failed to finish writing \"{}\"", path.string())); + } +} + +} + +[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path &path) +{ + auto partial_path = path; + partial_path += ".part"; + return partial_path; +} + +[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path &path) +{ + auto pending_path = path; + pending_path += ".children-pending"; + return pending_path; +} + +inline void write_file_children_pending(const std::filesystem::path &path, const std::vector &data) +{ + const auto partial_path = partial_tile_path(path); + const auto pending_path = children_pending_tile_path(path); + + try { + tile_downloader_detail::write_file_checked_direct(partial_path, data); + std::filesystem::rename(partial_path, pending_path); + } catch (...) { + std::error_code cleanup_error; + std::filesystem::remove(partial_path, cleanup_error); + throw; + } +} + +inline void mark_tile_children_complete(const std::filesystem::path &path) +{ + const auto pending_path = children_pending_tile_path(path); + std::filesystem::rename(pending_path, path); +} diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 5b750e29..2c1b83ec 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(unittests_terrainlib terrainlib/cow.cpp terrainlib/dataset.cpp terrainlib/enumerate.cpp + terrainlib/envelope.cpp terrainlib/fixed_vector.cpp terrainlib/geometry_frames.cpp terrainlib/grid.cpp @@ -72,7 +73,7 @@ if(TARGET tilebuilderlib) tilebuilder/alpine_raster_format.cpp tilebuilder/dataset_reading.cpp tilebuilder/depth_first_tile_traverser.cpp - tilebuilder/image.cpp + tilebuilder/image_writer.cpp tilebuilder/parallel_tile_generator.cpp tilebuilder/parallel_tiler.cpp tilebuilder/tile_heights_generator.cpp @@ -92,6 +93,20 @@ if(TARGET sfbuilderlib) atb_configure_test(unittests_sfbuilder) endif() +if(TARGET tile-downloader) + find_package(CURL REQUIRED) + add_executable(unittests_tile_downloader + tile_downloader/downloader.cpp + tile_downloader/http_client.cpp + tile_downloader/logger.cpp + tile_downloader/url_builder.cpp + tile_downloader/write_file.cpp + ) + target_include_directories(unittests_tile_downloader PRIVATE ${CMAKE_SOURCE_DIR}/src/tile_downloader) + target_link_libraries(unittests_tile_downloader PRIVATE terrainlib Catch2::Catch2WithMain CURL::libcurl) + atb_configure_test(unittests_tile_downloader) +endif() + if(TARGET dagbuilderlib) add_executable(unittests_dagbuilder catch2_helpers.h diff --git a/unittests/sf_builder/texture.cpp b/unittests/sf_builder/texture.cpp index f8b4ffdf..3da052b4 100644 --- a/unittests/sf_builder/texture.cpp +++ b/unittests/sf_builder/texture.cpp @@ -34,7 +34,7 @@ TEST_CASE("estimate_zoom_level", "[terrainbuilder]") { const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id tile(20, glm::uvec2(0, 1), radix::tile::Scheme::SlippyMap); + const radix::tile::Id tile(20, glm::uvec2(0, 1)); const radix::tile::SrsBounds tile_bounds = grid.srsBounds(tile, false); const radix::tile::SrsBounds shifted_bounds(tile_bounds.min + glm::dvec2(-100, 420), tile_bounds.max + glm::dvec2(-100, 420)); @@ -67,11 +67,11 @@ class AlwaysEmptyTileProvider : public TileProvider { }; TEST_CASE("texture assembler takes root tile if only available ", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const std::set expected_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -88,18 +88,18 @@ TEST_CASE("texture assembler takes root tile if only available ", "[terrainbuild } TEST_CASE("texture assembler ignores parent if all children are present", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}, - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}, + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const std::set expected_tiles = { - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -116,15 +116,15 @@ TEST_CASE("texture assembler ignores parent if all children are present", "[terr } TEST_CASE("texture assembler considers max zoom level", "[terrainbuilder]") { - const radix::tile::Id root_tile(3, glm::uvec2(5, 4), radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(3, glm::uvec2(5, 4)); const std::set available_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}, - {4, {10, 8}, radix::tile::Scheme::SlippyMap}, - {4, {11, 8}, radix::tile::Scheme::SlippyMap}, - {4, {10, 9}, radix::tile::Scheme::SlippyMap}, - {4, {11, 9}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}, + {4, {10, 8}}, + {4, {11, 8}}, + {4, {10, 9}}, + {4, {11, 9}}}; const std::set expected_tiles = { - {3, {5, 4}, radix::tile::Scheme::SlippyMap}}; + {3, {5, 4}}}; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -143,29 +143,29 @@ TEST_CASE("texture assembler considers max zoom level", "[terrainbuilder]") { TEST_CASE("texture assembler works for arbitrary bounds", "[terrainbuilder]") { const std::set available_tiles = { - // {21, {1048576, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048577, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048578, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048579, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048580, 1048576}, radix::tile::Scheme::Tms}, - // {21, {1048581, 1048576}, radix::tile::Scheme::Tms}, - {20, {524288, 524288}, radix::tile::Scheme::Tms}, - {20, {524289, 524288}, radix::tile::Scheme::Tms}, - {20, {524290, 524288}, radix::tile::Scheme::Tms}, - // {19, {262144, 262144}, radix::tile::Scheme::Tms}, - {19, {262145, 262144}, radix::tile::Scheme::Tms}}; + // {21, {1048576, 1048575}}, + {21, {1048577, 1048575}}, + {21, {1048578, 1048575}}, + {21, {1048579, 1048575}}, + {21, {1048580, 1048575}}, + // {21, {1048581, 1048575}}, + {20, {524288, 524287}}, + {20, {524289, 524287}}, + {20, {524290, 524287}}, + // {19, {262144, 262143}}, + {19, {262145, 262143}}}; const std::set expected_tiles = { - // {21, {1048576, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048577, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048578, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048579, 1048576}, radix::tile::Scheme::Tms}, - {21, {1048580, 1048576}, radix::tile::Scheme::Tms}, - // {21, {1048581, 1048576}, radix::tile::Scheme::Tms}, - {20, {524288, 524288}, radix::tile::Scheme::Tms}, - // {20, {524289, 524288}, radix::tile::Scheme::Tms}, - {20, {524290, 524288}, radix::tile::Scheme::Tms}, - // {19, {262144, 262144}, radix::tile::Scheme::Tms}, - // {19, {262145, 262144}, radix::tile::Scheme::Tms} + // {21, {1048576, 1048575}}, + {21, {1048577, 1048575}}, + {21, {1048578, 1048575}}, + {21, {1048579, 1048575}}, + {21, {1048580, 1048575}}, + // {21, {1048581, 1048575}}, + {20, {524288, 524287}}, + // {20, {524289, 524287}}, + {20, {524290, 524287}}, + // {19, {262144, 262143}}, + // {19, {262145, 262143}} }; const AvailabilityListEmptyTileProvider tile_provider(available_tiles); @@ -208,7 +208,7 @@ TEST_CASE("texture assembler does not fail if there are not tiles", "[terrainbui TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(0, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, + {radix::tile::Id(0, {0, 0}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -217,7 +217,7 @@ TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -234,8 +234,8 @@ TEST_CASE("texture assembler assembles single tile", "[terrainbuilder]") { TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(1, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 255, 0))}, + {radix::tile::Id(1, {0, 0}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 0, 255))}, + {radix::tile::Id(1, {0, 1}), cv::Mat(1, 1, CV_8UC3, cv::Vec3b(0, 255, 0))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -244,7 +244,7 @@ TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -262,9 +262,9 @@ TEST_CASE("texture assembler assembles two tiles", "[terrainbuilder]") { TEST_CASE("texture assembler correct order of texture writes", "[terrainbuilder]") { const std::unordered_map tiles_to_texture = { - {radix::tile::Id(0, {0, 0}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC1, uint8_t(1))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::Tms), cv::Mat(1, 1, CV_8UC1, uint8_t(2))}, - {radix::tile::Id(1, {0, 1}, radix::tile::Scheme::SlippyMap), cv::Mat(1, 1, CV_8UC1, uint8_t(3))}, + {radix::tile::Id(0, {0, 0}), cv::Mat(1, 1, CV_8UC1, uint8_t(1))}, + {radix::tile::Id(1, {0, 0}), cv::Mat(1, 1, CV_8UC1, uint8_t(2))}, + {radix::tile::Id(1, {0, 1}), cv::Mat(1, 1, CV_8UC1, uint8_t(3))}, }; std::vector tiles_to_splatter; std::transform(tiles_to_texture.begin(), tiles_to_texture.end(), std::back_inserter(tiles_to_splatter), @@ -275,7 +275,7 @@ TEST_CASE("texture assembler correct order of texture writes", "[terrainbuilder] const StaticTileProvider tile_provider(tiles_to_texture); const ctb::Grid grid = ctb::GlobalMercator(); - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); cv::Mat assembled_texture = terrainbuilder::splatter_tiles_to_texture( root_tile, grid, @@ -351,7 +351,7 @@ TEST_CASE("texture assembler reports the content region", "[terrainbuilder]") { tile_image.row(row).setTo(uint8_t(row)); } - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); const StaticTileProvider tile_provider({{root_tile, tile_image}}); const std::vector tiles_to_splatter = {root_tile}; diff --git a/unittests/sf_merger/mask.cpp b/unittests/sf_merger/mask.cpp index 9006484a..4689fd80 100644 --- a/unittests/sf_merger/mask.cpp +++ b/unittests/sf_merger/mask.cpp @@ -1,3 +1,5 @@ +#include "../catch2_helpers.h" + #include #include @@ -17,6 +19,58 @@ #include "octree/Space.h" #include "utils.h" +TEST_CASE("Mask simplification uses a fixed tenth of a metre") { + OGRSpatialReference projected; + REQUIRE(projected.importFromEPSG(31287) == OGRERR_NONE); + + const std::optional projected_tolerance = mask::simplification_tolerance(projected); + REQUIRE(projected_tolerance); + CHECK(*projected_tolerance == Catch::Approx(0.1)); + + OGRSpatialReference geographic; + REQUIRE(geographic.SetWellKnownGeogCS("WGS84") == OGRERR_NONE); + + const std::optional geographic_tolerance = mask::simplification_tolerance(geographic); + REQUIRE(geographic_tolerance); + CHECK(*geographic_tolerance == Catch::Approx( + 0.1 / geographic.GetSemiMajor() / geographic.GetAngularUnits())); +} + +TEST_CASE("Mask simplification preserves polygon topology") { + OGRLinearRing outer; + outer.addPoint(0.0, 0.0); + outer.addPoint(1.0, 0.01); + outer.addPoint(2.0, 0.0); + outer.addPoint(2.0, 2.0); + outer.addPoint(0.0, 2.0); + outer.addPoint(0.0, 0.0); + + OGRLinearRing hole; + hole.addPoint(0.5, 0.5); + hole.addPoint(1.0, 0.51); + hole.addPoint(1.5, 0.5); + hole.addPoint(1.5, 1.5); + hole.addPoint(0.5, 1.5); + hole.addPoint(0.5, 0.5); + + OGRPolygon polygon; + REQUIRE(polygon.addRing(&outer) == OGRERR_NONE); + REQUIRE(polygon.addRing(&hole) == OGRERR_NONE); + REQUIRE(polygon.IsValid()); + + const uint64_t original_point_count = mask::point_count(polygon); + std::unique_ptr simplified = mask::simplify_geometry(polygon, 0.1); + + REQUIRE(simplified); + REQUIRE_FALSE(simplified->IsEmpty()); + REQUIRE(simplified->IsValid()); + CHECK(mask::point_count(*simplified) < original_point_count); + + const OGRPolygon *simplified_polygon = simplified->toPolygon(); + REQUIRE(simplified_polygon); + CHECK(simplified_polygon->getNumInteriorRings() == 1); +} + namespace { Polygon2 make_square(const glm::dvec2 min, const glm::dvec2 max) { diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp new file mode 100644 index 00000000..109f2b1b --- /dev/null +++ b/unittests/terrainlib/envelope.cpp @@ -0,0 +1,636 @@ +#include "../catch2_helpers.h" + +#include "io/envelope.h" + +#include +#include + +#include +#include +#include + +namespace { + +namespace v1 { + +struct Payload { + std::uint32_t id; + std::string name; + + bool operator==(const Payload &) const = default; +}; + +} // namespace v1 + +namespace v2 { + +struct Payload { + std::uint64_t id; + std::string name; + bool enabled; + + static Payload from_previous(v1::Payload previous) + { + return { + .id = previous.id, + .name = std::move(previous.name), + .enabled = true, + }; + } + + bool operator==(const Payload &) const = default; +}; + +} // namespace v2 + +namespace v3 { + +struct Payload { + std::uint64_t id; + std::string label; + bool enabled; + std::vector samples; + + static Payload from_previous(v2::Payload previous) + { + return { + .id = previous.id, + .label = std::move(previous.name), + .enabled = previous.enabled, + .samples = {}, + }; + } + + bool operator==(const Payload &) const = default; +}; + +} // namespace v3 + +using Schema = io::envelope::PayloadSchema< + "test.Payload", + io::envelope::Version<1, v1::Payload>, + io::envelope::Version<2, v2::Payload>, + io::envelope::Version<3, v3::Payload>>; + +static_assert(std::is_aggregate_v); +static_assert(std::is_aggregate_v); +static_assert(std::is_aggregate_v); +static_assert(Schema::class_name == "test.Payload"); +static_assert(Schema::latest_version == 3); +static_assert(std::same_as); +static_assert(std::same_as, v1::Payload>); + +template +io::envelope::Bytes encode_value(const Value &value) +{ + io::envelope::Bytes bytes; + zpp::bits::out output(bytes); + output(value).or_throw(); + return bytes; +} + +io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) +{ + return encode_value(envelope); +} + +io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) +{ + io::envelope::Envelope envelope{}; + zpp::bits::in input(bytes); + input(envelope).or_throw(); + return envelope; +} + +io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes &uncompressed_data) +{ + const std::unique_ptr context{ + ZSTD_createCCtx(), &ZSTD_freeCCtx}; + if (!context + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) + || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { + throw std::runtime_error{"could not configure zstd test context"}; + } + + io::envelope::Bytes compressed_data(ZSTD_compressBound(uncompressed_data.size())); + const std::size_t compressed_size = ZSTD_compress2( + context.get(), + compressed_data.data(), + compressed_data.size(), + uncompressed_data.data(), + uncompressed_data.size()); + if (ZSTD_isError(compressed_size)) { + throw std::runtime_error{"could not create zstd test data"}; + } + compressed_data.resize(compressed_size); + return compressed_data; +} + +template +void check_error(const Result &result, const io::envelope::ErrorCode expected) +{ + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == expected); +} + +io::envelope::Bytes bytes_from_string(const std::string_view text) +{ + io::envelope::Bytes bytes; + bytes.reserve(text.size()); + for (const char character : text) { + bytes.push_back(static_cast(static_cast(character))); + } + return bytes; +} + +} // namespace + +TEST_CASE("Envelope round trips the latest payload version") +{ + const v3::Payload expected{ + .id = 42, + .label = "latest", + .enabled = false, + .samples = {1, 2, 3}, + }; + + const auto bytes = io::envelope::serialize(expected); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.magic == io::envelope::magic); + CHECK(envelope.class_name == Schema::class_name); + CHECK(envelope.class_version == 3); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + CHECK(envelope.checksum.empty()); + CHECK(envelope.compression_algorithm + == io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum); + CHECK(envelope.uncompressed_size == encode_value(expected).size()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); +} + +TEST_CASE("Envelope upgrades older payload versions") +{ + SECTION("version 1 is upgraded through every subsequent version") + { + const v1::Payload original{.id = 7, .name = "version one"}; + const auto bytes = io::envelope::serialize(original); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 7, + .label = "version one", + .enabled = true, + .samples = {}, + }); + } + + SECTION("version 2 is upgraded to the latest version") + { + const v2::Payload original{.id = 9, .name = "version two", .enabled = false}; + const auto bytes = io::envelope::serialize(original); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 9, + .label = "version two", + .enabled = false, + .samples = {}, + }); + } +} + +TEST_CASE("Envelope supports uncompressed data without a checksum") +{ + const v3::Payload expected{ + .id = 11, + .label = "plain", + .enabled = true, + .samples = {5, 8}, + }; + const auto bytes = io::envelope::serialize( + expected, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::None); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::None); + CHECK(envelope.checksum.empty()); + CHECK(envelope.uncompressed_size == envelope.compressed_data.size()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); +} + +TEST_CASE("CRC-32C uses its canonical hexadecimal representation") +{ + SECTION("standard test vector") + { + const auto compressed = io::envelope::compress_with_checksum( + bytes_from_string("123456789"), + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + CHECK(compressed->checksum == "e3069283"); + } + + SECTION("empty input") + { + const auto compressed = io::envelope::compress_with_checksum( + {}, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + CHECK(compressed->checksum == "00000000"); + } +} + +TEST_CASE("CRC-32C protects independently compressed data") +{ + const auto original = bytes_from_string("payload protected by CRC-32C"); + + SECTION("without compression") + { + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto corrupted = compressed->compressed_data; + corrupted.front() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("with zstd compression") + { + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + REQUIRE(compressed->checksum.size() == 8); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::Crc32c, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto wrong_checksum = compressed->checksum; + wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::Crc32c, + wrong_checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("malformed checksum") + { + check_error( + io::envelope::checked_decompress( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + "E3069283"), + io::envelope::ErrorCode::ChecksumMismatch); + check_error( + io::envelope::checked_decompress( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + } +} + +TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") +{ + SECTION("latest version") + { + const v3::Payload expected{ + .id = 12, + .label = "crc", + .enabled = true, + .samples = {3, 5, 8}, + }; + const auto bytes = io::envelope::serialize( + expected, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(bytes.has_value()); + + const auto envelope = decode_envelope(*bytes); + CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::Crc32c); + CHECK(envelope.checksum.size() == 8); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::None); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == expected); + + auto corrupted = envelope; + corrupted.compressed_data.front() ^= std::byte{0x01}; + check_error(io::envelope::deserialize(encode_envelope(corrupted)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("older version") + { + const v1::Payload original{.id = 13, .name = "crc version one"}; + const auto bytes = io::envelope::serialize( + original, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(bytes.has_value()); + + const auto result = io::envelope::deserialize(*bytes); + REQUIRE(result.has_value()); + CHECK(*result == v3::Payload{ + .id = 13, + .label = "crc version one", + .enabled = true, + .samples = {}, + }); + } +} + +TEST_CASE("Checked compression round trips and validates its checksum") +{ + const io::envelope::Bytes original(4096, std::byte{0x2a}); + const auto compressed = io::envelope::compress_with_checksum( + original, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(compressed.has_value()); + CHECK(compressed->compressed_data.size() < original.size()); + CHECK(compressed->checksum.empty()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + const auto empty_compressed = io::envelope::compress_with_checksum( + {}, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(empty_compressed.has_value()); + const auto empty_result = io::envelope::checked_decompress( + empty_compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + empty_compressed->checksum); + REQUIRE(empty_result.has_value()); + CHECK(empty_result->empty()); + + auto corrupted = compressed->compressed_data; + corrupted.back() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); +} + +TEST_CASE("Checked decompression uses its maximum when the format omits the content size") +{ + const io::envelope::Bytes original(4096, std::byte{0x37}); + const auto compressed_data = compress_without_content_size(original); + + const auto result = io::envelope::checked_decompress( + compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size()); + REQUIRE(result.has_value()); + CHECK(*result == original); + + check_error( + io::envelope::checked_decompress( + compressed_data, + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); +} + +TEST_CASE("Envelope rejects incompatible metadata") +{ + const v3::Payload payload{.id = 1, .label = "metadata", .enabled = true, .samples = {}}; + const auto serialized = io::envelope::serialize(payload); + REQUIRE(serialized.has_value()); + + SECTION("magic") + { + auto envelope = decode_envelope(*serialized); + ++envelope.magic; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidMagic); + } + + SECTION("class name") + { + auto envelope = decode_envelope(*serialized); + envelope.class_name = "other.Payload"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::WrongClassName); + } + + SECTION("class version") + { + auto envelope = decode_envelope(*serialized); + envelope.class_version = 99; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedClassVersion); + } + + SECTION("checksum algorithm") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = static_cast(99); + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); + } + + SECTION("compression algorithm") + { + auto envelope = decode_envelope(*serialized); + envelope.compression_algorithm = static_cast(99); + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); + } + + SECTION("algorithm combination") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidAlgorithmCombination); + } + + SECTION("external checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum = "not used by zstd"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::InvalidAlgorithmCombination); + } + + SECTION("malformed CRC-32C checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; + envelope.checksum = "1234"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("incorrect CRC-32C checksum") + { + auto envelope = decode_envelope(*serialized); + envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; + envelope.checksum = "00000000"; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::ChecksumMismatch); + } + + SECTION("uncompressed size is smaller than the payload") + { + auto envelope = decode_envelope(*serialized); + --envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("uncompressed size is larger than the payload") + { + auto envelope = decode_envelope(*serialized); + ++envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("zero uncompressed size does not mean unspecified") + { + auto envelope = decode_envelope(*serialized); + envelope.uncompressed_size = 0; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("uncompressed size exceeds the hard limit") + { + auto envelope = decode_envelope(*serialized); + envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::SizeLimitExceeded); + } + + SECTION("uncompressed size exceeds a caller limit") + { + auto envelope = decode_envelope(*serialized); + check_error( + io::envelope::deserialize( + encode_envelope(envelope), + static_cast(envelope.uncompressed_size - 1)), + io::envelope::ErrorCode::SizeLimitExceeded); + } +} + +TEST_CASE("Envelope validates the size of uncompressed data") +{ + const v3::Payload payload{.id = 2, .label = "plain", .enabled = true, .samples = {1}}; + const auto serialized = io::envelope::serialize( + payload, + io::envelope::CompressionAlgorithm::None, + io::envelope::ChecksumAlgorithm::None); + REQUIRE(serialized.has_value()); + + SECTION("declared size is smaller") + { + auto envelope = decode_envelope(*serialized); + --envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } + + SECTION("declared size is larger") + { + auto envelope = decode_envelope(*serialized); + ++envelope.uncompressed_size; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DecompressionFailed); + } +} + +TEST_CASE("Envelope reports malformed serialized data") +{ + SECTION("envelope") + { + const io::envelope::Bytes malformed{std::byte{0x01}, std::byte{0x02}}; + check_error(io::envelope::deserialize(malformed), + io::envelope::ErrorCode::DeserializationFailed); + } + + SECTION("payload") + { + const io::envelope::Envelope envelope{ + .magic = io::envelope::magic, + .class_name = std::string{Schema::class_name}, + .class_version = 3, + .checksum_algorithm = io::envelope::ChecksumAlgorithm::None, + .checksum = {}, + .compression_algorithm = io::envelope::CompressionAlgorithm::None, + .uncompressed_size = 1, + .compressed_data = {std::byte{0x01}}, + }; + check_error(io::envelope::deserialize(encode_envelope(envelope)), + io::envelope::ErrorCode::DeserializationFailed); + } +} diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index bac6225b..fd886bac 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -44,14 +44,14 @@ TEST_CASE("transcode roundtrip") { mesh.texture = cv::Mat3b(100, 100); cv::randu(*mesh.texture, cv::Scalar(0, 0, 0), cv::Scalar(256, 256, 256)); - const tl::expected encode_result = + const std::expected encode_result = mesh::encode(mesh, mesh::EncodeOptions{.texture_format = ".png"}); if (!encode_result.has_value()) { FAIL(encode_result.error()); } const mesh::Encoded encoded = encode_result.value(); - const tl::expected decode_result = + const std::expected decode_result = mesh::decode(encoded, mesh::DecodeOptions{}); if (!decode_result.has_value()) { FAIL(decode_result.error()); @@ -90,10 +90,10 @@ TEST_CASE("io roundtrip") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -133,10 +133,10 @@ TEST_CASE("io roundtrip high precision") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -176,10 +176,10 @@ TEST_CASE("io roundtrip no texture") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } @@ -213,10 +213,10 @@ TEST_CASE("io roundtrip no texture and uvs") { std::filesystem::remove(mesh_path); CHECK(!std::filesystem::exists(mesh_path)); - mesh::io::save_to_path(mesh, mesh_path); + REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const tl::expected result = mesh::io::load_from_path(mesh_path); + const std::expected result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { FAIL(result.error().description()); } diff --git a/unittests/terrainlib/progress_indicator_test.cpp b/unittests/terrainlib/progress_indicator_test.cpp index b994ddb7..826c6e8d 100644 --- a/unittests/terrainlib/progress_indicator_test.cpp +++ b/unittests/terrainlib/progress_indicator_test.cpp @@ -19,7 +19,10 @@ #include "ProgressIndicator.h" #include +#include +#include #include +#include #include #include @@ -100,4 +103,28 @@ TEST_CASE("progress indicator") monitoring_thread.join(); CHECK_THROWS(pi.task_finished()); } + + SECTION("monitoring can be stopped before all tasks finish") + { + ProgressIndicator pi(1); + auto monitoring_thread = pi.start_monitoring(); + + std::condition_variable_any fallback_condition; + std::mutex fallback_mutex; + std::jthread fallback([&](std::stop_token stop_token) { + std::unique_lock lock(fallback_mutex); + fallback_condition.wait_for(lock, stop_token, 2s, []() { return false; }); + if (!stop_token.stop_requested()) { + pi.task_finished(); + } + }); + + const auto before_stop = std::chrono::steady_clock::now(); + monitoring_thread.request_stop(); + monitoring_thread.join(); + const auto stop_duration = std::chrono::steady_clock::now() - before_stop; + fallback.request_stop(); + + CHECK(stop_duration < 250ms); + } } diff --git a/unittests/tile_downloader/downloader.cpp b/unittests/tile_downloader/downloader.cpp new file mode 100644 index 00000000..083ecd49 --- /dev/null +++ b/unittests/tile_downloader/downloader.cpp @@ -0,0 +1,102 @@ +#include "TileDownloader.h" + +#include +#include +#include + +#include + +namespace { + +class TemporaryPyramid { +public: + explicit TemporaryPyramid(std::string_view name) + : _path(std::filesystem::temp_directory_path() / name) { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::create_directories(_path); + } + + ~TemporaryPyramid() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + [[nodiscard]] const std::filesystem::path &path() const { + return _path; + } + + [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id &tile) const { + return google_tile_path(_path, tile, ".jpeg"); + } + + void create_pending(const radix::tile::Id &tile) const { + const auto path = tile_path(tile); + std::filesystem::create_directories(path.parent_path()); + write_file_children_pending(path, std::vector{'t', 'i', 'l', 'e'}); + } + + void create_complete(const radix::tile::Id &tile) const { + create_pending(tile); + mark_tile_children_complete(tile_path(tile)); + } + +private: + std::filesystem::path _path; +}; + +const TileUrlBuilder missing_file_url({ + "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", + TileYDirection::Down +}); + +} + +TEST_CASE("tile downloader promotes parents after completed children") +{ + const TemporaryPyramid pyramid("atb-downloader-complete-pyramid"); + const radix::tile::Id root{0, {0, 0}}; + const auto children = root.children(); + + pyramid.create_pending(root); + for (const auto &child : children) { + pyramid.create_pending(child); + } + + TileDownloader downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + REQUIRE(downloader.download_recursive(root)); + + REQUIRE(std::filesystem::exists(pyramid.tile_path(root))); + CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); + + for (const auto &child : children) { + CHECK(std::filesystem::exists(pyramid.tile_path(child))); + CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(child)))); + } + + REQUIRE(std::filesystem::remove(pyramid.tile_path(children.front()))); + TileDownloader resumed_downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + CHECK(resumed_downloader.download_recursive(root)); + CHECK_FALSE(std::filesystem::exists(pyramid.tile_path(children.front()))); +} + +TEST_CASE("tile downloader leaves ancestors pending after a child failure") +{ + const TemporaryPyramid pyramid("atb-downloader-failed-pyramid"); + const radix::tile::Id root{0, {0, 0}}; + const auto children = root.children(); + + pyramid.create_pending(root); + for (size_t i = 1; i < children.size(); ++i) { + pyramid.create_complete(children[i]); + } + + TileDownloader downloader(missing_file_url, pyramid.path(), 1u, root.zoom_level); + CHECK_FALSE(downloader.download_recursive(root)); + + CHECK_FALSE(std::filesystem::exists(pyramid.tile_path(root))); + CHECK(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); + for (size_t i = 1; i < children.size(); ++i) { + CHECK(std::filesystem::exists(pyramid.tile_path(children[i]))); + } +} diff --git a/unittests/tile_downloader/http_client.cpp b/unittests/tile_downloader/http_client.cpp new file mode 100644 index 00000000..61311a8c --- /dev/null +++ b/unittests/tile_downloader/http_client.cpp @@ -0,0 +1,62 @@ +#include "HttpClient.h" + +#include +#include +#include +#include + +#include + +namespace { + +class TemporaryFile { +public: + TemporaryFile() + : _path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") { + std::ofstream output(_path, std::ios::binary); + output << "response body"; + REQUIRE(output); + } + + ~TemporaryFile() { + std::error_code error; + std::filesystem::remove(_path, error); + } + + [[nodiscard]] std::string url() const { + return "file://" + _path.string(); + } + +private: + std::filesystem::path _path; +}; + +class CallbackError : public std::runtime_error { +public: + CallbackError() + : std::runtime_error("callback failed") {} +}; + +} + +TEST_CASE("http client propagates response writer exceptions") +{ + const TemporaryFile source; + HttpClient client([](std::vector &, const char *, size_t) { + throw CallbackError(); + }); + + CHECK_THROWS_AS(client.get(source.url()), CallbackError); +} + +TEST_CASE("http client propagates progress callback exceptions") +{ + const TemporaryFile source; + HttpClient client; + + CHECK_THROWS_AS( + client.get(source.url(), [](double) { + throw CallbackError(); + }), + CallbackError); +} diff --git a/unittests/tile_downloader/logger.cpp b/unittests/tile_downloader/logger.cpp new file mode 100644 index 00000000..c41ecb4c --- /dev/null +++ b/unittests/tile_downloader/logger.cpp @@ -0,0 +1,33 @@ +#include "TileLogger.h" + +#include +#include + +#include + +using namespace std::literals; + +TEST_CASE("tile logger session stops monitoring during exceptional unwinding") +{ + TileLogger logger(0); + + const auto before_throw = std::chrono::steady_clock::now(); + CHECK_THROWS_AS( + [&]() { + auto session = logger.start(); + throw std::runtime_error("download failed"); + }(), + std::runtime_error); + const auto unwind_duration = std::chrono::steady_clock::now() - before_throw; + + CHECK(unwind_duration < 250ms); +} + +TEST_CASE("tile logger session can finish normally") +{ + TileLogger logger(0); + auto session = logger.start(); + + logger.skipped(radix::tile::Id{0, {0, 0}}); + session.finish(); +} diff --git a/unittests/tile_downloader/url_builder.cpp b/unittests/tile_downloader/url_builder.cpp new file mode 100644 index 00000000..e6bdaccf --- /dev/null +++ b/unittests/tile_downloader/url_builder.cpp @@ -0,0 +1,54 @@ +#include + +#include "TileUrlBuilder.h" +#include "tile_path.h" + +namespace { +constexpr radix::tile::Id tile { 3, { 1, 2 } }; +} + +TEST_CASE("configured tile provider URLs") +{ + { + const TileUrlBuilder builder(tile_provider_config(TileDownloadProvider::Basemap)); + CHECK(builder.build_url(tile) == "https://mapsneu.wien.gv.at/basemap/bmaporthofoto30cm/normal/google3857/3/2/1.jpeg"); + } + + { + const TileUrlBuilder builder(tile_provider_config(TileDownloadProvider::Gataki)); + CHECK(builder.build_url(tile) == "https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/3/2/1.jpeg"); + } +} + +TEST_CASE("custom tile URL patterns") +{ + SECTION("zoom/x/y with downward y") + { + const TileUrlBuilder builder({ "https://example.test/{zoom}/{x}/{y}.png", TileYDirection::Down }); + CHECK(builder.build_url(tile) == "https://example.test/3/1/2.png"); + } + + SECTION("zoom/y/x with downward y") + { + const TileUrlBuilder builder({ "https://example.test/{zoom}/{y}/{x}.png", TileYDirection::Down }); + CHECK(builder.build_url(tile) == "https://example.test/3/2/1.png"); + } + + SECTION("upward legacy TMS y") + { + const TileUrlBuilder builder({ "https://example.test/{zoom}/{x}/{y}.png", TileYDirection::Up }); + CHECK(builder.build_url(tile) == "https://example.test/3/1/5.png"); + } +} + +TEST_CASE("tile URL patterns require all coordinate placeholders") +{ + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{x}/{y}.png", TileYDirection::Down }), std::invalid_argument); + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{zoom}/{y}.png", TileYDirection::Down }), std::invalid_argument); + CHECK_THROWS_AS(TileUrlBuilder({ "https://example.test/{zoom}/{x}.png", TileYDirection::Down }), std::invalid_argument); +} + +TEST_CASE("downloaded tile path uses Google and Mapbox layout") +{ + CHECK(google_tile_path("tiles", tile, ".jpeg") == std::filesystem::path("tiles/3/1/2.jpeg")); +} diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp new file mode 100644 index 00000000..2773c4ed --- /dev/null +++ b/unittests/tile_downloader/write_file.cpp @@ -0,0 +1,94 @@ +#include "write_file.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +class TemporaryOutput { +public: + explicit TemporaryOutput(std::string_view name) + : _path(std::filesystem::temp_directory_path() / name) { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::remove(partial_path(), error); + std::filesystem::remove(pending_path(), error); + } + + ~TemporaryOutput() { + std::error_code error; + std::filesystem::remove_all(_path, error); + std::filesystem::remove(partial_path(), error); + std::filesystem::remove(pending_path(), error); + } + + [[nodiscard]] const std::filesystem::path &path() const { + return _path; + } + + [[nodiscard]] std::filesystem::path partial_path() const { + return partial_tile_path(_path); + } + + [[nodiscard]] std::filesystem::path pending_path() const { + return children_pending_tile_path(_path); + } + +private: + std::filesystem::path _path; +}; + +} + +TEST_CASE("checked file writer persists the complete response") +{ + const TemporaryOutput output("atb-write-file-success.bin"); + const std::vector expected{'t', 'i', 'l', 'e'}; + + write_file_children_pending(output.path(), expected); + + CHECK_FALSE(std::filesystem::exists(output.path())); + CHECK_FALSE(std::filesystem::exists(output.partial_path())); + REQUIRE(std::filesystem::exists(output.pending_path())); + + std::ifstream input(output.pending_path(), std::ios::binary); + const auto begin = std::istreambuf_iterator(input); + const std::vector actual(begin, std::istreambuf_iterator{}); + CHECK(actual == expected); + + mark_tile_children_complete(output.path()); + CHECK(std::filesystem::exists(output.path())); + CHECK_FALSE(std::filesystem::exists(output.pending_path())); +} + +TEST_CASE("checked file writer preserves the final path when promotion fails") +{ + const TemporaryOutput output("atb-write-file-promotion-failure"); + REQUIRE(std::filesystem::create_directory(output.path())); + write_file_children_pending(output.path(), std::vector{'t', 'i', 'l', 'e'}); + + CHECK_THROWS_AS( + mark_tile_children_complete(output.path()), + std::filesystem::filesystem_error); + + CHECK(std::filesystem::is_directory(output.path())); + CHECK(std::filesystem::exists(output.pending_path())); +} + +#if defined(__linux__) +TEST_CASE("checked file writer reports persistence failures") +{ + const std::vector data(64 * 1024, 'x'); + + CHECK_THROWS_AS(write_file_children_pending("/dev/full", data), std::runtime_error); + CHECK(std::filesystem::is_character_file("/dev/full")); + CHECK_FALSE(std::filesystem::exists("/dev/full.part")); + CHECK_FALSE(std::filesystem::exists("/dev/full.children-pending")); +} +#endif diff --git a/unittests/tilebuilder/alpine_raster_format.cpp b/unittests/tilebuilder/alpine_raster_format.cpp index 00ec9bf4..a0d98223 100644 --- a/unittests/tilebuilder/alpine_raster_format.cpp +++ b/unittests/tilebuilder/alpine_raster_format.cpp @@ -24,7 +24,7 @@ #include #include -#include "Image.h" +#include #include "alpine_raster.h" #include "ctb/Grid.hpp" @@ -41,15 +41,15 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("raste write") { - const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes); - generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + const auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes); + generator.write(radix::tile::Descriptor { {0, glm::uvec2(0, 0)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/0/0/0.png")); - generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/1/2/3.png")); // check that a second write doesn't crash - generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, HeightData(257, 257)); + generator.write(radix::tile::Descriptor { {1, glm::uvec2(2, 3)}, {}, int(ctb::Grid::Srs::SphericalMercator), 256, 257 }, radix::Raster({ 257, 257 })); CHECK(std::filesystem::exists("./unittest_tiles/1/2/3.png")); // in the best case, we would read back the data and check it. but that's too much work for now. @@ -58,7 +58,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal SECTION("process all tiles") { - auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, testTypeValue2Border(TestType::value)); + auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, testTypeValue2Border(TestType::value)); generator.setWarnOnMissingOverviews(false); generator.process({ 0, 7 }); const auto tiles = generator.tiler().generateTiles({ 0, 7 }); @@ -71,7 +71,7 @@ TEMPLATE_TEST_CASE("alpine raster format, border ", "", std::true_type, std::fal #if defined(ALP_UNITTESTS_EXTENDED) && ALP_UNITTESTS_EXTENDED SECTION("process all tiles with max zoom") { - auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, testTypeValue2Border(TestType::value)); + auto generator = alpine_raster::make_generator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", "./unittest_tiles/", ctb::Grid::Srs::SphericalMercator, testTypeValue2Border(TestType::value)); generator.setWarnOnMissingOverviews(false); generator.process({ 4, 8 }); const auto tiles = generator.tiler().generateTiles({ 4, 8 }); diff --git a/unittests/tilebuilder/dataset_reading.cpp b/unittests/tilebuilder/dataset_reading.cpp index 17c714e9..ba2804db 100644 --- a/unittests/tilebuilder/dataset_reading.cpp +++ b/unittests/tilebuilder/dataset_reading.cpp @@ -27,6 +27,7 @@ #include "Dataset.h" #include "DatasetReader.h" #include "ctb/types.hpp" +#include "image_writer.h" #include "srs.h" using namespace radix; @@ -184,7 +185,7 @@ TEST_CASE("reading") if (ALP_UNITTESTS_DEBUG_IMAGES) { image::debugOut(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(ref_heights.begin(), ref_heights.end(), heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); const auto path = fmt::format("./diffs_{}_{}.png", test_name, dataset_name.substr(s, l)); image::debugOut(height_diffs, path); @@ -194,7 +195,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(ref_heights.size()); + }) / double(ref_heights.buffer_length()); // fmt::print("{} | {}; mse: {}, largest_abs_diff: {}\n", test_name, dataset_name.substr(s, l), mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -248,7 +249,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./diff_low_res_high_res.png"); } @@ -257,7 +258,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("mse: {}, largest_abs_diff: {}\n", mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -290,7 +291,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); } @@ -299,7 +300,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("mse: {}, largest_abs_diff: {}\n", mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); @@ -332,7 +333,7 @@ TEST_CASE("reading") image::debugOut(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); image::debugOut(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); - auto height_diffs = HeightData(render_width, render_height); + auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); image::debugOut(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); } @@ -341,7 +342,7 @@ TEST_CASE("reading") const auto t = std::abs(double(a) - double(b)); largest_abs_diff = std::max(t, largest_abs_diff); return t * t; - }) / double(low_res_heights.size()); + }) / double(low_res_heights.buffer_length()); // fmt::print("render w/h: {}/{}, mse: {}, largest_abs_diff: {}\n", render_width, render_height, mse, largest_abs_diff); CHECK(largest_abs_diff < double(max_abs_diff)); CHECK(mse < max_mse); diff --git a/unittests/tilebuilder/depth_first_tile_traverser.cpp b/unittests/tilebuilder/depth_first_tile_traverser.cpp index ef753a3a..5cf7fb5e 100644 --- a/unittests/tilebuilder/depth_first_tile_traverser.cpp +++ b/unittests/tilebuilder/depth_first_tile_traverser.cpp @@ -37,8 +37,8 @@ TEST_CASE("depth_first_tile_traverser interface") const auto aggregate_function = [](std::vector) { return ReadType {}; }; const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); + const radix::tile::Id root_id = { 0, { 0, 0 } }; const unsigned max_zoom_level = 3; traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, max_zoom_level); @@ -68,15 +68,15 @@ TEST_CASE("depth_first_tile_traverser basics") }; const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); + const radix::tile::Id root_id = { 0, { 0, 0 } }; SECTION("reads root tile #1") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 0); REQUIRE(read_tiles.size() == 1); CHECK(aggregate_calls.size() == 0); - CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}})); CHECK(result.d == glm::uvec2 { 0, 0 }); } @@ -84,7 +84,7 @@ TEST_CASE("depth_first_tile_traverser basics") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 2, { 1, 3 } }, 2); REQUIRE(read_tiles.size() == 1); - CHECK(read_tiles.contains(radix::tile::Id{2, {1, 3}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{2, {1, 3}})); CHECK(result.d == glm::uvec2 { 1, 3 }); } @@ -92,10 +92,10 @@ TEST_CASE("depth_first_tile_traverser basics") { traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, { 0, { 0, 0 } }, 1); REQUIRE(read_tiles.size() == 4); - CHECK(read_tiles.contains(radix::tile::Id{1, {0, 0}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {0, 1}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {1, 0}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{1, {1, 1}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{1, {0, 0}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {0, 1}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {1, 0}})); + CHECK(read_tiles.contains(radix::tile::Id{1, {1, 1}})); } SECTION("aggregate is called correctly") @@ -119,7 +119,7 @@ TEST_CASE("depth_first_tile_traverser austrian heights") const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_100m_mgi.tif").value(); // const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, radix::tile::Scheme::Tms); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); const auto tile_reader = DatasetReader(dataset, grid.getSRS(), 1, false); // const auto dataset_reader = DatasetReader() std::set read_tiles; @@ -142,14 +142,14 @@ TEST_CASE("depth_first_tile_traverser austrian heights") return aggr; }; - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const radix::tile::Id root_id = { 0, { 0, 0 } }; SECTION("reads root tile") { const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 0); REQUIRE(read_tiles.size() == 1); CHECK(aggregate_calls.size() == 0); - CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{0, {0, 0}})); CHECK(result.first >= 0); CHECK(result.first <= 4000); CHECK(result.second >= 0); @@ -161,14 +161,14 @@ TEST_CASE("depth_first_tile_traverser austrian heights") const auto result = traverse_depth_first_and_aggregate(tiler, read_function, aggregate_function, root_id, 6); CHECK(read_tiles.size() == 6); CHECK(aggregate_calls.size() == 9); - CHECK(read_tiles.contains(radix::tile::Id{6, {33, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {33, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {33, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {33, 21}})); - CHECK(read_tiles.contains(radix::tile::Id{6, {34, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {34, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {34, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {34, 21}})); - CHECK(read_tiles.contains(radix::tile::Id{6, {35, 41}, tiler.scheme()})); - CHECK(read_tiles.contains(radix::tile::Id{6, {35, 42}, tiler.scheme()})); + CHECK(read_tiles.contains(radix::tile::Id{6, {35, 22}})); + CHECK(read_tiles.contains(radix::tile::Id{6, {35, 21}})); CHECK(result.first >= 0); CHECK(result.first <= 500); CHECK(result.second >= 2000); @@ -187,8 +187,8 @@ TEST_CASE("depth_first_tile_traverser aggregate is not called with an empty vect // parent tile is produced, because its border overlaps the extents // child tiles have smaller pixels -> their border does not overlap the extents any more. bounds.min.x = (bounds.width() / 256) / 4; - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::Yes, radix::tile::Scheme::Tms); - const radix::tile::Id root_id = { 0, { 0, 0 }, tiler.scheme() }; + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::Yes); + const radix::tile::Id root_id = { 0, { 0, 0 } }; const auto read_function = [&](const radix::tile::Descriptor&) -> int { return 0; diff --git a/unittests/tilebuilder/image.cpp b/unittests/tilebuilder/image.cpp deleted file mode 100644 index 448aef27..00000000 --- a/unittests/tilebuilder/image.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - * Alpine Terrain Builder - * Copyright (C) 2022 alpinemaps.org - * Copyright (C) 2022 Adam Celarek - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *****************************************************************************/ - -#include -#include -#include -#include "Image.h" - -using Catch::Approx; - -TEST_CASE("image") -{ - SECTION("iteration") - { - HeightData d { 40, 60 }; - REQUIRE(d.size() == 40 * 60); - REQUIRE(d.height() == 60); - REQUIRE(d.width() == 40); - int v = 0; - std::ranges::for_each(d, [&](auto& d) { d = float(v++); }); - - v = 0; - for (unsigned r = 0; r < d.height(); ++r) { - for (unsigned c = 0; c < d.width(); ++c) { - REQUIRE(d.pixel(r, c) == Approx(float(v++))); - } - } - } - - SECTION("conversion") - { - HeightData d { 40, 60 }; - int v_init = 0; - std::ranges::for_each(d, [&](auto& d) { d = float(v_init++); }); - - Image image = image::transformImage(d, [&](auto v) { const auto b = uchar(255.F * v / float(v_init)); return glm::u8vec3(b, b, b); }); - const auto max = float(v_init); - v_init = 0; - for (unsigned r = 0; r < d.height(); ++r) { - for (unsigned c = 0; c < d.width(); ++c) { - const auto t = uchar(float(v_init) * 255.F / max); - REQUIRE(image.pixel(r, c).x == t); - REQUIRE(image.pixel(r, c).y == t); - REQUIRE(image.pixel(r, c).z == t); - v_init++; - } - } -// image::saveImageAsPng(image, "/home/madam/Documents/work/tuw/alpinemaps/tmp/test.png"); - } -} diff --git a/unittests/tilebuilder/image_writer.cpp b/unittests/tilebuilder/image_writer.cpp new file mode 100644 index 00000000..aeddf2f2 --- /dev/null +++ b/unittests/tilebuilder/image_writer.cpp @@ -0,0 +1,86 @@ +/***************************************************************************** + * Alpine Terrain Builder + * Copyright (C) 2022 alpinemaps.org + * Copyright (C) 2022 Adam Celarek + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *****************************************************************************/ + +#include +#include +#include + +#include +#include +#include + +#include "image_writer.h" + +TEST_CASE("tile builder writes radix rasters as PNG images") +{ + const auto output_path = std::filesystem::temp_directory_path() / "alpine_terrain_builder_image_writer_test.png"; + std::filesystem::remove(output_path); + + radix::Raster raster({ 2, 2 }); + raster.pixel({ 0, 0 }) = { 255, 0, 0 }; + raster.pixel({ 1, 0 }) = { 0, 255, 0 }; + raster.pixel({ 0, 1 }) = { 0, 0, 255 }; + raster.pixel({ 1, 1 }) = { 255, 255, 255 }; + + image::saveImageAsPng(raster, output_path.string()); + + const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); + REQUIRE(decoded.rows == 2); + REQUIRE(decoded.cols == 2); + CHECK(decoded.at(0, 0) == cv::Vec3b(255, 0, 0)); + CHECK(decoded.at(0, 1) == cv::Vec3b(255, 255, 255)); + CHECK(decoded.at(1, 0) == cv::Vec3b(0, 0, 255)); + CHECK(decoded.at(1, 1) == cv::Vec3b(0, 255, 0)); + + std::filesystem::remove(output_path); +} + +TEST_CASE("tile builder handles image writer edge cases") +{ + SECTION("empty debug raster is rejected") + { + CHECK_THROWS_AS(image::debugOut(radix::Raster {}, "empty.png"), std::invalid_argument); + } + + SECTION("constant debug raster is written as black") + { + const auto output_path = std::filesystem::temp_directory_path() / "alpine_terrain_builder_constant_raster_test.png"; + std::filesystem::remove(output_path); + + radix::Raster raster({ 2, 2 }); + std::ranges::fill(raster, 42.F); + image::debugOut(raster, output_path.string()); + + const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); + REQUIRE(decoded.rows == 2); + REQUIRE(decoded.cols == 2); + CHECK(cv::countNonZero(decoded.reshape(1)) == 0); + + std::filesystem::remove(output_path); + } + + SECTION("PNG write failure is reported") + { + const auto missing_directory = std::filesystem::temp_directory_path() / "alpine_terrain_builder_missing_directory"; + std::filesystem::remove_all(missing_directory); + const auto output_path = missing_directory / "image.png"; + + CHECK_THROWS_AS(image::saveImageAsPng(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); + } +} diff --git a/unittests/tilebuilder/parallel_tile_generator.cpp b/unittests/tilebuilder/parallel_tile_generator.cpp index ee57262d..f9326334 100644 --- a/unittests/tilebuilder/parallel_tile_generator.cpp +++ b/unittests/tilebuilder/parallel_tile_generator.cpp @@ -43,7 +43,7 @@ TEST_CASE("parallel tile generator") , m_validation_error_counter(validation_error_counter) { } - void write(const std::string& file_path, const radix::tile::Descriptor& tile, const HeightData& heights) const override + void write(const std::string& file_path, const radix::tile::Descriptor& tile, const radix::Raster& heights) const override { if (file_path.empty() || tile.gridSize != 256 || heights.width() != 256 || heights.height() != 256) (*m_validation_error_counter)++; @@ -55,7 +55,7 @@ TEST_CASE("parallel tile generator") }; std::filesystem::path base_path = "./unittest_tiles/"; - auto generator = ParallelTileGenerator::make(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, std::make_unique(&tile_counter, &validation_error_counter), base_path); + auto generator = ParallelTileGenerator::make(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, std::make_unique(&tile_counter, &validation_error_counter), base_path); generator.setWarnOnMissingOverviews(false); SECTION("dataset tiles only") { @@ -63,12 +63,12 @@ TEST_CASE("parallel tile generator") CHECK(validation_error_counter == 0); CHECK(tile_counter == 27); CHECK(std::filesystem::exists(base_path / "0" / "0" / "0.empty")); - CHECK(std::filesystem::exists(base_path / "1" / "1" / "1.empty")); - CHECK(std::filesystem::exists(base_path / "2" / "2" / "2.empty")); - CHECK(std::filesystem::exists(base_path / "3" / "4" / "5.empty")); - CHECK(std::filesystem::exists(base_path / "4" / "8" / "10.empty")); - CHECK(std::filesystem::exists(base_path / "5" / "16" / "20.empty")); - CHECK(std::filesystem::exists(base_path / "7" / "70" / "84.empty")); + CHECK(std::filesystem::exists(base_path / "1" / "1" / "0.empty")); + CHECK(std::filesystem::exists(base_path / "2" / "2" / "1.empty")); + CHECK(std::filesystem::exists(base_path / "3" / "4" / "2.empty")); + CHECK(std::filesystem::exists(base_path / "4" / "8" / "5.empty")); + CHECK(std::filesystem::exists(base_path / "5" / "16" / "11.empty")); + CHECK(std::filesystem::exists(base_path / "7" / "70" / "43.empty")); } SECTION("world wide tiles") { diff --git a/unittests/tilebuilder/parallel_tiler.cpp b/unittests/tilebuilder/parallel_tiler.cpp index 815b48ba..b18124cf 100644 --- a/unittests/tilebuilder/parallel_tiler.cpp +++ b/unittests/tilebuilder/parallel_tiler.cpp @@ -23,23 +23,21 @@ #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" #include -#include #include #include #include #include -#include using Catch::Approx; using namespace radix; -TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::false_type) +TEST_CASE("ParallelTiler") { // const auto bounds = radix::tile::SrsBounds(1'000'000, 6'000'000, 2'000'000, 6'700'000); // in m SECTION("mercator / level 0") { const auto grid = ctb::GlobalMercator(); - const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No); CHECK(tiler.northEastTile(0).coords == glm::uvec2(0, 0)); CHECK(tiler.southWestTile(0).coords == glm::uvec2(0, 0)); @@ -59,26 +57,25 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f CHECK(t.srsBounds.max.x == Approx(grid.getExtent().max.x)); } - SECTION("mercator tms / level 1 and 2") + SECTION("mercator / level 1 and 2") { const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, TestType::value ? 1 : 0)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, TestType::value ? 1 : 0)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, 0)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, 0)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, TestType::value ? 2 : 1)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, TestType::value ? 2 : 1)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, 1)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, 1)); { // https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/-5.80/62.29 - // this code is for TMS mapping, i.e., tile y = 0 is south const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 1 : 0)); + CHECK(t.id.coords == glm::uvec2(1, 0)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -92,7 +89,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 2 : 1)); + CHECK(t.id.coords == glm::uvec2(2, 1)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -103,10 +100,10 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("geodetic tms / level 0") + SECTION("geodetic / level 0") { const auto grid = ctb::GlobalGeodetic(64); - const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::Yes); CHECK(tiler.northEastTile(0).coords == glm::uvec2(1, 0)); CHECK(tiler.southWestTile(0).coords == glm::uvec2(0, 0)); @@ -140,24 +137,24 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f CHECK(t1.srsBounds.max.x == Approx(grid.getExtent().max.x + grid.resolution(0))); } - SECTION("geodetic tms / level 1 and 2") + SECTION("geodetic / level 1 and 2") { const auto grid = ctb::GlobalGeodetic(64); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(2, TestType::value ? 1 : 0)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(2, TestType::value ? 1 : 0)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(2, 0)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(2, 0)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(4, TestType::value ? 3 : 0)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(4, TestType::value ? 3 : 0)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(4, 0)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(4, 0)); { const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 1 : 0)); + CHECK(t.id.coords == glm::uvec2(2, 0)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -173,7 +170,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(4, TestType::value ? 3 : 0)); + CHECK(t.id.coords == glm::uvec2(4, 0)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -186,26 +183,25 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("mercator tms / level 1 and 2 (test with cape horn, on southern and western hemisphere)") + SECTION("mercator / level 1 and 2 (test with cape horn, on southern and western hemisphere)") { const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/capehorn/small.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::No); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(0, TestType::value ? 0 : 1)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(0, TestType::value ? 0 : 1)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(0, 1)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(0, 1)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(1, TestType::value ? 1 : 2)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(1, TestType::value ? 1 : 2)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(1, 2)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(1, 2)); { // https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#1/-5.80/62.29 - // this code is for TMS mapping, i.e., tile y = 0 is south const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(0, TestType::value ? 0 : 1)); + CHECK(t.id.coords == glm::uvec2(0, 1)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -219,7 +215,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 1 : 2)); + CHECK(t.id.coords == glm::uvec2(1, 2)); CHECK(t.gridSize == 256); CHECK(t.tileSize == 256); @@ -230,24 +226,24 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f } } - SECTION("geodetic tms / level 1 and 2 (test with cape horn, on southern and western hemisphere)") + SECTION("geodetic / level 1 and 2 (test with cape horn, on southern and western hemisphere)") { const auto grid = ctb::GlobalGeodetic(64); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/capehorn/small.tif").value(); - const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes, TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap); + const auto tiler = ParallelTiler(grid, dataset->bounds(grid.getSRS()), radix::tile::Border::Yes); - CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, TestType::value ? 0 : 1)); - CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, TestType::value ? 0 : 1)); + CHECK(tiler.northEastTile(1).coords == glm::uvec2(1, 1)); + CHECK(tiler.southWestTile(1).coords == glm::uvec2(1, 1)); - CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, TestType::value ? 0 : 3)); - CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, TestType::value ? 0 : 3)); + CHECK(tiler.northEastTile(2).coords == glm::uvec2(2, 3)); + CHECK(tiler.southWestTile(2).coords == glm::uvec2(2, 3)); { const auto l1_tiles = tiler.generateTiles(1); REQUIRE(l1_tiles.size() == 1); const auto t = l1_tiles.front(); CHECK(t.id.zoom_level == 1); - CHECK(t.id.coords == glm::uvec2(1, TestType::value ? 0 : 1)); + CHECK(t.id.coords == glm::uvec2(1, 1)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -263,7 +259,7 @@ TEMPLATE_TEST_CASE("ParallelTiler, using tms scheme", "", std::true_type, std::f REQUIRE(l2_tiles.size() == 1); const auto t = l2_tiles.front(); CHECK(t.id.zoom_level == 2); - CHECK(t.id.coords == glm::uvec2(2, TestType::value ? 0 : 3)); + CHECK(t.id.coords == glm::uvec2(2, 3)); CHECK(t.gridSize == 64); CHECK(t.tileSize == 65); @@ -281,7 +277,7 @@ TEST_CASE("ParallelTiler returns tiles for several zoom levels") { const auto dataset = Dataset(ALP_TEST_DATA_DIR "/austria/at_mgi.tif"); const auto grid = ctb::GlobalMercator(256); - const auto tiler = ParallelTiler(grid, dataset.bounds(grid.getSRS()), radix::tile::Border::Yes, radix::tile::Scheme::Tms); + const auto tiler = ParallelTiler(grid, dataset.bounds(grid.getSRS()), radix::tile::Border::Yes); SECTION("generate from 0 to 7") { diff --git a/unittests/tilebuilder/tile_heights_generator.cpp b/unittests/tilebuilder/tile_heights_generator.cpp index 43f74237..89c532ae 100644 --- a/unittests/tilebuilder/tile_heights_generator.cpp +++ b/unittests/tilebuilder/tile_heights_generator.cpp @@ -30,7 +30,7 @@ TEST_CASE("TileHeightsGenerator") constexpr auto file_name = "height_data.atb"; SECTION("mercator") { - const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); + const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::SphericalMercator, radix::tile::Border::Yes, base_path / file_name); generator.run(8); const auto heights = TileHeights::read_from(base_path / file_name); @@ -41,7 +41,7 @@ TEST_CASE("TileHeightsGenerator") } { - auto [min, max] = heights.query({ 8, { 138, 166 } }); // part of styria, lower and upper austria (https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#8/15.69/47.75) + auto [min, max] = heights.query({ 8, { 138, 89 } }); // part of styria, lower and upper austria (https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#8/15.69/47.75) CHECK(min > 300); CHECK(min < 400); CHECK(max > 1500); @@ -50,7 +50,7 @@ TEST_CASE("TileHeightsGenerator") } SECTION("geodetic") { - const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::WGS84, radix::tile::Scheme::Tms, radix::tile::Border::Yes, base_path / file_name); + const auto generator = TileHeightsGenerator(ALP_TEST_DATA_DIR "/austria/at_mgi.tif", ctb::Grid::Srs::WGS84, radix::tile::Border::Yes, base_path / file_name); generator.run(8); const auto heights = TileHeights::read_from(base_path / file_name); @@ -61,7 +61,7 @@ TEST_CASE("TileHeightsGenerator") } { - auto [min, max] = heights.query({ 8, { 270, 194 } }); // can't check the address easily, because there is no web service showing geodetic tile names. + auto [min, max] = heights.query({ 8, { 270, 61 } }); // can't check the address easily, because there is no web service showing geodetic tile names. CHECK(min > 500); CHECK(min < 700); CHECK(max > 3600); diff --git a/unittests/tilebuilder/top_down_tiler.cpp b/unittests/tilebuilder/top_down_tiler.cpp index cd1e33b5..61645361 100644 --- a/unittests/tilebuilder/top_down_tiler.cpp +++ b/unittests/tilebuilder/top_down_tiler.cpp @@ -21,7 +21,7 @@ #include "TopDownTiler.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" -#include +#include #include using namespace radix; @@ -49,18 +49,16 @@ void compare_tile_lists(const std::vector& a_tiles, std } -TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::false_type) +TEST_CASE("TopDownTiler") { - const auto scheme = TestType::value ? radix::tile::Scheme::Tms : radix::tile::Scheme::SlippyMap; - SECTION("mercator / level 0 all") { const auto grid = ctb::GlobalMercator(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 0, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 0, 0 }}); REQUIRE(tiles.size() == 4); - const auto parallel_tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, grid.getExtent(), radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } @@ -69,22 +67,22 @@ TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::f const auto grid = ctb::GlobalMercator(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 0, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 0, 0 }}); REQUIRE(tiles.size() == 1); - const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } SECTION("geodetic / level 0 east half") { const auto grid = ctb::GlobalGeodetic(); - const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, grid.getExtent(), radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 1, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 1, 0 }}); REQUIRE(tiles.size() == 4); - const auto parallel_tiler = ParallelTiler(grid, {{0, -90}, {180, 90}}, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, {{0, -90}, {180, 90}}, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } @@ -93,11 +91,11 @@ TEMPLATE_TEST_CASE("BottomUpTiler, using tms scheme", "", std::true_type, std::f const auto grid = ctb::GlobalGeodetic(); auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR "/austria/at_mgi.tif").value(); const auto bounds = dataset->bounds(grid.getSRS()); - const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto tiler = TopDownTiler(grid, bounds, radix::tile::Border::No); - const auto tiles = tiler.generateTiles({0, { 1, 0 }, scheme}); + const auto tiles = tiler.generateTiles({0, { 1, 0 }}); REQUIRE(tiles.size() == 1); - const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No, scheme); + const auto parallel_tiler = ParallelTiler(grid, bounds, radix::tile::Border::No); compare_tile_lists(tiles, parallel_tiler.generateTiles(1)); } }