From b6fe5eab314dbdecb6b3bd61371807f7c968f3a4 Mon Sep 17 00:00:00 2001 From: Ethan O'Connor Date: Thu, 16 Jul 2026 11:22:50 -0700 Subject: [PATCH 01/39] render: support affine raster source crops --- src/render/qpainter_renderer.cpp | 8 +++- src/render/render_ir.cpp | 27 ++++++++++- src/render/render_ir.h | 7 ++- src/render/template_layer_planner.cpp | 14 +----- src/render/vello/src/lib.rs | 35 +++++++++----- src/render/vello_renderer.cpp | 3 +- test/frame_pipeline_t.cpp | 4 +- test/template_layer_planner_t.cpp | 46 ++++++++++++++---- test/vello_renderer_t.cpp | 67 ++++++++++++++++++++++++++- test/vello_renderer_t.h | 1 + 10 files changed, 170 insertions(+), 42 deletions(-) diff --git a/src/render/qpainter_renderer.cpp b/src/render/qpainter_renderer.cpp index a01618e9f..b520aef16 100644 --- a/src/render/qpainter_renderer.cpp +++ b/src/render/qpainter_renderer.cpp @@ -261,10 +261,16 @@ void QPainterRenderer::render(QPainter& painter, const RenderIR& ir, } QImage image(bytes.data(), int(op.image->width), int(op.image->height), qsizetype(op.image->bytes_per_row), QImage::Format_RGBA8888); + auto const source = toQRectF(op.source).intersected( + QRectF(0, 0, image.width(), image.height()) + ); + if (!source.isValid() || source.isEmpty()) + return; painter.save(); painter.setRenderHint(QPainter::SmoothPixmapTransform, true); painter.setOpacity(painter.opacity() * op.opacity); - painter.drawImage(toQRectF(op.target), image); + painter.setWorldTransform(toQTransform(op.image_to_scene), true); + painter.drawImage(source, image, source); painter.restore(); } else if constexpr (std::is_same_v) diff --git a/src/render/render_ir.cpp b/src/render/render_ir.cpp index cbb8df334..147792075 100644 --- a/src/render/render_ir.cpp +++ b/src/render/render_ir.cpp @@ -172,7 +172,32 @@ void RenderIRBuilder::strokeEllipse(Rect bounds, Color color, StrokeStyle style, void RenderIRBuilder::drawImage(std::shared_ptr image, Rect target, double opacity) { - ir_->commands.emplace_back(DrawImage { std::move(image), target, opacity }); + auto const source = image + ? Rect { 0, 0, double(image->width), double(image->height) } + : Rect {}; + drawImage(std::move(image), source, target, opacity); +} + +void RenderIRBuilder::drawImage(std::shared_ptr image, Rect source, + Rect target, double opacity) +{ + Transform image_to_scene; + if (source.isValid() && target.isValid()) + { + image_to_scene.m11 = target.width / source.width; + image_to_scene.m22 = target.height / source.height; + image_to_scene.dx = target.x - source.x * image_to_scene.m11; + image_to_scene.dy = target.y - source.y * image_to_scene.m22; + } + drawImage(std::move(image), source, image_to_scene, opacity); +} + +void RenderIRBuilder::drawImage(std::shared_ptr image, Rect source, + Transform image_to_scene, double opacity) +{ + ir_->commands.emplace_back(DrawImage { + std::move(image), source, image_to_scene, opacity + }); } void RenderIRBuilder::drawLinePattern(PathPtr outline, Color color, double angle, diff --git a/src/render/render_ir.h b/src/render/render_ir.h index 10ddfe5a4..a1b942393 100644 --- a/src/render/render_ir.h +++ b/src/render/render_ir.h @@ -225,7 +225,8 @@ struct StrokeEllipse struct DrawImage { std::shared_ptr image; - Rect target; + Rect source; + Transform image_to_scene; double opacity = 1; }; @@ -284,6 +285,10 @@ class RenderIRBuilder QualityHint quality = QualityHint::Default); void drawImage(std::shared_ptr image, Rect target, double opacity = 1); + void drawImage(std::shared_ptr image, Rect source, + Rect target, double opacity = 1); + void drawImage(std::shared_ptr image, Rect source, + Transform image_to_scene, double opacity = 1); void drawLinePattern(PathPtr outline, Color color, double angle, double spacing, double offset, double line_width); void append(const RenderIR& scene); diff --git a/src/render/template_layer_planner.cpp b/src/render/template_layer_planner.cpp index 25cd01795..7e1e17090 100644 --- a/src/render/template_layer_planner.cpp +++ b/src/render/template_layer_planner.cpp @@ -215,18 +215,6 @@ RasterMosaic transparentMosaic(const std::vector& tiles) }; } -Rect fullImageTarget(const TileKey& tile) -{ - auto const scale_x = tile.target.width / tile.source.width; - auto const scale_y = tile.target.height / tile.source.height; - return { - tile.target.x - tile.source.x * scale_x, - tile.target.y - tile.source.y * scale_y, - tile.image.width * scale_x, - tile.image.height * scale_y, - }; -} - } // namespace class TemplateLayerPlanner::Impl @@ -502,7 +490,7 @@ class TemplateLayerPlanner::Impl RenderIRBuilder builder(next_revision_++); builder.pushTransform(key.template_to_map); for (auto const& [tile, image] : tiles) - builder.drawImage(image, fullImageTarget(tile)); + builder.drawImage(image, tile.source, tile.target); builder.popTransform(); auto scene = builder.finish(); layers_[&source] = { std::move(key), scene }; diff --git a/src/render/vello/src/lib.rs b/src/render/vello/src/lib.rs index ae8341c2a..4f9bda2da 100644 --- a/src/render/vello/src/lib.rs +++ b/src/render/vello/src/lib.rs @@ -149,7 +149,8 @@ mod ffi { fn scene_draw_image( scene: &mut SceneBuilder, image: &RetainedImage, - target: Rect, + source: Rect, + image_to_scene: Transform, opacity: f64, ) -> bool; fn scene_draw_line_pattern( @@ -528,11 +529,15 @@ fn new_retained_image( fn scene_draw_image( scene: &mut SceneBuilder, retained: &RetainedImage, - target: ffi::Rect, + source: ffi::Rect, + image_to_scene: ffi::Transform, opacity: f64, ) -> bool { scene.command_count += 1; - let Some(target) = rect(target) else { + let Some(source) = rect(source) else { + return false; + }; + let Some(image_to_scene) = finite_transform(image_to_scene) else { return false; }; let Some(image) = retained.image.clone() else { @@ -543,6 +548,15 @@ fn scene_draw_image( } let width = image.width; let height = image.height; + let source = Rect::new( + source.x0.max(0.0), + source.y0.max(0.0), + source.x1.min(f64::from(width)), + source.y1.min(f64::from(height)), + ); + if source.width() <= 0.0 || source.height() <= 0.0 { + return false; + } let brush = ImageBrush { image, sampler: ImageSampler { @@ -552,14 +566,13 @@ fn scene_draw_image( alpha: opacity.clamp(0.0, 1.0) as f32, }, }; - let image_to_target = Affine::translate((target.x0, target.y0)) - * Affine::scale_non_uniform( - target.width() / f64::from(width), - target.height() / f64::from(height), - ); - scene - .scene - .draw_image(&brush, scene.transform() * image_to_target); + scene.scene.fill( + Fill::NonZero, + scene.transform() * image_to_scene, + &brush, + None, + &source, + ); true } diff --git a/src/render/vello_renderer.cpp b/src/render/vello_renderer.cpp index 61884d65e..4279eb06c 100644 --- a/src/render/vello_renderer.cpp +++ b/src/render/vello_renderer.cpp @@ -287,7 +287,8 @@ class VelloRenderer::Impl throw std::logic_error("Vello received invalid immutable image data"); auto const image = retainImage(op.image); auto const accepted = ffi::scene_draw_image( - *builder, *image->image, ffiRect(op.target), op.opacity + *builder, *image->image, ffiRect(op.source), + ffiTransform(op.image_to_scene), op.opacity ); if (!accepted) throw std::logic_error("Vello rejected immutable image data"); diff --git a/test/frame_pipeline_t.cpp b/test/frame_pipeline_t.cpp index e9b819bba..c23f88572 100644 --- a/test/frame_pipeline_t.cpp +++ b/test/frame_pipeline_t.cpp @@ -458,8 +458,8 @@ void FramePipelineTest::overlayPatternsAndImagesStayRetained() if (auto const* image = std::get_if(&command)) { first_image = image->image; - QCOMPARE(image->target.width, 12.0); - QCOMPARE(image->target.height, 8.0); + QCOMPARE(image->source.width * image->image_to_scene.m11, 12.0); + QCOMPARE(image->source.height * image->image_to_scene.m22, 8.0); } } QCOMPARE(pattern_count, 2); diff --git a/test/template_layer_planner_t.cpp b/test/template_layer_planner_t.cpp index 35f6bfacc..9e8f2e6bb 100644 --- a/test/template_layer_planner_t.cpp +++ b/test/template_layer_planner_t.cpp @@ -138,10 +138,14 @@ std::size_t imageCommandCount(const render::VectorPass& pass) }); } -QImage renderReference(const render::FramePacket& frame) +QImage renderReference(const render::FramePacket& frame, + QColor background = Qt::white) { - QImage image(16, 16, QImage::Format_ARGB32_Premultiplied); - image.fill(Qt::white); + auto const width = qCeil(frame.view.width * frame.view.device_pixel_ratio); + auto const height = qCeil(frame.view.height * frame.view.device_pixel_ratio); + QImage image(width, height, QImage::Format_ARGB32_Premultiplied); + image.setDevicePixelRatio(frame.view.device_pixel_ratio); + image.fill(background); QPainter painter(&image); auto const completion = render::QPainterFrameRenderer().render(painter, frame); Q_ASSERT(completion.status == render::FrameStatus::Presented); @@ -157,10 +161,12 @@ QColor velloPixel(const render::VelloImage& image, QPoint point) return view.pixelColor(point); } -QImage renderVello(const render::FramePacketPtr& frame) +QImage renderVello( + const render::FramePacketPtr& frame, + render::Color background = { 65535, 65535, 65535, 65535 }) { render::VelloRenderer renderer; - auto const rendered = renderer.renderOffscreen(frame); + auto const rendered = renderer.renderOffscreen(frame, background); Q_ASSERT_X(rendered, Q_FUNC_INFO, renderer.lastError().c_str()); QImage view( rendered->rgba8.data(), int(rendered->width), int(rendered->height), @@ -388,6 +394,9 @@ void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() auto whole_plan = whole_planner.plan(whole_map, whole_view, { -10, -7, 20, 14 }, 3.2); QVERIFY(tiled_plan.complete); QVERIFY(whole_plan.complete); + QCOMPARE(tiled_plan.newly_resident_images, std::size_t(1)); + QCOMPARE(tiled_plan.below_map.size(), std::size_t(1)); + QCOMPARE(imageCommandCount(tiled_plan.below_map.front()), std::size_t(1)); auto const tiled_snapshot = tiled_map.publishRenderSnapshot(); auto const whole_snapshot = whole_map.publishRenderSnapshot(); @@ -395,12 +404,17 @@ void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() QVERIFY(whole_snapshot); render::FramePlanner tiled_frame_planner; render::FramePlanner whole_frame_planner; - auto const tiled = renderVello(scaledFrameFor( + auto const tiled_frame = scaledFrameFor( *tiled_snapshot, tiled_frame_planner, std::move(tiled_plan) - )); - auto const whole = renderVello(scaledFrameFor( + ); + auto const whole_frame = scaledFrameFor( *whole_snapshot, whole_frame_planner, std::move(whole_plan) - )); + ); + auto const transparent = render::Color { 0, 0, 0, 0 }; + auto const tiled = renderVello(tiled_frame, transparent); + auto const whole = renderVello(whole_frame, transparent); + auto const tiled_reference = renderReference(*tiled_frame, Qt::transparent); + auto const whole_reference = renderReference(*whole_frame, Qt::transparent); // The source boundary projects to x=32.25. Compare a narrow band across // it against one monolithic image, including fractional transform coverage. @@ -410,14 +424,26 @@ void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() { auto const actual = tiled.pixelColor(x, y); auto const expected = whole.pixelColor(x, y); + auto const reference_actual = tiled_reference.pixelColor(x, y); + auto const reference_expected = whole_reference.pixelColor(x, y); QVERIFY2( std::abs(actual.red() - expected.red()) <= 2 && std::abs(actual.green() - expected.green()) <= 2 - && std::abs(actual.blue() - expected.blue()) <= 2, + && std::abs(actual.blue() - expected.blue()) <= 2 + && std::abs(actual.alpha() - expected.alpha()) <= 2, qPrintable(QStringLiteral("tile seam at %1,%2: %3 vs %4") .arg(x).arg(y).arg(actual.name(QColor::HexArgb), expected.name(QColor::HexArgb))) ); + QVERIFY2( + std::abs(reference_actual.red() - reference_expected.red()) <= 2 + && std::abs(reference_actual.green() - reference_expected.green()) <= 2 + && std::abs(reference_actual.blue() - reference_expected.blue()) <= 2 + && std::abs(reference_actual.alpha() - reference_expected.alpha()) <= 2, + qPrintable(QStringLiteral("reference tile seam at %1,%2: %3 vs %4") + .arg(x).arg(y).arg(reference_actual.name(QColor::HexArgb), + reference_expected.name(QColor::HexArgb))) + ); } } } diff --git a/test/vello_renderer_t.cpp b/test/vello_renderer_t.cpp index 292777bc0..269d0cbaa 100644 --- a/test/vello_renderer_t.cpp +++ b/test/vello_renderer_t.cpp @@ -125,13 +125,14 @@ render::FramePacketPtr mapFrame(Map& map, QSize viewport, }); } -QImage referenceImage(const render::FramePacket& frame) +QImage referenceImage(const render::FramePacket& frame, + QColor background = Qt::white) { auto const width = qCeil(frame.view.width * frame.view.device_pixel_ratio); auto const height = qCeil(frame.view.height * frame.view.device_pixel_ratio); QImage image(width, height, QImage::Format_ARGB32_Premultiplied); image.setDevicePixelRatio(frame.view.device_pixel_ratio); - image.fill(Qt::white); + image.fill(background); QPainter painter(&image); painter.setRenderHint(QPainter::Antialiasing, true); auto const completion = render::QPainterFrameRenderer().render(painter, frame); @@ -295,6 +296,68 @@ void VelloRendererTest::offscreenGpuMatchesReference() QVERIFY(difference.high_delta_pixels < actual.width() * actual.height() / 50); } +void VelloRendererTest::affineImageSourceCropMatchesReference() +{ + constexpr auto width = std::uint32_t(6); + constexpr auto height = std::uint32_t(6); + auto pixels = std::make_shared>( + std::size_t(width * height * 4), std::uint8_t(0) + ); + for (auto y = std::uint32_t(0); y < height; ++y) + { + for (auto x = std::uint32_t(0); x < width; ++x) + { + auto const offset = std::size_t((y * width + x) * 4); + (*pixels)[offset + 0] = 255; + (*pixels)[offset + 1] = 0; + (*pixels)[offset + 2] = 255; + (*pixels)[offset + 3] = 255; + if (x >= 1 && x < 5 && y >= 1 && y < 5) + { + (*pixels)[offset + 0] = std::uint8_t(20 + x * 15); + (*pixels)[offset + 1] = std::uint8_t(160 + y * 12); + (*pixels)[offset + 2] = 35; + (*pixels)[offset + 3] = std::uint8_t(80 + x * 20); + } + } + } + auto image = std::make_shared(render::ImageData { + width, height, width * 4, std::move(pixels) + }); + + render::RenderIRBuilder builder(44, { 0, 0, 64, 64 }); + builder.drawImage( + std::move(image), + { 1, 1, 4, 4 }, + { 4, 1, -1, 4, 20, 10 } + ); + auto frame = std::make_shared(); + frame->id = 9; + frame->revision = 44; + frame->view = { 64, 64, 1, {} }; + frame->vector_passes.push_back({ builder.finish() }); + + render::VelloRenderer renderer; + auto const transparent = render::Color { 0, 0, 0, 0 }; + auto const rendered = renderer.renderOffscreen(frame, transparent); + QVERIFY2(rendered, renderer.lastError().c_str()); + auto const actual = imageFromVello(*rendered); + auto const expected = referenceImage(*frame, Qt::transparent); + + QCOMPARE(actual.pixelColor(8, 8), QColor(Qt::transparent)); + QCOMPARE(expected.pixelColor(8, 8), QColor(Qt::transparent)); + QVERIFY(actual.pixelColor(29, 25).green() > 100); + QVERIFY(actual.pixelColor(29, 25).alpha() > 50); + + auto const difference = compareImages(actual, expected); + qInfo() << "Affine cropped image Vello/QPainter mean channel delta" + << difference.mean_channel_delta + << "high-delta pixels" << difference.high_delta_pixels; + QVERIFY(difference.mean_channel_delta < 3.2); + QVERIFY(difference.high_delta_pixels + < 2 * (actual.width() + actual.height())); +} + void VelloRendererTest::miterLimitOneMatchesReference() { render::RenderIRBuilder builder(43, { 0, 0, 64, 64 }); diff --git a/test/vello_renderer_t.h b/test/vello_renderer_t.h index f8c10e1f7..7ca35dd27 100644 --- a/test/vello_renderer_t.h +++ b/test/vello_renderer_t.h @@ -18,6 +18,7 @@ private slots: void typedEncoderRetainsImmutableScenes(); void missingNativeTargetIsRetriable(); void offscreenGpuMatchesReference(); + void affineImageSourceCropMatchesReference(); void miterLimitOneMatchesReference(); void selectionHandleGlyphsSurviveHighDpi(); void mapCorpusMatchesReference(); From f07bfbfb3b2eb410fdfb9c31e76c69c4a980520e Mon Sep 17 00:00:00 2001 From: Ethan O'Connor Date: Thu, 16 Jul 2026 11:38:42 -0700 Subject: [PATCH 02/39] raster: share bounded asynchronous resources --- src/CMakeLists.txt | 1 + src/gdal/gdal_template.cpp | 152 ++++--- src/gdal/gdal_template.h | 38 +- src/templates/raster_resource_manager.cpp | 495 ++++++++++++++++++++++ src/templates/raster_resource_manager.h | 148 +++++++ test/CMakeLists.txt | 1 + test/gdal_tiled_t.cpp | 19 +- test/raster_resource_manager_t.cpp | 432 +++++++++++++++++++ 8 files changed, 1219 insertions(+), 67 deletions(-) create mode 100644 src/templates/raster_resource_manager.cpp create mode 100644 src/templates/raster_resource_manager.h create mode 100644 test/raster_resource_manager_t.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c86d4347f..eca48efd0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -249,6 +249,7 @@ set(Mapper_Common_SRCS templates/paint_on_template_feature.cpp templates/paint_on_template_tool.cpp + templates/raster_resource_manager.cpp templates/template.cpp templates/template_adjust.cpp templates/template_dialog_reopen.cpp diff --git a/src/gdal/gdal_template.cpp b/src/gdal/gdal_template.cpp index 181e143ee..84817bbc9 100644 --- a/src/gdal/gdal_template.cpp +++ b/src/gdal/gdal_template.cpp @@ -32,13 +32,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include @@ -144,16 +144,14 @@ QRect decodedSourceRectForTileImpl(const QSize& raster_size, struct TileReadCancellation { - const std::atomic* current_generation = nullptr; - std::uint64_t expected_generation = 0; + const RasterResourceManager::CancellationToken* token = nullptr; }; int continueCurrentTileRead(double, const char*, void* data) { auto const* cancellation = static_cast(data); - return !cancellation || !cancellation->current_generation - || cancellation->current_generation->load(std::memory_order_relaxed) - == cancellation->expected_generation; + return !cancellation || !cancellation->token + || !cancellation->token->isCancelled(); } } // namespace @@ -271,11 +269,11 @@ bool GdalTemplate::loadTemplateFileImpl() GdalManager(); CPLErrorReset(); auto const path = template_path.toUtf8(); - tiled_dataset.reset(GDALDataset::Open( + tiled_dataset = std::shared_ptr(GDALDataset::Open( path.constData(), GDAL_OF_RASTER | GDAL_OF_READONLY | GDAL_OF_THREAD_SAFE | GDAL_OF_VERBOSE_ERROR, nullptr, nullptr, nullptr - )); + ), GDALDatasetUniquePtrDeleter {}); if (!tiled_dataset) { setErrorString(tr("Failed to open tiled raster: %1") @@ -327,7 +325,10 @@ bool GdalTemplate::loadTemplateFileImpl() } } - tile_pool.setMaxThreadCount(workerCountForSource()); + raster_owner.setConcurrencyLimit( + RasterResourceManager::Lane::BlockingIo, + workerCountForSource() + ); return true; } @@ -509,12 +510,11 @@ void GdalTemplate::collectRasterTiles(const QRectF& map_clip_rect, void GdalTemplate::shutdownTiledSource() { - tile_generation.fetch_add(1, std::memory_order_relaxed); - tile_pool.clear(); - tile_pool.waitForDone(); + raster_owner.invalidate(); tiled_dataset.reset(); queued_tiles.clear(); + failed_tiles.clear(); tile_cache.clear(); tiled_raster_info = GdalImageReader::RasterInfo(); @@ -547,9 +547,9 @@ void GdalTemplate::queueWantedTiles(const TileWindow& window, bool replace_pendi if (replace_pending_tiles) { - tile_generation.fetch_add(1, std::memory_order_relaxed); - tile_pool.clear(); + raster_owner.invalidate(); queued_tiles.clear(); + failed_tiles.clear(); } if (window.isEmpty()) @@ -562,6 +562,9 @@ void GdalTemplate::queueWantedTiles(const TileWindow& window, bool replace_pendi { return; } + auto const failed = failed_tiles.constFind(key); + if (failed != failed_tiles.cend() && !failed->retry.hasExpired()) + return; auto const source = sourceRectForTile( tiled_raster_size, tiled_raster_info.block_size, key.tile_x, key.tile_y, key.subsampling @@ -631,7 +634,6 @@ void GdalTemplate::queueWantedTiles(const TileWindow& window, bool replace_pendi if (missing_tiles.empty()) return; - auto const generation = tile_generation.load(std::memory_order_relaxed); auto available_slots = std::max( 0, max_queued_screen_tiles - queued_tiles.size() ); @@ -642,36 +644,65 @@ void GdalTemplate::queueWantedTiles(const TileWindow& window, bool replace_pendi auto const key = missing.key; if (tile_cache.contains(key) || queued_tiles.contains(key)) continue; - queued_tiles.insert(key, generation); - tile_pool.start([this, key, generation] { - auto tile = readTileImage( - key.tile_x, key.tile_y, key.subsampling, generation - ); - QMetaObject::invokeMethod( - this, - [this, key, tile = std::move(tile), generation]() mutable { - if (tile.isNull()) - onTileLoadFailed(key, generation); - else - onTileLoaded(key, std::move(tile), generation); - }, - Qt::QueuedConnection - ); - }, missing.fallback ? 2 : 1); + auto const accepted = RasterResourceManager::instance().submit( + raster_owner, + RasterResourceManager::Lane::BlockingIo, + missing.fallback + ? RasterResourceManager::Priority::Coverage + : RasterResourceManager::Priority::Visible, + this, + [ + dataset = tiled_dataset, + raster_info = tiled_raster_info, + raster_size = tiled_raster_size, + key, + receiver = this + ](const RasterResourceManager::CancellationToken& cancellation) mutable { + auto tile = readTileImage( + dataset, raster_info, raster_size, + key.tile_x, key.tile_y, key.subsampling, &cancellation + ); + return RasterResourceManager::Completion { + [receiver, key, tile = std::move(tile)]() mutable { + if (tile.isNull()) + receiver->onTileLoadFailed(key); + else + receiver->onTileLoaded(key, std::move(tile)); + } + }; + } + ); + if (!accepted) + continue; + queued_tiles.insert(key); --available_slots; } } QImage GdalTemplate::readTileImage( - int tile_x, int tile_y, int subsampling, - std::optional generation) const + int tile_x, int tile_y, int subsampling) const +{ + return readTileImage( + tiled_dataset, tiled_raster_info, tiled_raster_size, + tile_x, tile_y, subsampling, nullptr + ); +} + +QImage GdalTemplate::readTileImage( + const std::shared_ptr& dataset, + const GdalImageReader::RasterInfo& raster_info, + const QSize& raster_size, + int tile_x, + int tile_y, + int subsampling, + const RasterResourceManager::CancellationToken* cancellation) { - if (!tiled_dataset) + if (!dataset) return {}; auto const src = decodedSourceRectForTile( - tiled_raster_size, tiled_raster_info.block_size, tile_x, tile_y, subsampling + raster_size, raster_info.block_size, tile_x, tile_y, subsampling ); if (src.isEmpty()) return {}; @@ -680,7 +711,7 @@ QImage GdalTemplate::readTileImage( auto const output_w = std::max(1, (src.width() + safe_subsampling - 1) / safe_subsampling); auto const output_h = std::max(1, (src.height() + safe_subsampling - 1) / safe_subsampling); - QImage tile(output_w, output_h, tiled_raster_info.image_format); + QImage tile(output_w, output_h, raster_info.image_format); if (tile.isNull()) return {}; @@ -690,40 +721,40 @@ QImage GdalTemplate::readTileImage( INIT_RASTERIO_EXTRA_ARG(extra_arg); if (safe_subsampling > 1) extra_arg.eResampleAlg = GRIORA_Average; - TileReadCancellation cancellation { &tile_generation, generation.value_or(0) }; - if (generation) + TileReadCancellation read_cancellation { cancellation }; + if (cancellation) { extra_arg.pfnProgress = continueCurrentTileRead; - extra_arg.pProgressData = &cancellation; + extra_arg.pProgressData = &read_cancellation; } CPLErrorReset(); - auto bands = tiled_raster_info.bands; - auto result = tiled_dataset->RasterIO( + auto bands = raster_info.bands; + auto result = dataset->RasterIO( GF_Read, src.x(), src.y(), src.width(), src.height(), - tile.bits() + tiled_raster_info.band_offset, output_w, output_h, + tile.bits() + raster_info.band_offset, output_w, output_h, GDT_Byte, bands.count(), bands.data(), - tiled_raster_info.pixel_space, tile.bytesPerLine(), - tiled_raster_info.band_space, + raster_info.pixel_space, tile.bytesPerLine(), + raster_info.band_space, &extra_arg); if (result >= CE_Warning) return {}; - tiled_raster_info.postprocessing(tile); + raster_info.postprocessing(tile); return tile; } void GdalTemplate::onTileLoaded( - const GdalTileKey& key, QImage tile_image, std::uint64_t generation) + const GdalTileKey& key, QImage tile_image) { - if (queued_tiles.value(key) == generation) - queued_tiles.remove(key); - if (generation != tile_generation.load(std::memory_order_relaxed) || !isTiledSource()) + queued_tiles.remove(key); + if (!isTiledSource()) return; + failed_tiles.remove(key); auto const cost = tileCacheCost(tile_image); tile_cache.insert(key, new QImage(std::move(tile_image)), cost); @@ -733,10 +764,27 @@ void GdalTemplate::onTileLoaded( void GdalTemplate::onTileLoadFailed( - const GdalTileKey& key, std::uint64_t generation) + const GdalTileKey& key) { - if (queued_tiles.value(key) == generation) - queued_tiles.remove(key); + queued_tiles.remove(key); + if (!isTiledSource()) + return; + + auto& failure = failed_tiles[key]; + failure.attempts = std::min(failure.attempts + 1, 8); + auto const delay = std::min(30'000, 250 << (failure.attempts - 1)); + failure.retry = QDeadlineTimer(delay); + + queueWantedTiles(wanted_window, false); + auto const expected_generation = raster_owner.generation(); + QTimer::singleShot(delay, this, [this, key, expected_generation] { + if (!isTiledSource() || raster_owner.generation() != expected_generation) + return; + auto const failed = failed_tiles.constFind(key); + if (failed == failed_tiles.cend() || !failed->retry.hasExpired()) + return; + queueWantedTiles(wanted_window, false); + }); } diff --git a/src/gdal/gdal_template.h b/src/gdal/gdal_template.h index ebacdef1e..c4b34b687 100644 --- a/src/gdal/gdal_template.h +++ b/src/gdal/gdal_template.h @@ -20,22 +20,23 @@ #ifndef OPENORIENTEERING_GDAL_TEMPLATE_H #define OPENORIENTEERING_GDAL_TEMPLATE_H -#include #include -#include +#include #include #include +#include #include #include #include +#include #include #include -#include #include #include "gdal/gdal_image_reader.h" +#include "templates/raster_resource_manager.h" #include "templates/template.h" #include "templates/template_image.h" @@ -154,10 +155,18 @@ class GdalTemplate : public TemplateImage void shutdownTiledSource(); void queueWantedTiles(const TileWindow& window, bool replace_pending_tiles); - QImage readTileImage(int tile_x, int tile_y, int subsampling, - std::optional generation = std::nullopt) const; - void onTileLoaded(const GdalTileKey& key, QImage tile_image, std::uint64_t generation); - void onTileLoadFailed(const GdalTileKey& key, std::uint64_t generation); + QImage readTileImage(int tile_x, int tile_y, int subsampling) const; + static QImage readTileImage( + const std::shared_ptr& dataset, + const GdalImageReader::RasterInfo& raster_info, + const QSize& raster_size, + int tile_x, + int tile_y, + int subsampling, + const RasterResourceManager::CancellationToken* cancellation + ); + void onTileLoaded(const GdalTileKey& key, QImage tile_image); + void onTileLoadFailed(const GdalTileKey& key); void markTileAreaDirty(int tile_x, int tile_y, int subsampling); TileWindow tileWindowForMapRect(const QRectF& map_rect, int subsampling) const; const QImage* findBestCachedTile(int tile_x, int tile_y, int subsampling, QRectF* source_rect) const; @@ -181,16 +190,23 @@ class GdalTemplate : public TemplateImage int chooseTiledSubsampling(double scale) const; int workerCountForSource() const; - GDALDatasetUniquePtr tiled_dataset; + struct TileFailure + { + int attempts = 0; + QDeadlineTimer retry; + }; + + std::shared_ptr tiled_dataset; GdalImageReader::RasterInfo tiled_raster_info; QSize tiled_raster_size; TileWindow wanted_window; QPoint tiled_origin_tile; bool has_tiled_origin_tile = false; - std::atomic tile_generation{0}; - QThreadPool tile_pool; - QHash queued_tiles; + RasterResourceManager::Owner raster_owner = + RasterResourceManager::instance().createOwner(); + QSet queued_tiles; + QHash failed_tiles; QCache tile_cache; }; diff --git a/src/templates/raster_resource_manager.cpp b/src/templates/raster_resource_manager.cpp new file mode 100644 index 000000000..032fd4cec --- /dev/null +++ b/src/templates/raster_resource_manager.cpp @@ -0,0 +1,495 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +#include "templates/raster_resource_manager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace OpenOrienteering { + +struct RasterResourceManager::CancellationToken::OwnerState +{ + std::uint64_t id = 0; + std::atomic generation { 1 }; + std::atomic_bool retired { false }; + std::array concurrency_limit { 1, 1 }; + std::array active {}; +}; + +struct RasterResourceManager::Owner::SharedState + : std::enable_shared_from_this +{ + struct Job + { + std::shared_ptr owner; + std::uint64_t generation = 0; + Lane lane = Lane::BlockingIo; + Priority priority = Priority::Background; + QPointer receiver; + Work work; + std::uint64_t sequence = 0; + }; + + struct LaneState + { + explicit LaneState(int thread_limit) + { + pool.setMaxThreadCount(std::max(1, thread_limit)); + pool.setExpiryTimeout(30'000); + } + + QThreadPool pool; + std::vector> pending; + std::uint64_t last_owner = 0; + int active = 0; + }; + + SharedState(RasterResourceManager* context, Limits limits) + : context(context) + , blocking(limits.blocking_io_threads) + , decode(limits.decode_threads) + , max_pending_per_owner(std::max( + 1, limits.max_pending_per_owner + )) + , max_pending_per_lane(std::max( + max_pending_per_owner, limits.max_pending_per_lane + )) + {} + + LaneState& laneState(Lane lane) + { + return lane == Lane::BlockingIo ? blocking : decode; + } + + const LaneState& laneState(Lane lane) const + { + return lane == Lane::BlockingIo ? blocking : decode; + } + + static std::size_t laneIndex(Lane lane) + { + return static_cast(lane); + } + + bool jobIsCurrent(const Job& job) const noexcept + { + return !job.owner->retired.load(std::memory_order_relaxed) + && job.owner->generation.load(std::memory_order_relaxed) + == job.generation; + } + + void purgeInvalidLocked(LaneState& lane) + { + std::erase_if(lane.pending, [this](auto const& job) { + return !jobIsCurrent(*job); + }); + } + + std::vector>::iterator chooseNextLocked( + Lane lane_id, + LaneState& lane) + { + purgeInvalidLocked(lane); + auto const lane_index = laneIndex(lane_id); + std::vector owners; + for (auto const& job : lane.pending) + { + if (job->owner->active[lane_index] + < job->owner->concurrency_limit[lane_index]) + { + owners.push_back(job->owner->id); + } + } + if (owners.empty()) + return lane.pending.end(); + + std::ranges::sort(owners); + owners.erase(std::unique(owners.begin(), owners.end()), owners.end()); + auto owner = std::ranges::upper_bound(owners, lane.last_owner); + if (owner == owners.end()) + owner = owners.begin(); + + auto selected = lane.pending.end(); + for (auto it = lane.pending.begin(); it != lane.pending.end(); ++it) + { + if ((*it)->owner->id != *owner + || (*it)->owner->active[lane_index] + >= (*it)->owner->concurrency_limit[lane_index]) + { + continue; + } + if (selected == lane.pending.end() + || (*it)->priority < (*selected)->priority + || ((*it)->priority == (*selected)->priority + && (*it)->sequence < (*selected)->sequence)) + { + selected = it; + } + } + if (selected != lane.pending.end()) + lane.last_owner = *owner; + return selected; + } + + void dispatchLocked(Lane lane_id) + { + auto& lane = laneState(lane_id); + while (!shutting_down + && lane.active < lane.pool.maxThreadCount()) + { + auto selected = chooseNextLocked(lane_id, lane); + if (selected == lane.pending.end()) + break; + + auto job = std::move(*selected); + lane.pending.erase(selected); + ++lane.active; + ++job->owner->active[laneIndex(lane_id)]; + auto self = shared_from_this(); + lane.pool.start([self = std::move(self), job = std::move(job)] { + Completion completion; + try + { + completion = job->work(CancellationToken { + job->owner, job->generation + }); + } + catch (const std::exception& error) + { + qWarning() << "Raster worker failed:" << error.what(); + } + catch (...) + { + qWarning() << "Raster worker failed with an unknown exception"; + } + self->finish(std::move(job), std::move(completion)); + }); + } + } + + void finish(std::shared_ptr job, Completion completion) + { + bool deliver = false; + { + std::lock_guard lock(mutex); + auto& lane = laneState(job->lane); + --lane.active; + --job->owner->active[laneIndex(job->lane)]; + deliver = !shutting_down && completion && jobIsCurrent(*job); + dispatchLocked(Lane::BlockingIo); + dispatchLocked(Lane::Decode); + } + + if (!deliver) + return; + + auto self = shared_from_this(); + QMetaObject::invokeMethod( + context, + [self = std::move(self), job = std::move(job), + completion = std::move(completion)]() mutable { + { + std::lock_guard lock(self->mutex); + if (self->shutting_down || !self->jobIsCurrent(*job) + || !job->receiver) + { + return; + } + } + completion(); + }, + Qt::QueuedConnection + ); + } + + void invalidate(const std::shared_ptr& owner, bool retire) + { + if (!owner) + return; + if (retire) + owner->retired.store(true, std::memory_order_relaxed); + owner->generation.fetch_add(1, std::memory_order_relaxed); + + std::lock_guard lock(mutex); + if (retire) + owners.erase(owner->id); + purgeInvalidLocked(blocking); + purgeInvalidLocked(decode); + dispatchLocked(Lane::BlockingIo); + dispatchLocked(Lane::Decode); + } + + void setConcurrencyLimit( + const std::shared_ptr& owner, + Lane lane, + int limit) + { + if (!owner) + return; + std::lock_guard lock(mutex); + owner->concurrency_limit[laneIndex(lane)] = std::max(1, limit); + dispatchLocked(lane); + } + + int concurrencyLimit( + const std::shared_ptr& owner, + Lane lane) const + { + if (!owner) + return 0; + std::lock_guard lock(mutex); + return owner->concurrency_limit[laneIndex(lane)]; + } + + RasterResourceManager* context = nullptr; + mutable std::mutex mutex; + LaneState blocking; + LaneState decode; + std::unordered_map> owners; + std::uint64_t next_owner_id = 1; + std::uint64_t next_sequence = 1; + std::size_t max_pending_per_owner = 128; + std::size_t max_pending_per_lane = 2048; + bool shutting_down = false; +}; + +RasterResourceManager::CancellationToken::CancellationToken( + std::weak_ptr owner, + std::uint64_t generation) noexcept + : owner_(std::move(owner)) + , generation_(generation) +{} + +bool RasterResourceManager::CancellationToken::isCancelled() const noexcept +{ + auto const owner = owner_.lock(); + return !owner + || owner->retired.load(std::memory_order_relaxed) + || owner->generation.load(std::memory_order_relaxed) != generation_; +} + +RasterResourceManager::Owner::Owner( + std::weak_ptr manager, + std::shared_ptr state) noexcept + : manager_(std::move(manager)) + , state_(std::move(state)) +{} + +RasterResourceManager::Owner::Owner(Owner&& other) noexcept + : manager_(std::move(other.manager_)) + , state_(std::move(other.state_)) +{} + +RasterResourceManager::Owner& RasterResourceManager::Owner::operator=( + Owner&& other) noexcept +{ + if (this != &other) + { + retire(); + manager_ = std::move(other.manager_); + state_ = std::move(other.state_); + } + return *this; +} + +RasterResourceManager::Owner::~Owner() +{ + retire(); +} + +bool RasterResourceManager::Owner::isValid() const noexcept +{ + return state_ && !state_->retired.load(std::memory_order_relaxed); +} + +void RasterResourceManager::Owner::invalidate() +{ + if (auto manager = manager_.lock()) + manager->invalidate(state_, false); +} + +void RasterResourceManager::Owner::setConcurrencyLimit(Lane lane, int limit) +{ + if (auto manager = manager_.lock()) + manager->setConcurrencyLimit(state_, lane, limit); +} + +int RasterResourceManager::Owner::concurrencyLimit(Lane lane) const +{ + if (auto manager = manager_.lock()) + return manager->concurrencyLimit(state_, lane); + return 0; +} + +std::uint64_t RasterResourceManager::Owner::generation() const noexcept +{ + return state_ ? state_->generation.load(std::memory_order_relaxed) : 0; +} + +void RasterResourceManager::Owner::retire() +{ + if (!state_) + return; + if (auto manager = manager_.lock()) + manager->invalidate(state_, true); + else + state_->retired.store(true, std::memory_order_relaxed); + state_.reset(); + manager_.reset(); +} + +RasterResourceManager::Limits RasterResourceManager::defaultLimits() +{ + auto const ideal = std::max(1, QThread::idealThreadCount()); +#ifdef Q_OS_ANDROID + return { std::min(2, ideal), std::min(2, ideal) }; +#else + return { + std::clamp(ideal / 2, 1, 4), + std::clamp(ideal - 1, 1, 4), + }; +#endif +} + +RasterResourceManager& RasterResourceManager::instance() +{ + static RasterResourceManager manager; + return manager; +} + +RasterResourceManager::RasterResourceManager(Limits limits, QObject* parent) + : QObject(parent) +{ + if (limits.blocking_io_threads <= 0 || limits.decode_threads <= 0) + limits = defaultLimits(); + state_ = std::make_shared(this, limits); +} + +RasterResourceManager::~RasterResourceManager() +{ + auto state = std::move(state_); + if (!state) + return; + { + std::lock_guard lock(state->mutex); + state->shutting_down = true; + for (auto const& [id, weak_owner] : state->owners) + { + Q_UNUSED(id) + if (auto owner = weak_owner.lock()) + { + owner->retired.store(true, std::memory_order_relaxed); + owner->generation.fetch_add(1, std::memory_order_relaxed); + } + } + state->blocking.pending.clear(); + state->decode.pending.clear(); + } + state->blocking.pool.clear(); + state->decode.pool.clear(); + state->blocking.pool.waitForDone(); + state->decode.pool.waitForDone(); +} + +RasterResourceManager::Owner RasterResourceManager::createOwner( + int concurrency_limit) +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto owner = std::make_shared(); + { + std::lock_guard lock(state_->mutex); + if (state_->next_owner_id == std::numeric_limits::max()) + qFatal("Raster resource owner id space exhausted"); + owner->id = state_->next_owner_id++; + owner->concurrency_limit.fill(std::max(1, concurrency_limit)); + state_->owners[owner->id] = owner; + } + return Owner { state_, std::move(owner) }; +} + +bool RasterResourceManager::submit( + const Owner& owner, + Lane lane, + Priority priority, + QObject* receiver, + Work work) +{ + Q_ASSERT(QThread::currentThread() == thread()); + Q_ASSERT(!receiver || receiver->thread() == thread()); + if (owner.manager_.lock() != state_ + || !owner.isValid() || !receiver || !work) + return false; + + auto job = std::make_shared(); + job->owner = owner.state_; + job->generation = owner.generation(); + job->lane = lane; + job->priority = priority; + job->receiver = receiver; + job->work = std::move(work); + { + std::lock_guard lock(state_->mutex); + if (state_->shutting_down || !state_->jobIsCurrent(*job)) + return false; + auto& lane_state = state_->laneState(lane); + auto const owner_pending = std::ranges::count_if( + lane_state.pending, + [&owner](auto const& pending) { + return pending->owner == owner.state_; + } + ); + if (lane_state.pending.size() >= state_->max_pending_per_lane + || std::size_t(owner_pending) >= state_->max_pending_per_owner) + { + return false; + } + if (state_->next_sequence == std::numeric_limits::max()) + qFatal("Raster resource job sequence space exhausted"); + job->sequence = state_->next_sequence++; + lane_state.pending.push_back(std::move(job)); + state_->dispatchLocked(lane); + } + return true; +} + +int RasterResourceManager::threadLimit(Lane lane) const +{ + std::lock_guard lock(state_->mutex); + return state_->laneState(lane).pool.maxThreadCount(); +} + +int RasterResourceManager::activeCount(Lane lane) const +{ + std::lock_guard lock(state_->mutex); + return state_->laneState(lane).active; +} + +std::size_t RasterResourceManager::pendingCount(Lane lane) const +{ + std::lock_guard lock(state_->mutex); + return state_->laneState(lane).pending.size(); +} + +} // namespace OpenOrienteering diff --git a/src/templates/raster_resource_manager.h b/src/templates/raster_resource_manager.h new file mode 100644 index 000000000..c42b6365f --- /dev/null +++ b/src/templates/raster_resource_manager.h @@ -0,0 +1,148 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +#ifndef OPENORIENTEERING_RASTER_RESOURCE_MANAGER_H +#define OPENORIENTEERING_RASTER_RESOURCE_MANAGER_H + +#include +#include +#include +#include + +#include + +namespace OpenOrienteering { + +/** + * Application-wide execution policy for asynchronous raster work. + * + * Sources continue to own demand, typed caches, and retry policy. The manager + * owns only bounded execution, fair ordering, cancellation generations, and + * receiver-safe delivery back to the application thread. + */ +class RasterResourceManager final : public QObject +{ +public: + enum class Lane : std::uint8_t + { + BlockingIo, + Decode, + }; + + enum class Priority : std::uint8_t + { + Coverage, + Visible, + Background, + }; + + struct Limits + { + int blocking_io_threads = 0; + int decode_threads = 0; + std::size_t max_pending_per_owner = 128; + std::size_t max_pending_per_lane = 2048; + }; + + class CancellationToken + { + public: + bool isCancelled() const noexcept; + + private: + friend class RasterResourceManager; + struct OwnerState; + + CancellationToken(std::weak_ptr owner, + std::uint64_t generation) noexcept; + + std::weak_ptr owner_; + std::uint64_t generation_ = 0; + }; + + class Owner + { + public: + Owner() = default; + Owner(Owner&& other) noexcept; + Owner& operator=(Owner&& other) noexcept; + ~Owner(); + + Owner(const Owner&) = delete; + Owner& operator=(const Owner&) = delete; + + bool isValid() const noexcept; + void invalidate(); + void setConcurrencyLimit(Lane lane, int limit); + int concurrencyLimit(Lane lane) const; + std::uint64_t generation() const noexcept; + + private: + friend class RasterResourceManager; + using OwnerState = CancellationToken::OwnerState; + struct SharedState; + + Owner(std::weak_ptr manager, + std::shared_ptr state) noexcept; + void retire(); + + std::weak_ptr manager_; + std::shared_ptr state_; + }; + + using Completion = std::function; + using Work = std::function; + + static Limits defaultLimits(); + static RasterResourceManager& instance(); + + explicit RasterResourceManager( + Limits limits = defaultLimits(), + QObject* parent = nullptr + ); + ~RasterResourceManager() override; + + RasterResourceManager(const RasterResourceManager&) = delete; + RasterResourceManager& operator=(const RasterResourceManager&) = delete; + + Owner createOwner(int concurrency_limit = 1); + + /** + * Queues work for a source owner. + * + * The work runs on the selected lane. Its returned completion runs on this + * manager's thread only when the owner generation is still current and the + * receiver still exists. The receiver must share the manager's thread. + * + * Work may outlive both owner invalidation and receiver destruction. It + * must capture only thread-safe values or shared immutable backend state; + * it must never dereference the owner or receiver. Only the returned + * completion may access receiver-owned state. Captured values must also be + * safe to destroy on a worker thread when work is canceled or stale. + */ + bool submit(const Owner& owner, + Lane lane, + Priority priority, + QObject* receiver, + Work work); + + int threadLimit(Lane lane) const; + int activeCount(Lane lane) const; + std::size_t pendingCount(Lane lane) const; + +private: + using SharedState = Owner::SharedState; + std::shared_ptr state_; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d3fc2c062..6ce5da5d9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -204,6 +204,7 @@ add_system_test(map_printer_t) add_system_test(object_t) add_system_test(object_query_t) add_system_test(path_object_t) +add_system_test(raster_resource_manager_t) add_system_test(template_layer_planner_t) target_link_libraries(template_layer_planner_t PRIVATE Mapper::VelloBackend) add_system_test(render_ir_t) diff --git a/test/gdal_tiled_t.cpp b/test/gdal_tiled_t.cpp index 79148f4c0..9a3ec8640 100644 --- a/test/gdal_tiled_t.cpp +++ b/test/gdal_tiled_t.cpp @@ -154,7 +154,7 @@ private slots: QCOMPARE(GdalTemplate::chooseTileSubsampling(0.25, { 64, 64 }), 4); } - void threadSafeDatasetDecodesOnQtPool() + void threadSafeDatasetDecodesOnSharedScheduler() { QTemporaryDir dir; QVERIFY(dir.isValid()); @@ -168,8 +168,17 @@ private slots: QVERIFY(source.loadTemplateFileImpl()); QVERIFY(source.isTiledSource()); QVERIFY(source.tiled_dataset->IsThreadSafe(GDAL_OF_RASTER)); - QVERIFY(source.tile_pool.maxThreadCount() >= 1); - QVERIFY(source.tile_pool.maxThreadCount() <= 4); + QVERIFY(source.raster_owner.concurrencyLimit( + RasterResourceManager::Lane::BlockingIo + ) >= 1); + QVERIFY(source.raster_owner.concurrencyLimit( + RasterResourceManager::Lane::BlockingIo + ) <= 4); + QVERIFY( + RasterResourceManager::instance().threadLimit( + RasterResourceManager::Lane::BlockingIo + ) <= 4 + ); GdalTemplate::TileWindow window { 0, 0, 1, 1, 1 }; source.queueWantedTiles(window, true); @@ -198,7 +207,9 @@ private slots: Map map; GdalTemplate source(path, &map); QVERIFY(source.loadTemplateFileImpl()); - source.tile_pool.setMaxThreadCount(1); + source.raster_owner.setConcurrencyLimit( + RasterResourceManager::Lane::BlockingIo, 1 + ); source.queueWantedTiles({ 0, 0, 15, 15, 1 }, true); QVERIFY(source.queued_tiles.size() <= 64); QTRY_VERIFY_WITH_TIMEOUT( diff --git a/test/raster_resource_manager_t.cpp b/test/raster_resource_manager_t.cpp new file mode 100644 index 000000000..3317efe30 --- /dev/null +++ b/test/raster_resource_manager_t.cpp @@ -0,0 +1,432 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "templates/raster_resource_manager.h" + +using namespace OpenOrienteering; + +namespace { + +using Lane = RasterResourceManager::Lane; +using Priority = RasterResourceManager::Priority; + +RasterResourceManager::Work gatedWork( + QSemaphore& started, + QSemaphore& gate, + std::function completion = {}) +{ + return [&started, &gate, completion = std::move(completion)]( + const RasterResourceManager::CancellationToken&) mutable { + started.release(); + gate.acquire(); + return std::move(completion); + }; +} + +class WorkDrain +{ +public: + WorkDrain( + RasterResourceManager& manager, + std::initializer_list owners, + std::initializer_list gates) + : manager_(manager) + , owners_(owners) + , gates_(gates) + {} + + ~WorkDrain() + { + for (auto* owner : owners_) + if (owner) + owner->invalidate(); + for (auto* gate : gates_) + if (gate) + gate->release(4096); + + QElapsedTimer timer; + timer.start(); + while (timer.elapsed() < 5000 + && (manager_.activeCount(Lane::BlockingIo) != 0 + || manager_.activeCount(Lane::Decode) != 0 + || manager_.pendingCount(Lane::BlockingIo) != 0 + || manager_.pendingCount(Lane::Decode) != 0)) + { + QCoreApplication::processEvents(); + QThread::msleep(1); + } + } + +private: + RasterResourceManager& manager_; + std::vector owners_; + std::vector gates_; +}; + +} // namespace + +class RasterResourceManagerTest : public QObject +{ +Q_OBJECT + +private slots: + void laneAndOwnerLimitsAreIndependent() + { + RasterResourceManager manager({ 2, 1 }); + auto owner = manager.createOwner(1); + QSemaphore io_started; + QSemaphore io_gate; + QSemaphore decode_started; + QSemaphore decode_gate; + auto completed = 0; + WorkDrain drain(manager, { &owner }, { &io_gate, &decode_gate }); + + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(io_started, io_gate, [&] { ++completed; }) + )); + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(io_started, io_gate, [&] { ++completed; }) + )); + QVERIFY(manager.submit( + owner, Lane::Decode, Priority::Visible, this, + gatedWork(decode_started, decode_gate, [&] { ++completed; }) + )); + + QVERIFY(io_started.tryAcquire(1, 1000)); + QVERIFY(!io_started.tryAcquire(1, 100)); + QVERIFY(decode_started.tryAcquire(1, 1000)); + QCOMPARE(manager.activeCount(Lane::BlockingIo), 1); + QCOMPARE(manager.activeCount(Lane::Decode), 1); + + io_gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(completed, 1, 1000); + QVERIFY(io_started.tryAcquire(1, 1000)); + decode_gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(completed, 2, 1000); + io_gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(completed, 3, 1000); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::Decode), 0, 1000); + } + + void blockingIoDoesNotStarveDecode() + { + RasterResourceManager manager({ 1, 1 }); + auto io_owner = manager.createOwner(); + auto decode_owner = manager.createOwner(); + QSemaphore io_started; + QSemaphore io_gate; + QSemaphore decode_started; + QSemaphore decode_gate; + WorkDrain drain( + manager, { &io_owner, &decode_owner }, { &io_gate, &decode_gate } + ); + + QVERIFY(manager.submit( + io_owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(io_started, io_gate) + )); + QVERIFY(io_started.tryAcquire(1, 1000)); + QVERIFY(manager.submit( + decode_owner, Lane::Decode, Priority::Visible, this, + gatedWork(decode_started, decode_gate) + )); + QVERIFY(decode_started.tryAcquire(1, 1000)); + + io_gate.release(); + decode_gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::Decode), 0, 1000); + } + + void invalidationCancelsOnlyItsOwner() + { + RasterResourceManager manager({ 1, 1 }); + auto first = manager.createOwner(); + auto second = manager.createOwner(); + QSemaphore first_started; + QSemaphore second_started; + QSemaphore second_gate; + auto first_delivered = false; + auto second_delivered = false; + WorkDrain drain( + manager, { &first, &second }, { &second_gate } + ); + + QVERIFY(manager.submit( + first, Lane::BlockingIo, Priority::Visible, this, + [&](const RasterResourceManager::CancellationToken& token) { + first_started.release(); + while (!token.isCancelled()) + QThread::msleep(1); + return RasterResourceManager::Completion { + [&] { first_delivered = true; } + }; + } + )); + QVERIFY(manager.submit( + second, Lane::BlockingIo, Priority::Visible, this, + gatedWork(second_started, second_gate, [&] { second_delivered = true; }) + )); + QVERIFY(first_started.tryAcquire(1, 1000)); + + first.invalidate(); + QVERIFY(second_started.tryAcquire(1, 1000)); + second_gate.release(); + QTRY_VERIFY_WITH_TIMEOUT(second_delivered, 1000); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + QVERIFY(!first_delivered); + } + + void staleAndDestroyedReceiverCompletionsAreSuppressed() + { + RasterResourceManager manager({ 2, 1 }); + auto stale_owner = manager.createOwner(); + auto receiver_owner = manager.createOwner(); + QSemaphore stale_started; + QSemaphore stale_gate; + QSemaphore receiver_started; + QSemaphore receiver_gate; + auto stale_delivered = false; + auto receiver_delivered = false; + auto* receiver = new QObject; + WorkDrain drain( + manager, { &stale_owner, &receiver_owner }, + { &stale_gate, &receiver_gate } + ); + + QVERIFY(manager.submit( + stale_owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(stale_started, stale_gate, [&] { stale_delivered = true; }) + )); + QVERIFY(manager.submit( + receiver_owner, Lane::BlockingIo, Priority::Visible, receiver, + gatedWork( + receiver_started, receiver_gate, + [&] { receiver_delivered = true; } + ) + )); + QVERIFY(stale_started.tryAcquire(1, 1000)); + QVERIFY(receiver_started.tryAcquire(1, 1000)); + + stale_owner.invalidate(); + delete receiver; + stale_gate.release(); + receiver_gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + QCoreApplication::processEvents(); + QVERIFY(!stale_delivered); + QVERIFY(!receiver_delivered); + } + + void activeWorkMayOutliveOwnerAndReceiver() + { + RasterResourceManager manager({ 1, 1 }); + QSemaphore started; + QSemaphore gate; + std::atomic_bool work_finished = false; + auto delivered = false; + WorkDrain drain(manager, {}, { &gate }); + + { + auto owner = manager.createOwner(); + auto receiver = std::make_unique(); + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, receiver.get(), + [&](const RasterResourceManager::CancellationToken&) { + started.release(); + gate.acquire(); + work_finished.store(true, std::memory_order_relaxed); + return RasterResourceManager::Completion { + [&] { delivered = true; } + }; + } + )); + QVERIFY(started.tryAcquire(1, 1000)); + } + + gate.release(); + QTRY_VERIFY_WITH_TIMEOUT( + work_finished.load(std::memory_order_relaxed), 1000 + ); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + QCoreApplication::processEvents(); + QVERIFY(!delivered); + } + + void priorityAndOwnerOrderingAreDeterministic() + { + RasterResourceManager manager({ 1, 1 }); + auto first = manager.createOwner(); + auto second = manager.createOwner(); + QSemaphore started; + QSemaphore gate; + std::mutex order_mutex; + std::vector order; + WorkDrain drain(manager, { &first, &second }, { &gate }); + + auto submit = [&](RasterResourceManager::Owner& owner, + Priority priority, + int marker) { + return manager.submit( + owner, Lane::BlockingIo, priority, this, + [&, marker](const RasterResourceManager::CancellationToken&) { + { + std::lock_guard lock(order_mutex); + order.push_back(marker); + } + started.release(); + gate.acquire(); + return RasterResourceManager::Completion {}; + } + ); + }; + + QVERIFY(submit(first, Priority::Background, 10)); + QVERIFY(started.tryAcquire(1, 1000)); + QVERIFY(submit(first, Priority::Visible, 11)); + QVERIFY(submit(first, Priority::Visible, 12)); + QVERIFY(submit(second, Priority::Visible, 21)); + QVERIFY(submit(second, Priority::Visible, 22)); + QVERIFY(submit(second, Priority::Coverage, 20)); + + for (int index = 0; index < 5; ++index) + { + gate.release(); + QVERIFY(started.tryAcquire(1, 1000)); + } + gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + + std::lock_guard lock(order_mutex); + QCOMPARE(order.size(), std::size_t(6)); + QCOMPARE(order[0], 10); + QCOMPARE(order[1], 20); + QCOMPARE(order[2], 11); + QCOMPARE(order[3], 21); + QCOMPARE(order[4], 12); + QCOMPARE(order[5], 22); + } + + void laneCapAndOwnerParallelismAreBounded() + { + RasterResourceManager manager({ 2, 1 }); + auto owner = manager.createOwner(2); + QSemaphore started; + QSemaphore gate; + WorkDrain drain(manager, { &owner }, { &gate }); + + for (int index = 0; index < 3; ++index) + { + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(started, gate) + )); + } + QVERIFY(started.tryAcquire(2, 1000)); + QVERIFY(!started.tryAcquire(1, 100)); + QCOMPARE(manager.activeCount(Lane::BlockingIo), 2); + gate.release(2); + QVERIFY(started.tryAcquire(1, 1000)); + gate.release(); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + } + + void queueBoundsRejectExcessWork() + { + RasterResourceManager::Limits limits; + limits.blocking_io_threads = 1; + limits.decode_threads = 1; + limits.max_pending_per_owner = 2; + limits.max_pending_per_lane = 3; + RasterResourceManager manager(limits); + auto owner = manager.createOwner(1); + QSemaphore started; + QSemaphore gate; + WorkDrain drain(manager, { &owner }, { &gate }); + + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(started, gate) + )); + QVERIFY(started.tryAcquire(1, 1000)); + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(started, gate) + )); + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(started, gate) + )); + QVERIFY(!manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + gatedWork(started, gate) + )); + QCOMPARE(manager.pendingCount(Lane::BlockingIo), std::size_t(2)); + } + + void exceptionReleasesSlotAndCompletionUsesManagerThread() + { + RasterResourceManager manager({ 1, 1 }); + auto owner = manager.createOwner(); + WorkDrain drain(manager, { &owner }, {}); + std::atomic_bool work_off_manager_thread = false; + auto completion_on_manager_thread = false; + auto delivered = false; + + QTest::ignoreMessage( + QtWarningMsg, + QRegularExpression(QStringLiteral("Raster worker failed:.*boom")) + ); + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + [&](const RasterResourceManager::CancellationToken&) { + work_off_manager_thread.store( + QThread::currentThread() != manager.thread(), + std::memory_order_relaxed + ); + throw std::runtime_error("boom"); + return RasterResourceManager::Completion {}; + } + )); + QVERIFY(manager.submit( + owner, Lane::BlockingIo, Priority::Visible, this, + [&](const RasterResourceManager::CancellationToken&) { + return RasterResourceManager::Completion { + [&] { + completion_on_manager_thread = + QThread::currentThread() == manager.thread(); + delivered = true; + } + }; + } + )); + + QTRY_VERIFY_WITH_TIMEOUT(delivered, 1000); + QVERIFY(work_off_manager_thread.load(std::memory_order_relaxed)); + QVERIFY(completion_on_manager_thread); + QTRY_COMPARE_WITH_TIMEOUT(manager.activeCount(Lane::BlockingIo), 0, 1000); + } +}; + +QTEST_MAIN(RasterResourceManagerTest) +#include "raster_resource_manager_t.moc" From 2e4868115bda0635f69a30f248082be18e4e4e58 Mon Sep 17 00:00:00 2001 From: Ethan O'Connor Date: Thu, 16 Jul 2026 11:46:57 -0700 Subject: [PATCH 03/39] raster: support direct image-to-map tiles --- src/render/template_layer_planner.cpp | 48 +++++++++-- src/templates/template_image.h | 23 ++++++ test/template_layer_planner_t.cpp | 112 +++++++++++++++++++++----- test/template_layer_planner_t.h | 2 + 4 files changed, 158 insertions(+), 27 deletions(-) diff --git a/src/render/template_layer_planner.cpp b/src/render/template_layer_planner.cpp index 7e1e17090..fd603090e 100644 --- a/src/render/template_layer_planner.cpp +++ b/src/render/template_layer_planner.cpp @@ -50,6 +50,8 @@ struct TileKey ImageKey image; Rect target; Rect source; + Transform image_to_scene; + bool direct_to_map = false; bool operator==(const TileKey& other) const { @@ -57,7 +59,14 @@ struct TileKey && target.x == other.target.x && target.y == other.target.y && target.width == other.target.width && target.height == other.target.height && source.x == other.source.x && source.y == other.source.y - && source.width == other.source.width && source.height == other.source.height; + && source.width == other.source.width && source.height == other.source.height + && image_to_scene.m11 == other.image_to_scene.m11 + && image_to_scene.m12 == other.image_to_scene.m12 + && image_to_scene.m21 == other.image_to_scene.m21 + && image_to_scene.m22 == other.image_to_scene.m22 + && image_to_scene.dx == other.image_to_scene.dx + && image_to_scene.dy == other.image_to_scene.dy + && direct_to_map == other.direct_to_map; } }; @@ -215,6 +224,18 @@ RasterMosaic transparentMosaic(const std::vector& tiles) }; } +Rect fullImageTarget(const TileKey& tile) +{ + auto const scale_x = tile.target.width / tile.source.width; + auto const scale_y = tile.target.height / tile.source.height; + return { + tile.target.x - tile.source.x * scale_x, + tile.target.y - tile.source.y * scale_y, + tile.image.width * scale_x, + tile.image.height * scale_y, + }; +} + } // namespace class TemplateLayerPlanner::Impl @@ -416,6 +437,10 @@ class TemplateLayerPlanner::Impl image_key, fromQRectF(tile.template_rect), fromQRectF(tile.source_rect), + tile.has_image_to_map + ? fromQTransform(tile.image_to_map) + : Transform {}, + tile.has_image_to_map, }; full_key.tiles.push_back(tile_key); source_images.push_back({ std::move(tile_key), tile.image }); @@ -436,7 +461,11 @@ class TemplateLayerPlanner::Impl return !isOpaque(tile.key.image, tile.image); } ); - if (has_transparency) + auto const has_direct_tiles = std::ranges::any_of( + source_images, + [](auto const& tile) { return tile.key.direct_to_map; } + ); + if (has_transparency && !has_direct_tiles) { if (on_screen && result.newly_resident_images >= max_new_images_per_frame) { @@ -488,10 +517,19 @@ class TemplateLayerPlanner::Impl if (next_revision_ == std::numeric_limits::max()) qFatal("Raster layer revision space exhausted"); RenderIRBuilder builder(next_revision_++); - builder.pushTransform(key.template_to_map); for (auto const& [tile, image] : tiles) - builder.drawImage(image, tile.source, tile.target); - builder.popTransform(); + { + if (tile.direct_to_map) + { + builder.drawImage(image, tile.source, tile.image_to_scene); + } + else + { + builder.pushTransform(key.template_to_map); + builder.drawImage(image, fullImageTarget(tile)); + builder.popTransform(); + } + } auto scene = builder.finish(); layers_[&source] = { std::move(key), scene }; return scene; diff --git a/src/templates/template_image.h b/src/templates/template_image.h index 849388990..868dbfa60 100644 --- a/src/templates/template_image.h +++ b/src/templates/template_image.h @@ -23,6 +23,7 @@ #define OPENORIENTEERING_TEMPLATE_IMAGE_H #include +#include #include #include @@ -53,12 +54,34 @@ class MapCoordF; struct RasterTemplateTile { + RasterTemplateTile() = default; + RasterTemplateTile( + QImage image, + QRectF template_rect, + QRectF source_rect, + quint64 cache_key = 0, + bool missing = false, + bool provisional = false, + QTransform image_to_map = {}, + bool has_image_to_map = false) + : image(std::move(image)) + , template_rect(std::move(template_rect)) + , source_rect(std::move(source_rect)) + , cache_key(cache_key) + , missing(missing) + , provisional(provisional) + , image_to_map(std::move(image_to_map)) + , has_image_to_map(has_image_to_map) + {} + QImage image; QRectF template_rect; QRectF source_rect; quint64 cache_key = 0; bool missing = false; bool provisional = false; + QTransform image_to_map; + bool has_image_to_map = false; }; diff --git a/test/template_layer_planner_t.cpp b/test/template_layer_planner_t.cpp index 9e8f2e6bb..e5a660585 100644 --- a/test/template_layer_planner_t.cpp +++ b/test/template_layer_planner_t.cpp @@ -347,20 +347,54 @@ void TemplateLayerPlannerTest::marksFallbackLayersIncomplete() QCOMPARE(imageCommandCount(plan.below_map.front()), std::size_t(1)); } -void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() +void TemplateLayerPlannerTest::respectsExplicitImageToMapTransform() { - QImage source(16, 8, QImage::Format_RGBA8888); - for (int y = 0; y < source.height(); ++y) - { - for (int x = 0; x < source.width(); ++x) - { - source.setPixelColor( - x, y, - QColor(20 + x * 12, 190 - x * 7, 30 + y * 20, 80 + x * 8) - ); - } - } + Map map; + MapView view { &map }; + auto direct = tile(solidImage({ 4, 3 }, Qt::magenta), { 100, 200, 4, 3 }); + direct.image_to_map = QTransform(1.25, 0.2, -0.15, 0.9, -2.5, -1.25); + direct.has_image_to_map = true; + auto raster = std::make_unique( + &map, QVector { std::move(direct) }, QRectF(-4, -3, 8, 6) + ); + raster->setTemplateX(5000); + raster->setTemplateY(-3000); + raster->setTemplateScaleX(7); + raster->setTemplateRotation(0.35); + map.addTemplate(0, std::move(raster)); + map.setFirstFrontTemplate(1); + view.setTemplateVisibility(map.getTemplate(0), { 1, true }); + + render::TemplateLayerPlanner planner; + auto const plan = planner.plan(map, view, { -8, -8, 16, 16 }, 1); + QVERIFY(plan.complete); + QCOMPARE(plan.below_map.size(), std::size_t(1)); + QCOMPARE(imageCommandCount(plan.below_map.front()), std::size_t(1)); + auto const command = std::ranges::find_if( + plan.below_map.front().scene->commands, + [](auto const& value) { return std::holds_alternative(value); } + ); + QVERIFY(command != plan.below_map.front().scene->commands.end()); + auto const& image = std::get(*command); + QCOMPARE(image.source.x, 0.0); + QCOMPARE(image.source.y, 0.0); + QCOMPARE(image.source.width, 4.0); + QCOMPARE(image.source.height, 3.0); + QCOMPARE(image.image_to_scene.m11, 1.25); + QCOMPARE(image.image_to_scene.m12, 0.2); + QCOMPARE(image.image_to_scene.m21, -0.15); + QCOMPARE(image.image_to_scene.m22, 0.9); + QCOMPARE(image.image_to_scene.dx, -2.5); + QCOMPARE(image.image_to_scene.dy, -1.25); +} +namespace { + +void verifyGutterSeams( + QImage source, + std::size_t expected_images, + int max_channel_delta) +{ Map tiled_map; MapView tiled_view { &tiled_map }; auto left = source.copy(0, 0, 9, 8); @@ -394,9 +428,9 @@ void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() auto whole_plan = whole_planner.plan(whole_map, whole_view, { -10, -7, 20, 14 }, 3.2); QVERIFY(tiled_plan.complete); QVERIFY(whole_plan.complete); - QCOMPARE(tiled_plan.newly_resident_images, std::size_t(1)); + QCOMPARE(tiled_plan.newly_resident_images, expected_images); QCOMPARE(tiled_plan.below_map.size(), std::size_t(1)); - QCOMPARE(imageCommandCount(tiled_plan.below_map.front()), std::size_t(1)); + QCOMPARE(imageCommandCount(tiled_plan.below_map.front()), expected_images); auto const tiled_snapshot = tiled_map.publishRenderSnapshot(); auto const whole_snapshot = whole_map.publishRenderSnapshot(); @@ -427,19 +461,19 @@ void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() auto const reference_actual = tiled_reference.pixelColor(x, y); auto const reference_expected = whole_reference.pixelColor(x, y); QVERIFY2( - std::abs(actual.red() - expected.red()) <= 2 - && std::abs(actual.green() - expected.green()) <= 2 - && std::abs(actual.blue() - expected.blue()) <= 2 - && std::abs(actual.alpha() - expected.alpha()) <= 2, + std::abs(actual.red() - expected.red()) <= max_channel_delta + && std::abs(actual.green() - expected.green()) <= max_channel_delta + && std::abs(actual.blue() - expected.blue()) <= max_channel_delta + && std::abs(actual.alpha() - expected.alpha()) <= max_channel_delta, qPrintable(QStringLiteral("tile seam at %1,%2: %3 vs %4") .arg(x).arg(y).arg(actual.name(QColor::HexArgb), expected.name(QColor::HexArgb))) ); QVERIFY2( - std::abs(reference_actual.red() - reference_expected.red()) <= 2 - && std::abs(reference_actual.green() - reference_expected.green()) <= 2 - && std::abs(reference_actual.blue() - reference_expected.blue()) <= 2 - && std::abs(reference_actual.alpha() - reference_expected.alpha()) <= 2, + std::abs(reference_actual.red() - reference_expected.red()) <= max_channel_delta + && std::abs(reference_actual.green() - reference_expected.green()) <= max_channel_delta + && std::abs(reference_actual.blue() - reference_expected.blue()) <= max_channel_delta + && std::abs(reference_actual.alpha() - reference_expected.alpha()) <= max_channel_delta, qPrintable(QStringLiteral("reference tile seam at %1,%2: %3 vs %4") .arg(x).arg(y).arg(reference_actual.name(QColor::HexArgb), reference_expected.name(QColor::HexArgb))) @@ -448,4 +482,38 @@ void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() } } +} // namespace + +void TemplateLayerPlannerTest::preservesOpaqueGuttersWithoutTileSeams() +{ + QImage source(16, 8, QImage::Format_RGB32); + for (int y = 0; y < source.height(); ++y) + { + for (int x = 0; x < source.width(); ++x) + { + source.setPixelColor( + x, y, + QColor(20 + x * 12, 190 - x * 7, 30 + y * 20) + ); + } + } + verifyGutterSeams(std::move(source), 2, 8); +} + +void TemplateLayerPlannerTest::preservesTransparentGuttersWithoutTileSeams() +{ + QImage source(16, 8, QImage::Format_RGBA8888); + for (int y = 0; y < source.height(); ++y) + { + for (int x = 0; x < source.width(); ++x) + { + source.setPixelColor( + x, y, + QColor(20 + x * 12, 190 - x * 7, 30 + y * 20, 80 + x * 8) + ); + } + } + verifyGutterSeams(std::move(source), 1, 2); +} + QTEST_MAIN(TemplateLayerPlannerTest) diff --git a/test/template_layer_planner_t.h b/test/template_layer_planner_t.h index 3c7e1b031..5dfdcdc03 100644 --- a/test/template_layer_planner_t.h +++ b/test/template_layer_planner_t.h @@ -19,6 +19,8 @@ private slots: void recordsVectorMapAndTrackTemplates(); void boundsImageAdmissionAndPreservesVelloIdentity(); void marksFallbackLayersIncomplete(); + void respectsExplicitImageToMapTransform(); + void preservesOpaqueGuttersWithoutTileSeams(); void preservesTransparentGuttersWithoutTileSeams(); }; From c7deabc49ea0d0884029caed6c4954e31bc36e98 Mon Sep 17 00:00:00 2001 From: Ethan O'Connor Date: Thu, 16 Jul 2026 12:02:33 -0700 Subject: [PATCH 04/39] imagery: add resolved source snapshot core --- CMakeLists.txt | 1 + src/CMakeLists.txt | 1 + src/imagery/CMakeLists.txt | 32 + src/imagery/imagery_source.cpp | 392 ++++++++++ src/imagery/imagery_source.h | 176 +++++ src/imagery/imagery_source_snapshot.cpp | 918 ++++++++++++++++++++++++ src/imagery/imagery_source_snapshot.h | 56 ++ src/imagery/tile_matrix_set.cpp | 209 ++++++ src/imagery/tile_matrix_set.h | 86 +++ test/CMakeLists.txt | 4 + test/imagery_core_t.cpp | 243 +++++++ test/imagery_core_t.h | 26 + 12 files changed, 2144 insertions(+) create mode 100644 src/imagery/CMakeLists.txt create mode 100644 src/imagery/imagery_source.cpp create mode 100644 src/imagery/imagery_source.h create mode 100644 src/imagery/imagery_source_snapshot.cpp create mode 100644 src/imagery/imagery_source_snapshot.h create mode 100644 src/imagery/tile_matrix_set.cpp create mode 100644 src/imagery/tile_matrix_set.h create mode 100644 test/imagery_core_t.cpp create mode 100644 test/imagery_core_t.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 206c2b299..2b80f4822 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -216,6 +216,7 @@ if(Mapper_WITH_COVE) add_feature_info(Mapper_WITH_COVE ON "Contour line vectorization") add_subdirectory(3rd-party/cove) endif() +add_subdirectory(src/imagery) if(Mapper_USE_GDAL) add_subdirectory(src/gdal) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index eca48efd0..415db9887 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -350,6 +350,7 @@ qt_add_resources(Mapper_Common "mapper-translations" target_link_libraries(Mapper_Common PUBLIC Clipper2::Clipper2 + Mapper::ImageryCore Mapper::RenderIR ${PROJ_LIBRARIES} Qt6::Widgets diff --git a/src/imagery/CMakeLists.txt b/src/imagery/CMakeLists.txt new file mode 100644 index 000000000..9476d63a2 --- /dev/null +++ b/src/imagery/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# Copyright 2026 Ethan O'Connor +# +# This file is part of OpenOrienteering. +# + +set(MAPPER_IMAGERY_CORE_SOURCES + imagery_source.cpp + imagery_source.h + imagery_source_snapshot.cpp + imagery_source_snapshot.h + tile_matrix_set.cpp + tile_matrix_set.h +) + +add_library(mapper-imagery-core STATIC ${MAPPER_IMAGERY_CORE_SOURCES}) +add_library(Mapper::ImageryCore ALIAS mapper-imagery-core) +mapper_target_defaults(mapper-imagery-core) +target_include_directories(mapper-imagery-core PUBLIC "${PROJECT_SOURCE_DIR}/src") +target_link_libraries(mapper-imagery-core PUBLIC Qt6::Core) +target_compile_definitions(mapper-imagery-core PRIVATE + QT_NO_CAST_FROM_ASCII + QT_NO_CAST_TO_ASCII + QT_USE_QSTRINGBUILDER +) +set_target_properties(mapper-imagery-core PROPERTIES + AUTOMOC OFF + AUTOUIC OFF + PREFIX "" +) + +mapper_translations_sources(${MAPPER_IMAGERY_CORE_SOURCES}) diff --git a/src/imagery/imagery_source.cpp b/src/imagery/imagery_source.cpp new file mode 100644 index 000000000..2141de050 --- /dev/null +++ b/src/imagery/imagery_source.cpp @@ -0,0 +1,392 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "imagery/imagery_source.h" + +#include +#include + +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +bool fail(QString* error, const QString& message) +{ + if (error) + *error = message; + return false; +} + +bool containsControl(const QString& value) +{ + for (auto const character : value) + { + auto const code = character.unicode(); + if (code < 0x20 || code == 0x7f) + return true; + } + return false; +} + +bool validPlainText(const QString& value, qsizetype maximum) +{ + return value.size() <= maximum && !containsControl(value); +} + +bool validId(const QString& value) +{ + static const QRegularExpression pattern( + QStringLiteral("^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + ); + return pattern.match(value).hasMatch(); +} + +bool validMediaType(const QString& value) +{ + static const QRegularExpression pattern( + QStringLiteral("^image/[A-Za-z0-9!#$&^_.+-]{1,96}$") + ); + return pattern.match(value).hasMatch(); +} + +bool validSha256(const QByteArray& value) +{ + if (value.size() != 64) + return false; + for (auto const byte : value) + { + if (!((byte >= '0' && byte <= '9') || (byte >= 'a' && byte <= 'f'))) + return false; + } + return true; +} + +bool validateHttpUrl(const QUrl& url, QString* error) +{ + if (!url.isValid() || url.isRelative() || url.host().isEmpty()) + return fail(error, QStringLiteral("URL must be absolute and contain a host")); + auto const scheme = url.scheme().toLower(); + if (scheme != QLatin1String("http") && scheme != QLatin1String("https")) + return fail(error, QStringLiteral("URL must use HTTP or HTTPS")); + if (!url.userName().isEmpty() || !url.password().isEmpty()) + return fail(error, QStringLiteral("URL user information is not allowed")); + if (url.hasFragment()) + return fail(error, QStringLiteral("URL fragments are not allowed")); + auto const text = url.toString(QUrl::FullyEncoded); + if (text.size() > 8192 || containsControl(text)) + return fail(error, QStringLiteral("URL is too long or contains a control character")); + if (error) + error->clear(); + return true; +} + +bool validateOptionalHttpUrl(const QUrl& url, QString* error) +{ + if (url.isEmpty()) + return true; + return validateHttpUrl(url, error); +} + +bool validateProvenance(const ImageryProvenance& provenance, QString* error) +{ + if (!validPlainText(provenance.method, 256) + || !validPlainText(provenance.author, 512) + || !validPlainText(provenance.notes, 4096)) + { + return fail(error, QStringLiteral("Registration provenance contains invalid text")); + } + if (!provenance.observed.isNull() && !provenance.observed.isValid()) + return fail(error, QStringLiteral("Registration provenance date is invalid")); + if (provenance.rms_error + && (!std::isfinite(*provenance.rms_error) || *provenance.rms_error < 0)) + { + return fail(error, QStringLiteral("Registration RMS error must be finite and nonnegative")); + } + return true; +} + +qsizetype authorityEnd(const QString& url) +{ + auto const scheme_end = url.indexOf(QStringLiteral("://")); + if (scheme_end < 0) + return -1; + auto result = url.size(); + for (auto const separator : { QLatin1Char('/'), QLatin1Char('?'), QLatin1Char('#') }) + { + auto const position = url.indexOf(separator, scheme_end + 3); + if (position >= 0) + result = std::min(result, position); + } + return result; +} + +} // namespace + +QString categoryName(ImageryCategory category) +{ + switch (category) + { + case ImageryCategory::Aerial: return QStringLiteral("aerial"); + case ImageryCategory::Satellite: return QStringLiteral("satellite"); + case ImageryCategory::Map: return QStringLiteral("map"); + case ImageryCategory::Elevation: return QStringLiteral("elevation"); + case ImageryCategory::Other: return QStringLiteral("other"); + } + Q_UNREACHABLE_RETURN(QStringLiteral("other")); +} + +std::optional categoryFromName(const QString& name) +{ + if (name == QLatin1String("aerial")) + return ImageryCategory::Aerial; + if (name == QLatin1String("satellite")) + return ImageryCategory::Satellite; + if (name == QLatin1String("map")) + return ImageryCategory::Map; + if (name == QLatin1String("elevation")) + return ImageryCategory::Elevation; + if (name == QLatin1String("other")) + return ImageryCategory::Other; + return std::nullopt; +} + +QString tileRowSchemeName(TileRowScheme scheme) +{ + return scheme == TileRowScheme::Tms ? QStringLiteral("tms") : QStringLiteral("xyz"); +} + +std::optional tileRowSchemeFromName(const QString& name) +{ + if (name == QLatin1String("xyz")) + return TileRowScheme::Xyz; + if (name == QLatin1String("tms")) + return TileRowScheme::Tms; + return std::nullopt; +} + +bool TileUrlTemplate::validate(QString* error) const +{ + if (value.isEmpty() || value.size() > 8192 || containsControl(value)) + return fail(error, QStringLiteral("Tile URL template is empty, too long, or contains a control character")); + if (value.contains(QStringLiteral("${"))) + return fail(error, QStringLiteral("Tile URL placeholders must use exact {z}, {x}, and {y} spelling")); + if (value.count(QStringLiteral("{z}")) != 1 + || value.count(QStringLiteral("{x}")) != 1 + || value.count(QStringLiteral("{y}")) != 1) + { + return fail(error, QStringLiteral("Tile URL template must contain each of {z}, {x}, and {y} exactly once")); + } + + auto remainder = value; + remainder.remove(QStringLiteral("{z}")); + remainder.remove(QStringLiteral("{x}")); + remainder.remove(QStringLiteral("{y}")); + if (remainder.contains(QLatin1Char('{')) || remainder.contains(QLatin1Char('}'))) + return fail(error, QStringLiteral("Tile URL template contains an unsupported placeholder")); + + auto const authority_end = authorityEnd(value); + if (authority_end < 0) + return fail(error, QStringLiteral("Tile URL template is not an absolute URL")); + for (auto const& placeholder : { + QStringLiteral("{z}"), QStringLiteral("{x}"), QStringLiteral("{y}") + }) + { + if (value.indexOf(placeholder) < authority_end) + return fail(error, QStringLiteral("Tile URL placeholders are not allowed in the authority")); + } + + auto probe = value; + probe.replace(QStringLiteral("{z}"), QStringLiteral("0")); + probe.replace(QStringLiteral("{x}"), QStringLiteral("0")); + probe.replace(QStringLiteral("{y}"), QStringLiteral("0")); + return validateHttpUrl(QUrl(probe, QUrl::StrictMode), error); +} + +QUrl TileUrlTemplate::expand(const TileMatrix& matrix, + qint64 column, + qint64 canonical_top_row, + TileRowScheme scheme, + QString* error) const +{ + if (!validate(error)) + return {}; + if (!matrix.contains(column, canonical_top_row)) + { + fail(error, QStringLiteral("Tile coordinate falls outside the matrix")); + return {}; + } + + auto request_row = canonical_top_row; + if (scheme == TileRowScheme::Tms) + request_row = matrix.matrix_height - 1 - canonical_top_row; + + auto expanded = value; + expanded.replace(QStringLiteral("{z}"), matrix.id); + expanded.replace(QStringLiteral("{x}"), QString::number(column)); + expanded.replace(QStringLiteral("{y}"), QString::number(request_row)); + QUrl result(expanded, QUrl::StrictMode); + if (!validateHttpUrl(result, error)) + return {}; + return result; +} + +bool ResolvedImagerySource::validate(QString* error) const +{ + if (!validId(metadata.id)) + return fail(error, QStringLiteral("Imagery source ID is invalid")); + if (metadata.name.trimmed().isEmpty() || !validPlainText(metadata.name, 512) + || !validPlainText(metadata.description, 4096)) + { + return fail(error, QStringLiteral("Imagery source metadata contains invalid text")); + } + if (!metadata.start_date.isNull() && !metadata.start_date.isValid()) + return fail(error, QStringLiteral("Imagery start date is invalid")); + if (!metadata.end_date.isNull() && !metadata.end_date.isValid()) + return fail(error, QStringLiteral("Imagery end date is invalid")); + if (metadata.start_date.isValid() && metadata.end_date.isValid() + && metadata.start_date > metadata.end_date) + { + return fail(error, QStringLiteral("Imagery start date follows its end date")); + } + + if (!validPlainText(notices.attribution_text, 2048) + || !validPlainText(notices.notes, 4096)) + { + return fail(error, QStringLiteral("Imagery notices contain invalid text")); + } + for (auto const* url : { + ¬ices.attribution_url, ¬ices.source_url, + ¬ices.terms_url, ¬ices.privacy_url + }) + { + if (!validateOptionalHttpUrl(*url, error)) + return false; + } + + if (tile_urls.isEmpty() || tile_urls.size() > 8) + return fail(error, QStringLiteral("Imagery source must contain between one and eight tile URL templates")); + QSet unique_templates; + for (auto const& tile_url : tile_urls) + { + if (!tile_url.validate(error)) + return false; + if (unique_templates.contains(tile_url.value)) + return fail(error, QStringLiteral("Imagery source contains a duplicate tile URL template")); + unique_templates.insert(tile_url.value); + } + + if (!validMediaType(media_type)) + return fail(error, QStringLiteral("Imagery source media type is invalid")); + if (!tile_matrix_set.validateDyadicTopLeft(error)) + return false; + if (min_zoom < 0 || max_zoom < min_zoom + || !tile_matrix_set.matrixForZoom(min_zoom) + || !tile_matrix_set.matrixForZoom(max_zoom)) + { + return fail(error, QStringLiteral("Imagery source zoom range is invalid")); + } + if (!validateTileMatrixLimits(tile_limits, tile_matrix_set, error)) + return false; + for (auto const& limit : tile_limits) + { + if (limit.zoom < min_zoom || limit.zoom > max_zoom) + return fail(error, QStringLiteral("Tile limits fall outside the usable zoom range")); + } + + if (!validateOptionalHttpUrl(request.referer, error)) + return false; + QSet status_codes; + for (auto const status : request.empty_http_status_codes) + { + if (status < 100 || status > 599 || status_codes.contains(status)) + return fail(error, QStringLiteral("Empty-tile HTTP status codes must be unique values from 100 through 599")); + status_codes.insert(status); + } + + if (catalog_provenance) + { + auto const& value = *catalog_provenance; + if (!validId(value.catalog_id) || !validId(value.source_id) + || value.source_id != metadata.id || value.catalog_revision <= 0) + { + return fail(error, QStringLiteral("Catalog source provenance identity is invalid")); + } + if (!validSha256(value.catalog_sha256) + || !validSha256(value.full_fingerprint) + || !validSha256(value.operational_fingerprint)) + { + return fail(error, QStringLiteral("Catalog source provenance fingerprints must be lowercase SHA-256")); + } + } + + if (registration) + { + auto const& value = *registration; + if (value.source_crs != tile_matrix_set.crs + || value.target_crs != tile_matrix_set.crs) + { + return fail(error, QStringLiteral("Translation registration frames must match the tile matrix set CRS")); + } + if (!std::isfinite(value.dx) || !std::isfinite(value.dy)) + return fail(error, QStringLiteral("Translation registration must be finite")); + if (!validPlainText(value.target_frame_id, 256)) + return fail(error, QStringLiteral("Translation target frame ID is invalid")); + if (!validateProvenance(value.provenance, error)) + return false; + } + + if (error) + error->clear(); + return true; +} + +const TileMatrixLimits* ResolvedImagerySource::limitsForZoom(int zoom) const noexcept +{ + for (auto const& limit : tile_limits) + { + if (limit.zoom == zoom) + return &limit; + } + return nullptr; +} + +QUrl ResolvedImagerySource::tileUrl(int template_index, + int zoom, + qint64 column, + qint64 canonical_top_row, + QString* error) const +{ + if (template_index < 0 || template_index >= tile_urls.size()) + { + fail(error, QStringLiteral("Tile URL template index is out of range")); + return {}; + } + if (zoom < min_zoom || zoom > max_zoom) + { + fail(error, QStringLiteral("Tile zoom falls outside the usable range")); + return {}; + } + auto const* matrix = tile_matrix_set.matrixForZoom(zoom); + if (!matrix || !matrix->contains(column, canonical_top_row)) + { + fail(error, QStringLiteral("Tile coordinate falls outside the matrix")); + return {}; + } + if (auto const* limits = limitsForZoom(zoom); + limits && !limits->contains(column, canonical_top_row)) + { + fail(error, QStringLiteral("Tile coordinate falls outside source limits")); + return {}; + } + return tile_urls.at(template_index).expand( + *matrix, column, canonical_top_row, row_scheme, error + ); +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/imagery_source.h b/src/imagery/imagery_source.h new file mode 100644 index 000000000..d296d33e7 --- /dev/null +++ b/src/imagery/imagery_source.h @@ -0,0 +1,176 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_SOURCE_H +#define OPENORIENTEERING_IMAGERY_SOURCE_H + +#include + +#include +#include +#include +#include +#include + +#include "imagery/tile_matrix_set.h" + +namespace OpenOrienteering::imagery { + +enum class ImageryCategory +{ + Aerial, + Satellite, + Map, + Elevation, + Other, +}; + +enum class TileRowScheme +{ + Xyz, + Tms, +}; + +QString categoryName(ImageryCategory category); +std::optional categoryFromName(const QString& name); +QString tileRowSchemeName(TileRowScheme scheme); +std::optional tileRowSchemeFromName(const QString& name); + +struct ImageryMetadata +{ + QString id; + QString name; + QString description; + ImageryCategory category = ImageryCategory::Other; + QDate start_date; + QDate end_date; + + bool operator==(const ImageryMetadata&) const = default; +}; + +struct ImageryNotices +{ + QString attribution_text; + QUrl attribution_url; + QUrl source_url; + QUrl terms_url; + QUrl privacy_url; + QString notes; + + bool operator==(const ImageryNotices&) const = default; +}; + +struct ImageryRequestPolicy +{ + QUrl referer; + QVector empty_http_status_codes { 204, 404 }; + + bool operator==(const ImageryRequestPolicy&) const = default; +}; + +/** + * Identity of the installed catalog snapshot from which a source was resolved. + * + * This is intentionally distinct from surveyed registration provenance. + * Embedding it in a map permits an explicit future update comparison without + * making the map depend on a mutable catalog installation. + */ +struct CatalogSourceProvenance +{ + QString catalog_id; + int catalog_revision = 0; + QByteArray catalog_sha256; + QString source_id; + QByteArray full_fingerprint; + QByteArray operational_fingerprint; + + bool operator==(const CatalogSourceProvenance&) const = default; +}; + +struct ImageryProvenance +{ + QString method; + QDate observed; + QString author; + std::optional rms_error; + QString notes; + + bool operator==(const ImageryProvenance&) const = default; +}; + +/** + * The only registration operation executable by the resolved runtime. + * + * Direction and units are intentionally typed rather than configurable: + * source-to-corrected, with dx/dy in the shared CRS linear unit. + */ +struct TranslationRegistration +{ + QString source_crs; + QString target_crs; + QString target_frame_id; + double dx = 0; + double dy = 0; + ImageryProvenance provenance; + + bool operator==(const TranslationRegistration&) const = default; +}; + +struct TileUrlTemplate +{ + QString value; + + bool validate(QString* error = nullptr) const; + QUrl expand(const TileMatrix& matrix, + qint64 column, + qint64 canonical_top_row, + TileRowScheme scheme, + QString* error = nullptr) const; + + bool operator==(const TileUrlTemplate&) const = default; +}; + +/** + * A fully resolved, self-contained source ready for request scheduling. + * + * This model deliberately cannot carry affine or grid-shift registration. + * Catalog definitions requiring those operations must remain disabled until a + * runtime with explicit support resolves them. + */ +struct ResolvedImagerySource +{ + ImageryMetadata metadata; + ImageryNotices notices; + QVector tile_urls; + TileRowScheme row_scheme = TileRowScheme::Xyz; + QString media_type = QStringLiteral("image/png"); + TileMatrixSet tile_matrix_set; + int min_zoom = 0; + int max_zoom = -1; + QVector tile_limits; + ImageryRequestPolicy request; + std::optional catalog_provenance; + std::optional registration; + + bool validate(QString* error = nullptr) const; + const TileMatrixLimits* limitsForZoom(int zoom) const noexcept; + QUrl tileUrl(int template_index, + int zoom, + qint64 column, + qint64 canonical_top_row, + QString* error = nullptr) const; + + bool operator==(const ResolvedImagerySource&) const = default; +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/imagery_source_snapshot.cpp b/src/imagery/imagery_source_snapshot.cpp new file mode 100644 index 000000000..13d045b3d --- /dev/null +++ b/src/imagery/imagery_source_snapshot.cpp @@ -0,0 +1,918 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "imagery/imagery_source_snapshot.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +bool fail(QString* error, const QString& message) +{ + if (error) + *error = message; + return false; +} + +template +std::optional failOptional(QString* error, const QString& message) +{ + fail(error, message); + return std::nullopt; +} + +bool hasOnlyKeys(const QJsonObject& object, + std::initializer_list allowed, + const QString& path, + QString* error) +{ + QSet keys(allowed.begin(), allowed.end()); + for (auto it = object.begin(); it != object.end(); ++it) + { + if (!keys.contains(it.key())) + return fail(error, QStringLiteral("%1 contains unknown member %2").arg(path, it.key())); + } + return true; +} + +bool requiredObject(const QJsonObject& parent, + const QString& name, + QJsonObject* output, + const QString& path, + QString* error) +{ + auto const value = parent.value(name); + if (!value.isObject()) + return fail(error, QStringLiteral("%1.%2 must be an object").arg(path, name)); + *output = value.toObject(); + return true; +} + +bool requiredArray(const QJsonObject& parent, + const QString& name, + QJsonArray* output, + const QString& path, + QString* error) +{ + auto const value = parent.value(name); + if (!value.isArray()) + return fail(error, QStringLiteral("%1.%2 must be an array").arg(path, name)); + *output = value.toArray(); + return true; +} + +bool requiredString(const QJsonObject& parent, + const QString& name, + QString* output, + const QString& path, + QString* error) +{ + auto const value = parent.value(name); + if (!value.isString()) + return fail(error, QStringLiteral("%1.%2 must be a string").arg(path, name)); + *output = value.toString(); + return true; +} + +bool integerValue(const QJsonValue& value, + qint64 minimum, + qint64 maximum, + qint64* output) +{ + if (!value.isDouble()) + return false; + auto const number = value.toDouble(); + if (!std::isfinite(number) || std::floor(number) != number + || number < double(minimum) || number > double(maximum)) + { + return false; + } + *output = qint64(number); + return true; +} + +bool requiredInteger(const QJsonObject& parent, + const QString& name, + qint64 minimum, + qint64 maximum, + qint64* output, + const QString& path, + QString* error) +{ + if (!integerValue(parent.value(name), minimum, maximum, output)) + { + return fail(error, QStringLiteral("%1.%2 must be an integer in range").arg(path, name)); + } + return true; +} + +bool requiredFinite(const QJsonObject& parent, + const QString& name, + double* output, + const QString& path, + QString* error) +{ + auto const value = parent.value(name); + if (!value.isDouble() || !std::isfinite(value.toDouble())) + return fail(error, QStringLiteral("%1.%2 must be finite").arg(path, name)); + *output = value.toDouble(); + return true; +} + +void insertIfNotEmpty(QJsonObject& object, const QString& name, const QString& value) +{ + if (!value.isEmpty()) + object.insert(name, value); +} + +void insertIfValid(QJsonObject& object, const QString& name, const QDate& value) +{ + if (value.isValid()) + object.insert(name, value.toString(Qt::ISODate)); +} + +void insertIfValid(QJsonObject& object, const QString& name, const QUrl& value) +{ + if (!value.isEmpty()) + object.insert(name, value.toString(QUrl::FullyEncoded)); +} + +QJsonObject metadataObject(const ImageryMetadata& metadata) +{ + QJsonObject object { + { QStringLiteral("category"), categoryName(metadata.category) }, + { QStringLiteral("id"), metadata.id }, + { QStringLiteral("name"), metadata.name }, + }; + insertIfNotEmpty(object, QStringLiteral("description"), metadata.description); + insertIfValid(object, QStringLiteral("startDate"), metadata.start_date); + insertIfValid(object, QStringLiteral("endDate"), metadata.end_date); + return object; +} + +QJsonObject noticesObject(const ImageryNotices& notices) +{ + QJsonObject object; + insertIfNotEmpty(object, QStringLiteral("attributionText"), notices.attribution_text); + insertIfValid(object, QStringLiteral("attributionUrl"), notices.attribution_url); + insertIfValid(object, QStringLiteral("sourceUrl"), notices.source_url); + insertIfValid(object, QStringLiteral("termsUrl"), notices.terms_url); + insertIfValid(object, QStringLiteral("privacyUrl"), notices.privacy_url); + insertIfNotEmpty(object, QStringLiteral("notes"), notices.notes); + return object; +} + +QJsonObject requestObject(const ImageryRequestPolicy& request) +{ + QJsonObject object; + insertIfValid(object, QStringLiteral("referer"), request.referer); + auto codes = request.empty_http_status_codes; + std::sort(codes.begin(), codes.end()); + QJsonArray array; + for (auto const code : codes) + array.push_back(code); + object.insert(QStringLiteral("emptyHttpStatusCodes"), array); + return object; +} + +QJsonValue catalogProvenanceValue( + const std::optional& provenance) +{ + if (!provenance) + return QJsonValue(QJsonValue::Null); + auto const& value = *provenance; + return QJsonObject { + { QStringLiteral("catalogId"), value.catalog_id }, + { QStringLiteral("catalogRevision"), value.catalog_revision }, + { QStringLiteral("catalogSha256"), QString::fromLatin1(value.catalog_sha256) }, + { QStringLiteral("fullFingerprint"), QString::fromLatin1(value.full_fingerprint) }, + { QStringLiteral("operationalFingerprint"), QString::fromLatin1(value.operational_fingerprint) }, + { QStringLiteral("sourceId"), value.source_id }, + }; +} + +QJsonObject provenanceObject(const ImageryProvenance& provenance) +{ + QJsonObject object; + insertIfNotEmpty(object, QStringLiteral("method"), provenance.method); + insertIfValid(object, QStringLiteral("observed"), provenance.observed); + insertIfNotEmpty(object, QStringLiteral("author"), provenance.author); + if (provenance.rms_error) + object.insert(QStringLiteral("rmsError"), *provenance.rms_error); + insertIfNotEmpty(object, QStringLiteral("notes"), provenance.notes); + return object; +} + +QJsonValue registrationValue(const std::optional& registration) +{ + if (!registration) + return QJsonValue(QJsonValue::Null); + auto const& value = *registration; + QJsonObject target_frame { + { QStringLiteral("crs"), value.target_crs }, + }; + insertIfNotEmpty(target_frame, QStringLiteral("id"), value.target_frame_id); + return QJsonObject { + { QStringLiteral("direction"), QStringLiteral("source-to-corrected") }, + { QStringLiteral("operation"), QJsonObject { + { QStringLiteral("dx"), value.dx }, + { QStringLiteral("dy"), value.dy }, + { QStringLiteral("type"), QStringLiteral("translation2d") }, + { QStringLiteral("unit"), QStringLiteral("crs") }, + } }, + { QStringLiteral("provenance"), provenanceObject(value.provenance) }, + { QStringLiteral("sourceFrame"), QJsonObject { + { QStringLiteral("crs"), value.source_crs }, + } }, + { QStringLiteral("targetFrame"), target_frame }, + }; +} + +QJsonObject matrixSetObject(const TileMatrixSet& matrix_set) +{ + QJsonArray matrices; + for (auto const& matrix : matrix_set.matrices) + { + matrices.push_back(QJsonObject { + { QStringLiteral("cellSize"), matrix.cell_size }, + { QStringLiteral("id"), matrix.id }, + { QStringLiteral("matrixHeight"), double(matrix.matrix_height) }, + { QStringLiteral("matrixWidth"), double(matrix.matrix_width) }, + { QStringLiteral("pointOfOrigin"), QJsonArray { + matrix.point_of_origin.x(), matrix.point_of_origin.y() + } }, + { QStringLiteral("tileHeight"), matrix.tile_size.height() }, + { QStringLiteral("tileWidth"), matrix.tile_size.width() }, + { QStringLiteral("zoom"), matrix.zoom }, + }); + } + return { + { QStringLiteral("crs"), matrix_set.crs }, + { QStringLiteral("id"), matrix_set.id }, + { QStringLiteral("matrices"), matrices }, + }; +} + +QJsonArray limitsArray(QVector limits) +{ + std::sort(limits.begin(), limits.end(), [](auto const& first, auto const& second) { + return first.zoom < second.zoom; + }); + QJsonArray result; + for (auto const& limit : limits) + { + result.push_back(QJsonObject { + { QStringLiteral("maxColumn"), double(limit.max_column) }, + { QStringLiteral("maxRow"), double(limit.max_row) }, + { QStringLiteral("minColumn"), double(limit.min_column) }, + { QStringLiteral("minRow"), double(limit.min_row) }, + { QStringLiteral("zoom"), limit.zoom }, + }); + } + return result; +} + +QJsonObject sourceObject(const ResolvedImagerySource& source) +{ + QJsonArray tiles; + for (auto const& url : source.tile_urls) + tiles.push_back(url.value); + return { + { QStringLiteral("catalogProvenance"), catalogProvenanceValue(source.catalog_provenance) }, + { QStringLiteral("format"), ImagerySourceSnapshotCodec::formatIdentifier() }, + { QStringLiteral("limits"), limitsArray(source.tile_limits) }, + { QStringLiteral("maxZoom"), source.max_zoom }, + { QStringLiteral("mediaType"), source.media_type }, + { QStringLiteral("metadata"), metadataObject(source.metadata) }, + { QStringLiteral("minZoom"), source.min_zoom }, + { QStringLiteral("notices"), noticesObject(source.notices) }, + { QStringLiteral("registration"), registrationValue(source.registration) }, + { QStringLiteral("request"), requestObject(source.request) }, + { QStringLiteral("scheme"), tileRowSchemeName(source.row_scheme) }, + { QStringLiteral("tileMatrixSet"), matrixSetObject(source.tile_matrix_set) }, + { QStringLiteral("tiles"), tiles }, + { QStringLiteral("version"), ImagerySourceSnapshotCodec::version }, + }; +} + +QByteArray scalarJson(const QJsonValue& value) +{ + auto encoded = QJsonDocument(QJsonArray { value }).toJson(QJsonDocument::Compact); + return encoded.mid(1, encoded.size() - 2); +} + +bool appendDeterministicJson(const QJsonValue& value, QByteArray& output, QString* error) +{ + switch (value.type()) + { + case QJsonValue::Null: + output += "null"; + return true; + case QJsonValue::Bool: + output += value.toBool() ? "true" : "false"; + return true; + case QJsonValue::Double: + { + auto const number = value.toDouble(); + if (!std::isfinite(number)) + return fail(error, QStringLiteral("Snapshot contains a nonfinite number")); + if (number == 0) + output += '0'; + else + output += scalarJson(value); + return true; + } + case QJsonValue::String: + output += scalarJson(value); + return true; + case QJsonValue::Array: + { + output += '['; + auto const array = value.toArray(); + for (int index = 0; index < array.size(); ++index) + { + if (index) + output += ','; + if (!appendDeterministicJson(array.at(index), output, error)) + return false; + } + output += ']'; + return true; + } + case QJsonValue::Object: + { + output += '{'; + auto const object = value.toObject(); + auto keys = object.keys(); + std::sort(keys.begin(), keys.end()); + for (int index = 0; index < keys.size(); ++index) + { + if (index) + output += ','; + output += scalarJson(keys.at(index)); + output += ':'; + if (!appendDeterministicJson(object.value(keys.at(index)), output, error)) + return false; + } + output += '}'; + return true; + } + case QJsonValue::Undefined: + return fail(error, QStringLiteral("Snapshot contains an undefined JSON value")); + } + return fail(error, QStringLiteral("Snapshot contains an unknown JSON value")); +} + +QByteArray encodeSource(const ResolvedImagerySource& source, QString* error) +{ + QByteArray output; + if (!appendDeterministicJson(sourceObject(source), output, error)) + output.clear(); + return output; +} + +bool optionalString(const QJsonObject& object, + const QString& name, + QString* output, + const QString& path, + QString* error) +{ + auto const value = object.value(name); + if (value.isUndefined()) + return true; + if (!value.isString()) + return fail(error, QStringLiteral("%1.%2 must be a string").arg(path, name)); + *output = value.toString(); + return true; +} + +bool optionalDate(const QJsonObject& object, + const QString& name, + QDate* output, + const QString& path, + QString* error) +{ + auto const value = object.value(name); + if (value.isUndefined()) + return true; + if (!value.isString()) + return fail(error, QStringLiteral("%1.%2 must be an ISO date").arg(path, name)); + auto const date = QDate::fromString(value.toString(), Qt::ISODate); + if (!date.isValid()) + return fail(error, QStringLiteral("%1.%2 must be an ISO date").arg(path, name)); + *output = date; + return true; +} + +bool optionalUrl(const QJsonObject& object, + const QString& name, + QUrl* output, + const QString& path, + QString* error) +{ + auto const value = object.value(name); + if (value.isUndefined()) + return true; + if (!value.isString()) + return fail(error, QStringLiteral("%1.%2 must be a URL string").arg(path, name)); + *output = QUrl(value.toString(), QUrl::StrictMode); + return true; +} + +bool decodeMetadata(const QJsonObject& object, ImageryMetadata* metadata, QString* error) +{ + auto const path = QStringLiteral("$.metadata"); + if (!hasOnlyKeys(object, { + QStringLiteral("id"), QStringLiteral("name"), QStringLiteral("description"), + QStringLiteral("category"), QStringLiteral("startDate"), QStringLiteral("endDate") + }, path, error)) + { + return false; + } + QString category; + if (!requiredString(object, QStringLiteral("id"), &metadata->id, path, error) + || !requiredString(object, QStringLiteral("name"), &metadata->name, path, error) + || !requiredString(object, QStringLiteral("category"), &category, path, error) + || !optionalString(object, QStringLiteral("description"), &metadata->description, path, error) + || !optionalDate(object, QStringLiteral("startDate"), &metadata->start_date, path, error) + || !optionalDate(object, QStringLiteral("endDate"), &metadata->end_date, path, error)) + { + return false; + } + auto parsed = categoryFromName(category); + if (!parsed) + return fail(error, QStringLiteral("$.metadata.category is unsupported")); + metadata->category = *parsed; + return true; +} + +bool decodeNotices(const QJsonObject& object, ImageryNotices* notices, QString* error) +{ + auto const path = QStringLiteral("$.notices"); + if (!hasOnlyKeys(object, { + QStringLiteral("attributionText"), QStringLiteral("attributionUrl"), + QStringLiteral("sourceUrl"), QStringLiteral("termsUrl"), + QStringLiteral("privacyUrl"), QStringLiteral("notes") + }, path, error)) + { + return false; + } + return optionalString(object, QStringLiteral("attributionText"), ¬ices->attribution_text, path, error) + && optionalUrl(object, QStringLiteral("attributionUrl"), ¬ices->attribution_url, path, error) + && optionalUrl(object, QStringLiteral("sourceUrl"), ¬ices->source_url, path, error) + && optionalUrl(object, QStringLiteral("termsUrl"), ¬ices->terms_url, path, error) + && optionalUrl(object, QStringLiteral("privacyUrl"), ¬ices->privacy_url, path, error) + && optionalString(object, QStringLiteral("notes"), ¬ices->notes, path, error); +} + +bool decodeRequest(const QJsonObject& object, ImageryRequestPolicy* request, QString* error) +{ + auto const path = QStringLiteral("$.request"); + if (!hasOnlyKeys(object, { + QStringLiteral("referer"), QStringLiteral("emptyHttpStatusCodes") + }, path, error) + || !optionalUrl(object, QStringLiteral("referer"), &request->referer, path, error)) + { + return false; + } + QJsonArray codes; + if (!requiredArray(object, QStringLiteral("emptyHttpStatusCodes"), &codes, path, error) + || codes.size() > 32) + { + return false; + } + request->empty_http_status_codes.clear(); + for (int index = 0; index < codes.size(); ++index) + { + qint64 code = 0; + if (!integerValue(codes.at(index), 100, 599, &code)) + return fail(error, QStringLiteral("$.request.emptyHttpStatusCodes contains an invalid code")); + request->empty_http_status_codes.push_back(int(code)); + } + return true; +} + +bool decodeCatalogProvenance( + const QJsonValue& value, + std::optional* provenance, + QString* error) +{ + if (value.isNull()) + return true; + if (!value.isObject()) + return fail(error, QStringLiteral("$.catalogProvenance must be null or an object")); + auto const object = value.toObject(); + auto const path = QStringLiteral("$.catalogProvenance"); + if (!hasOnlyKeys(object, { + QStringLiteral("catalogId"), QStringLiteral("catalogRevision"), + QStringLiteral("catalogSha256"), QStringLiteral("sourceId"), + QStringLiteral("fullFingerprint"), QStringLiteral("operationalFingerprint") + }, path, error)) + { + return false; + } + + CatalogSourceProvenance decoded; + QString catalog_sha256; + QString full_fingerprint; + QString operational_fingerprint; + qint64 revision = 0; + if (!requiredString(object, QStringLiteral("catalogId"), &decoded.catalog_id, path, error) + || !requiredInteger(object, QStringLiteral("catalogRevision"), 1, + std::numeric_limits::max(), &revision, path, error) + || !requiredString(object, QStringLiteral("catalogSha256"), &catalog_sha256, path, error) + || !requiredString(object, QStringLiteral("sourceId"), &decoded.source_id, path, error) + || !requiredString(object, QStringLiteral("fullFingerprint"), &full_fingerprint, path, error) + || !requiredString(object, QStringLiteral("operationalFingerprint"), &operational_fingerprint, path, error)) + { + return false; + } + decoded.catalog_revision = int(revision); + decoded.catalog_sha256 = catalog_sha256.toLatin1(); + decoded.full_fingerprint = full_fingerprint.toLatin1(); + decoded.operational_fingerprint = operational_fingerprint.toLatin1(); + *provenance = std::move(decoded); + return true; +} + +bool decodeProvenance(const QJsonObject& object, ImageryProvenance* provenance, QString* error) +{ + auto const path = QStringLiteral("$.registration.provenance"); + if (!hasOnlyKeys(object, { + QStringLiteral("method"), QStringLiteral("observed"), + QStringLiteral("author"), QStringLiteral("rmsError"), QStringLiteral("notes") + }, path, error) + || !optionalString(object, QStringLiteral("method"), &provenance->method, path, error) + || !optionalDate(object, QStringLiteral("observed"), &provenance->observed, path, error) + || !optionalString(object, QStringLiteral("author"), &provenance->author, path, error) + || !optionalString(object, QStringLiteral("notes"), &provenance->notes, path, error)) + { + return false; + } + auto const rms = object.value(QStringLiteral("rmsError")); + if (!rms.isUndefined()) + { + if (!rms.isDouble() || !std::isfinite(rms.toDouble())) + return fail(error, QStringLiteral("$.registration.provenance.rmsError must be finite")); + provenance->rms_error = rms.toDouble(); + } + return true; +} + +bool decodeRegistration(const QJsonValue& value, + std::optional* registration, + QString* error) +{ + if (value.isNull()) + return true; + if (!value.isObject()) + return fail(error, QStringLiteral("$.registration must be null or an object")); + auto const object = value.toObject(); + auto const path = QStringLiteral("$.registration"); + if (!hasOnlyKeys(object, { + QStringLiteral("direction"), QStringLiteral("sourceFrame"), + QStringLiteral("targetFrame"), QStringLiteral("operation"), + QStringLiteral("provenance") + }, path, error)) + { + return false; + } + + QString direction; + QJsonObject source_frame; + QJsonObject target_frame; + QJsonObject operation; + QJsonObject provenance; + if (!requiredString(object, QStringLiteral("direction"), &direction, path, error) + || direction != QLatin1String("source-to-corrected") + || !requiredObject(object, QStringLiteral("sourceFrame"), &source_frame, path, error) + || !requiredObject(object, QStringLiteral("targetFrame"), &target_frame, path, error) + || !requiredObject(object, QStringLiteral("operation"), &operation, path, error) + || !requiredObject(object, QStringLiteral("provenance"), &provenance, path, error)) + { + if (direction != QLatin1String("source-to-corrected") && error) + *error = QStringLiteral("$.registration.direction is unsupported"); + return false; + } + if (!hasOnlyKeys(source_frame, { QStringLiteral("crs") }, + QStringLiteral("$.registration.sourceFrame"), error) + || !hasOnlyKeys(target_frame, { QStringLiteral("crs"), QStringLiteral("id") }, + QStringLiteral("$.registration.targetFrame"), error)) + { + return false; + } + + QString type; + QString unit; + TranslationRegistration decoded; + if (!requiredString(operation, QStringLiteral("type"), &type, + QStringLiteral("$.registration.operation"), error)) + { + return false; + } + if (type != QLatin1String("translation2d")) + { + if (type == QLatin1String("affine2d") || type == QLatin1String("gridShift")) + return fail(error, QStringLiteral("Resolved runtime does not support %1 registration").arg(type)); + return fail(error, QStringLiteral("Resolved runtime registration operation is unknown")); + } + if (!hasOnlyKeys(operation, { + QStringLiteral("type"), QStringLiteral("unit"), + QStringLiteral("dx"), QStringLiteral("dy") + }, QStringLiteral("$.registration.operation"), error)) + { + return false; + } + if (!requiredString(operation, QStringLiteral("unit"), &unit, + QStringLiteral("$.registration.operation"), error) + || unit != QLatin1String("crs") + || !requiredFinite(operation, QStringLiteral("dx"), &decoded.dx, + QStringLiteral("$.registration.operation"), error) + || !requiredFinite(operation, QStringLiteral("dy"), &decoded.dy, + QStringLiteral("$.registration.operation"), error) + || !requiredString(source_frame, QStringLiteral("crs"), &decoded.source_crs, + QStringLiteral("$.registration.sourceFrame"), error) + || !requiredString(target_frame, QStringLiteral("crs"), &decoded.target_crs, + QStringLiteral("$.registration.targetFrame"), error) + || !optionalString(target_frame, QStringLiteral("id"), &decoded.target_frame_id, + QStringLiteral("$.registration.targetFrame"), error) + || !decodeProvenance(provenance, &decoded.provenance, error)) + { + if (unit != QLatin1String("crs") && error) + *error = QStringLiteral("$.registration.operation.unit is unsupported"); + return false; + } + *registration = std::move(decoded); + return true; +} + +bool decodeMatrixSet(const QJsonObject& object, TileMatrixSet* matrix_set, QString* error) +{ + auto const path = QStringLiteral("$.tileMatrixSet"); + if (!hasOnlyKeys(object, { + QStringLiteral("id"), QStringLiteral("crs"), QStringLiteral("matrices") + }, path, error) + || !requiredString(object, QStringLiteral("id"), &matrix_set->id, path, error) + || !requiredString(object, QStringLiteral("crs"), &matrix_set->crs, path, error)) + { + return false; + } + QJsonArray matrices; + if (!requiredArray(object, QStringLiteral("matrices"), &matrices, path, error) + || matrices.isEmpty() || matrices.size() > 63) + { + return fail(error, QStringLiteral("$.tileMatrixSet.matrices has an invalid size")); + } + for (int index = 0; index < matrices.size(); ++index) + { + if (!matrices.at(index).isObject()) + return fail(error, QStringLiteral("$.tileMatrixSet.matrices contains a non-object")); + auto const matrix_object = matrices.at(index).toObject(); + auto const matrix_path = QStringLiteral("$.tileMatrixSet.matrices[%1]").arg(index); + if (!hasOnlyKeys(matrix_object, { + QStringLiteral("id"), QStringLiteral("zoom"), QStringLiteral("cellSize"), + QStringLiteral("pointOfOrigin"), QStringLiteral("tileWidth"), + QStringLiteral("tileHeight"), QStringLiteral("matrixWidth"), + QStringLiteral("matrixHeight") + }, matrix_path, error)) + { + return false; + } + TileMatrix matrix; + qint64 zoom = 0; + qint64 tile_width = 0; + qint64 tile_height = 0; + if (!requiredString(matrix_object, QStringLiteral("id"), &matrix.id, matrix_path, error) + || !requiredInteger(matrix_object, QStringLiteral("zoom"), 0, 62, &zoom, matrix_path, error) + || !requiredFinite(matrix_object, QStringLiteral("cellSize"), &matrix.cell_size, matrix_path, error) + || !requiredInteger(matrix_object, QStringLiteral("tileWidth"), 1, + std::numeric_limits::max(), &tile_width, matrix_path, error) + || !requiredInteger(matrix_object, QStringLiteral("tileHeight"), 1, + std::numeric_limits::max(), &tile_height, matrix_path, error) + || !requiredInteger(matrix_object, QStringLiteral("matrixWidth"), 1, + 9007199254740991LL, &matrix.matrix_width, matrix_path, error) + || !requiredInteger(matrix_object, QStringLiteral("matrixHeight"), 1, + 9007199254740991LL, &matrix.matrix_height, matrix_path, error)) + { + return false; + } + auto const origin = matrix_object.value(QStringLiteral("pointOfOrigin")); + if (!origin.isArray() || origin.toArray().size() != 2 + || !origin.toArray().at(0).isDouble() + || !origin.toArray().at(1).isDouble() + || !std::isfinite(origin.toArray().at(0).toDouble()) + || !std::isfinite(origin.toArray().at(1).toDouble())) + { + return fail(error, QStringLiteral("%1.pointOfOrigin must contain two finite numbers").arg(matrix_path)); + } + matrix.zoom = int(zoom); + matrix.tile_size = QSize(int(tile_width), int(tile_height)); + matrix.point_of_origin = QPointF( + origin.toArray().at(0).toDouble(), + origin.toArray().at(1).toDouble() + ); + matrix_set->matrices.push_back(std::move(matrix)); + } + return true; +} + +bool decodeLimits(const QJsonArray& array, + QVector* limits, + QString* error) +{ + if (array.size() > 63) + return fail(error, QStringLiteral("$.limits exceeds the supported zoom count")); + for (int index = 0; index < array.size(); ++index) + { + if (!array.at(index).isObject()) + return fail(error, QStringLiteral("$.limits contains a non-object")); + auto const object = array.at(index).toObject(); + auto const path = QStringLiteral("$.limits[%1]").arg(index); + if (!hasOnlyKeys(object, { + QStringLiteral("zoom"), QStringLiteral("minColumn"), + QStringLiteral("maxColumn"), QStringLiteral("minRow"), + QStringLiteral("maxRow") + }, path, error)) + { + return false; + } + TileMatrixLimits limit; + qint64 zoom = 0; + if (!requiredInteger(object, QStringLiteral("zoom"), 0, 62, &zoom, path, error) + || !requiredInteger(object, QStringLiteral("minColumn"), 0, 9007199254740991LL, + &limit.min_column, path, error) + || !requiredInteger(object, QStringLiteral("maxColumn"), 0, 9007199254740991LL, + &limit.max_column, path, error) + || !requiredInteger(object, QStringLiteral("minRow"), 0, 9007199254740991LL, + &limit.min_row, path, error) + || !requiredInteger(object, QStringLiteral("maxRow"), 0, 9007199254740991LL, + &limit.max_row, path, error)) + { + return false; + } + limit.zoom = int(zoom); + limits->push_back(limit); + } + return true; +} + +bool decodeSourceObject(const QJsonObject& object, + ResolvedImagerySource* source, + QString* error) +{ + if (!hasOnlyKeys(object, { + QStringLiteral("format"), QStringLiteral("version"), QStringLiteral("catalogProvenance"), + QStringLiteral("metadata"), QStringLiteral("notices"), + QStringLiteral("tiles"), QStringLiteral("scheme"), + QStringLiteral("mediaType"), QStringLiteral("tileMatrixSet"), + QStringLiteral("minZoom"), QStringLiteral("maxZoom"), + QStringLiteral("limits"), QStringLiteral("request"), + QStringLiteral("registration") + }, QStringLiteral("$"), error)) + { + return false; + } + QString format; + QString scheme; + QJsonObject metadata; + QJsonObject notices; + QJsonObject matrix_set; + QJsonObject request; + QJsonArray tiles; + QJsonArray limits; + qint64 version = 0; + qint64 min_zoom = 0; + qint64 max_zoom = 0; + if (!requiredString(object, QStringLiteral("format"), &format, QStringLiteral("$"), error) + || format != ImagerySourceSnapshotCodec::formatIdentifier() + || !requiredInteger(object, QStringLiteral("version"), 1, 1, &version, + QStringLiteral("$"), error) + || !requiredObject(object, QStringLiteral("metadata"), &metadata, QStringLiteral("$"), error) + || !requiredObject(object, QStringLiteral("notices"), ¬ices, QStringLiteral("$"), error) + || !requiredArray(object, QStringLiteral("tiles"), &tiles, QStringLiteral("$"), error) + || !requiredString(object, QStringLiteral("scheme"), &scheme, QStringLiteral("$"), error) + || !requiredString(object, QStringLiteral("mediaType"), &source->media_type, + QStringLiteral("$"), error) + || !requiredObject(object, QStringLiteral("tileMatrixSet"), &matrix_set, + QStringLiteral("$"), error) + || !requiredInteger(object, QStringLiteral("minZoom"), 0, 62, &min_zoom, + QStringLiteral("$"), error) + || !requiredInteger(object, QStringLiteral("maxZoom"), 0, 62, &max_zoom, + QStringLiteral("$"), error) + || !requiredArray(object, QStringLiteral("limits"), &limits, QStringLiteral("$"), error) + || !requiredObject(object, QStringLiteral("request"), &request, QStringLiteral("$"), error)) + { + if (format != ImagerySourceSnapshotCodec::formatIdentifier() && error) + *error = QStringLiteral("Snapshot format identifier is unsupported"); + return false; + } + auto parsed_scheme = tileRowSchemeFromName(scheme); + if (!parsed_scheme) + return fail(error, QStringLiteral("$.scheme is unsupported")); + source->row_scheme = *parsed_scheme; + source->min_zoom = int(min_zoom); + source->max_zoom = int(max_zoom); + + if (tiles.isEmpty() || tiles.size() > 8) + return fail(error, QStringLiteral("$.tiles has an invalid size")); + for (auto const& value : tiles) + { + if (!value.isString()) + return fail(error, QStringLiteral("$.tiles contains a non-string")); + source->tile_urls.push_back({ value.toString() }); + } + + return decodeMetadata(metadata, &source->metadata, error) + && decodeNotices(notices, &source->notices, error) + && decodeMatrixSet(matrix_set, &source->tile_matrix_set, error) + && decodeLimits(limits, &source->tile_limits, error) + && decodeRequest(request, &source->request, error) + && decodeCatalogProvenance(object.value(QStringLiteral("catalogProvenance")), + &source->catalog_provenance, error) + && decodeRegistration(object.value(QStringLiteral("registration")), + &source->registration, error); +} + +QByteArray fingerprint(const QByteArray& bytes) +{ + return QCryptographicHash::hash(bytes, QCryptographicHash::Sha256).toHex(); +} + +} // namespace + +QString ImagerySourceSnapshotCodec::formatIdentifier() +{ + return QStringLiteral("org.openorienteering.imagery-source-snapshot"); +} + +std::optional ImagerySourceSnapshotCodec::encode( + const ResolvedImagerySource& source, + QString* error) +{ + if (!source.validate(error)) + return std::nullopt; + auto json = encodeSource(source, error); + if (json.isEmpty()) + return std::nullopt; + if (json.size() > maximum_size) + return failOptional(error, QStringLiteral("Resolved imagery snapshot exceeds the size limit")); + if (error) + error->clear(); + return ImagerySourceSnapshot { + source, + json, + fingerprint(json), + }; +} + +std::optional ImagerySourceSnapshotCodec::decode( + const QByteArray& json, + QString* error) +{ + if (json.isEmpty() || json.size() > maximum_size) + return failOptional(error, QStringLiteral("Resolved imagery snapshot has an invalid size")); + + QJsonParseError parse_error; + auto const document = QJsonDocument::fromJson(json, &parse_error); + if (parse_error.error != QJsonParseError::NoError || !document.isObject()) + { + return failOptional( + error, + QStringLiteral("Resolved imagery snapshot is invalid JSON: %1") + .arg(parse_error.errorString()) + ); + } + + ResolvedImagerySource source; + if (!decodeSourceObject(document.object(), &source, error) + || !source.validate(error)) + { + return std::nullopt; + } + auto canonical = encodeSource(source, error); + if (canonical.isEmpty()) + return std::nullopt; + if (error) + error->clear(); + return ImagerySourceSnapshot { + std::move(source), + canonical, + fingerprint(canonical), + }; +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/imagery_source_snapshot.h b/src/imagery/imagery_source_snapshot.h new file mode 100644 index 000000000..367f99ae8 --- /dev/null +++ b/src/imagery/imagery_source_snapshot.h @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_SOURCE_SNAPSHOT_H +#define OPENORIENTEERING_IMAGERY_SOURCE_SNAPSHOT_H + +#include + +#include +#include + +#include "imagery/imagery_source.h" + +namespace OpenOrienteering::imagery { + +struct ImagerySourceSnapshot +{ + ResolvedImagerySource source; + QByteArray canonical_json; + QByteArray sha256; +}; + +/** + * Deterministic JSON persistence for one fully resolved runtime source. + * + * The fingerprint is the lowercase SHA-256 digest of canonical_json. Decoding + * re-encodes accepted input so formatting differences cannot alter identity. + */ +class ImagerySourceSnapshotCodec +{ +public: + static constexpr int version = 1; + static constexpr qsizetype maximum_size = 1024 * 1024; + + static QString formatIdentifier(); + static std::optional encode( + const ResolvedImagerySource& source, + QString* error = nullptr + ); + static std::optional decode( + const QByteArray& json, + QString* error = nullptr + ); +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/tile_matrix_set.cpp b/src/imagery/tile_matrix_set.cpp new file mode 100644 index 000000000..b8f8ed579 --- /dev/null +++ b/src/imagery/tile_matrix_set.cpp @@ -0,0 +1,209 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "imagery/tile_matrix_set.h" + +#include +#include +#include + +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +bool fail(QString* error, const QString& message) +{ + if (error) + *error = message; + return false; +} + +bool finite(double value) +{ + return std::isfinite(value); +} + +bool nearlyEqual(double first, double second) +{ + auto const scale = std::max({ 1.0, std::abs(first), std::abs(second) }); + return std::abs(first - second) <= scale * 1.0e-10; +} + +bool normalizedEpsg(const QString& crs) +{ + static const QRegularExpression pattern( + QStringLiteral("^EPSG:[1-9][0-9]{0,8}$") + ); + return pattern.match(crs).hasMatch(); +} + +} // namespace + +bool CrsBounds::isValid() const noexcept +{ + return finite(west) && finite(south) && finite(east) && finite(north) + && west < east && south < north; +} + +bool TileMatrix::contains(qint64 column, qint64 row) const noexcept +{ + return column >= 0 && row >= 0 + && column < matrix_width && row < matrix_height; +} + +CrsBounds TileMatrix::tileBounds(qint64 column, qint64 row) const noexcept +{ + if (!contains(column, row) || !finite(cell_size) || cell_size <= 0 + || tile_size.width() <= 0 || tile_size.height() <= 0) + { + return {}; + } + + auto const tile_width = cell_size * double(tile_size.width()); + auto const tile_height = cell_size * double(tile_size.height()); + auto const west = point_of_origin.x() + double(column) * tile_width; + auto const north = point_of_origin.y() - double(row) * tile_height; + return { west, north - tile_height, west + tile_width, north }; +} + +bool TileMatrixLimits::contains(qint64 column, qint64 row) const noexcept +{ + return column >= min_column && column <= max_column + && row >= min_row && row <= max_row; +} + +const TileMatrix* TileMatrixSet::matrixForZoom(int zoom) const noexcept +{ + if (zoom >= 0 && zoom < matrices.size() && matrices.at(zoom).zoom == zoom) + return &matrices.at(zoom); + for (auto const& matrix : matrices) + { + if (matrix.zoom == zoom) + return &matrix; + } + return nullptr; +} + +bool TileMatrixSet::validateDyadicTopLeft(QString* error) const +{ + if (id.trimmed().isEmpty()) + return fail(error, QStringLiteral("Tile matrix set ID is empty")); + if (!normalizedEpsg(crs)) + return fail(error, QStringLiteral("Tile matrix set CRS must be a normalized EPSG code")); + if (matrices.isEmpty()) + return fail(error, QStringLiteral("Tile matrix set has no matrices")); + if (matrices.size() > 63) + return fail(error, QStringLiteral("Tile matrix set exceeds the supported zoom count")); + + auto const& first = matrices.first(); + if (first.zoom != 0 || first.id != QLatin1String("0")) + return fail(error, QStringLiteral("A dyadic tile matrix set must begin at zoom 0")); + if (!finite(first.cell_size) || first.cell_size <= 0) + return fail(error, QStringLiteral("Tile matrix cell size must be finite and positive")); + if (!finite(first.point_of_origin.x()) || !finite(first.point_of_origin.y())) + return fail(error, QStringLiteral("Tile matrix origin must be finite")); + if (first.tile_size.width() <= 0 || first.tile_size.height() <= 0) + return fail(error, QStringLiteral("Tile dimensions must be positive")); + if (first.matrix_width <= 0 || first.matrix_height <= 0) + return fail(error, QStringLiteral("Tile matrix dimensions must be positive")); + + for (int index = 0; index < matrices.size(); ++index) + { + auto const& matrix = matrices.at(index); + if (matrix.zoom != index || matrix.id != QString::number(index)) + return fail(error, QStringLiteral("Dyadic matrix IDs must be contiguous decimal zooms")); + if (!finite(matrix.cell_size) || matrix.cell_size <= 0 + || !finite(matrix.point_of_origin.x()) || !finite(matrix.point_of_origin.y())) + { + return fail(error, QStringLiteral("Tile matrix geometry must be finite and positive")); + } + if (matrix.tile_size != first.tile_size) + return fail(error, QStringLiteral("Dyadic matrices must use one tile size")); + if (!nearlyEqual(matrix.point_of_origin.x(), first.point_of_origin.x()) + || !nearlyEqual(matrix.point_of_origin.y(), first.point_of_origin.y())) + { + return fail(error, QStringLiteral("Dyadic matrices must use one top-left origin")); + } + if (matrix.matrix_width <= 0 || matrix.matrix_height <= 0) + return fail(error, QStringLiteral("Tile matrix dimensions must be positive")); + if (index == 0) + continue; + + auto const& previous = matrices.at(index - 1); + if (!nearlyEqual(matrix.cell_size * 2, previous.cell_size)) + return fail(error, QStringLiteral("Each matrix cell size must halve at the next zoom")); + if (previous.matrix_width > std::numeric_limits::max() / 2 + || previous.matrix_height > std::numeric_limits::max() / 2 + || matrix.matrix_width != previous.matrix_width * 2 + || matrix.matrix_height != previous.matrix_height * 2) + { + return fail(error, QStringLiteral("Each matrix dimension must double at the next zoom")); + } + } + + if (error) + error->clear(); + return true; +} + +TileMatrixSet TileMatrixSet::webMercatorQuad() +{ + constexpr auto max_zoom = 24; + constexpr auto half_world = 20037508.342789244; + constexpr auto tile_pixels = 256; + constexpr auto base_cell_size = (2 * half_world) / tile_pixels; + + TileMatrixSet result; + result.id = QStringLiteral("WebMercatorQuad"); + result.crs = QStringLiteral("EPSG:3857"); + result.matrices.reserve(max_zoom + 1); + for (int zoom = 0; zoom <= max_zoom; ++zoom) + { + auto const dimension = qint64(1) << zoom; + result.matrices.push_back({ + QString::number(zoom), + zoom, + base_cell_size / double(dimension), + QPointF(-half_world, half_world), + QSize(tile_pixels, tile_pixels), + dimension, + dimension, + }); + } + return result; +} + +bool validateTileMatrixLimits(const QVector& limits, + const TileMatrixSet& matrix_set, + QString* error) +{ + QVector seen_zooms; + seen_zooms.reserve(limits.size()); + for (auto const& limit : limits) + { + auto const* matrix = matrix_set.matrixForZoom(limit.zoom); + if (!matrix) + return fail(error, QStringLiteral("Tile limits refer to an unknown zoom")); + if (seen_zooms.contains(limit.zoom)) + return fail(error, QStringLiteral("Tile limits contain a duplicate zoom")); + seen_zooms.push_back(limit.zoom); + if (limit.min_column < 0 || limit.min_row < 0 + || limit.min_column > limit.max_column || limit.min_row > limit.max_row + || limit.max_column >= matrix->matrix_width + || limit.max_row >= matrix->matrix_height) + { + return fail(error, QStringLiteral("Tile limits fall outside their matrix")); + } + } + if (error) + error->clear(); + return true; +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/tile_matrix_set.h b/src/imagery/tile_matrix_set.h new file mode 100644 index 000000000..10faa3b73 --- /dev/null +++ b/src/imagery/tile_matrix_set.h @@ -0,0 +1,86 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_TILE_MATRIX_SET_H +#define OPENORIENTEERING_IMAGERY_TILE_MATRIX_SET_H + +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +struct CrsBounds +{ + double west = 0; + double south = 0; + double east = 0; + double north = 0; + + bool isValid() const noexcept; + bool operator==(const CrsBounds&) const = default; +}; + +/** + * One top-left-origin matrix in a dyadic tile pyramid. + * + * Rows are always represented in canonical top-to-bottom order. A source's + * XYZ/TMS row convention is applied only when expanding a request URL. + */ +struct TileMatrix +{ + QString id; + int zoom = -1; + double cell_size = 0; + QPointF point_of_origin; + QSize tile_size; + qint64 matrix_width = 0; + qint64 matrix_height = 0; + + bool contains(qint64 column, qint64 row) const noexcept; + CrsBounds tileBounds(qint64 column, qint64 row) const noexcept; + bool operator==(const TileMatrix&) const = default; +}; + +struct TileMatrixLimits +{ + int zoom = -1; + qint64 min_column = 0; + qint64 max_column = -1; + qint64 min_row = 0; + qint64 max_row = -1; + + bool contains(qint64 column, qint64 row) const noexcept; + bool operator==(const TileMatrixLimits&) const = default; +}; + +struct TileMatrixSet +{ + QString id; + QString crs; + QVector matrices; + + const TileMatrix* matrixForZoom(int zoom) const noexcept; + bool validateDyadicTopLeft(QString* error = nullptr) const; + + static TileMatrixSet webMercatorQuad(); + + bool operator==(const TileMatrixSet&) const = default; +}; + +bool validateTileMatrixLimits(const QVector& limits, + const TileMatrixSet& matrix_set, + QString* error = nullptr); + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6ce5da5d9..45a837605 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -187,6 +187,10 @@ add_unit_test(util_t ../src/util/util ../src/settings ) +# Low-level imagery contracts stay independent of the full Mapper runtime. +add_test_helper(imagery_core_t) +target_link_libraries(imagery_core_t PRIVATE Mapper::ImageryCore) + # Benchmarks add_system_test(coord_xml_t MANUAL) diff --git a/test/imagery_core_t.cpp b/test/imagery_core_t.cpp new file mode 100644 index 000000000..50dae3ca7 --- /dev/null +++ b/test/imagery_core_t.cpp @@ -0,0 +1,243 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "imagery_core_t.h" + +#include + +#include +#include +#include + +#include "imagery/imagery_source.h" +#include "imagery/imagery_source_snapshot.h" +#include "imagery/tile_matrix_set.h" + +using namespace OpenOrienteering; + +namespace { + +imagery::ResolvedImagerySource sourceFixture() +{ + imagery::ResolvedImagerySource source; + source.metadata.id = QStringLiteral("org.example.aerial-2026"); + source.metadata.name = QStringLiteral("Example Aerial 2026"); + source.metadata.description = QStringLiteral("Synthetic source for imagery core tests"); + source.metadata.category = imagery::ImageryCategory::Aerial; + source.metadata.start_date = QDate(2026, 1, 1); + source.metadata.end_date = QDate(2026, 6, 30); + source.notices.attribution_text = QStringLiteral("Example Mapping Club"); + source.notices.attribution_url = QUrl(QStringLiteral("https://example.test/attribution")); + source.notices.source_url = QUrl(QStringLiteral("https://example.test/source")); + source.notices.terms_url = QUrl(QStringLiteral("https://example.test/terms")); + source.tile_urls = { + { QStringLiteral("https://tiles-a.example.test/aerial/{z}/{x}/{y}.png?style=base") }, + { QStringLiteral("https://tiles-b.example.test/aerial/{z}/{x}/{y}.png?style=base") }, + }; + source.row_scheme = imagery::TileRowScheme::Xyz; + source.media_type = QStringLiteral("image/png"); + source.tile_matrix_set = imagery::TileMatrixSet::webMercatorQuad(); + source.min_zoom = 0; + source.max_zoom = 20; + source.tile_limits = { + { 3, 1, 6, 0, 7 }, + }; + source.request.referer = QUrl(QStringLiteral("https://example.test/map/")); + source.request.empty_http_status_codes = { 204, 404 }; + source.catalog_provenance = imagery::CatalogSourceProvenance { + QStringLiteral("org.example.imagery"), + 7, + QByteArray(64, 'a'), + source.metadata.id, + QByteArray(64, 'b'), + QByteArray(64, 'c'), + }; + source.registration = imagery::TranslationRegistration { + QStringLiteral("EPSG:3857"), + QStringLiteral("EPSG:3857"), + QStringLiteral("org.example.survey-frame-2026"), + -0.42, + 0.17, + { + QStringLiteral("survey-control"), + QDate(2026, 6, 20), + QStringLiteral("Example Mapping Team"), + 0.12, + QStringLiteral("Fit from six synthetic control points"), + }, + }; + return source; +} + +} // namespace + +void ImageryCoreTest::webMercatorQuadIsDyadic() +{ + auto const matrix_set = imagery::TileMatrixSet::webMercatorQuad(); + QString error; + QVERIFY2(matrix_set.validateDyadicTopLeft(&error), qPrintable(error)); + QCOMPARE(matrix_set.id, QStringLiteral("WebMercatorQuad")); + QCOMPARE(matrix_set.crs, QStringLiteral("EPSG:3857")); + QCOMPARE(matrix_set.matrices.size(), 25); + + auto const* zoom_zero = matrix_set.matrixForZoom(0); + auto const* zoom_24 = matrix_set.matrixForZoom(24); + QVERIFY(zoom_zero); + QVERIFY(zoom_24); + QCOMPARE(zoom_zero->tile_size, QSize(256, 256)); + QCOMPARE(zoom_zero->matrix_width, qint64(1)); + QCOMPARE(zoom_24->matrix_width, qint64(1) << 24); + QVERIFY(std::abs(zoom_zero->cell_size / zoom_24->cell_size - double(qint64(1) << 24)) < 1.0e-6); + + auto const bounds = zoom_zero->tileBounds(0, 0); + QVERIFY(bounds.isValid()); + constexpr auto half_world = 20037508.342789244; + QVERIFY(std::abs(bounds.west + half_world) < 1.0e-6); + QVERIFY(std::abs(bounds.south + half_world) < 1.0e-6); + QVERIFY(std::abs(bounds.east - half_world) < 1.0e-6); + QVERIFY(std::abs(bounds.north - half_world) < 1.0e-6); +} + +void ImageryCoreTest::expandsXyzAndTmsRows() +{ + auto source = sourceFixture(); + QString error; + QVERIFY2(source.validate(&error), qPrintable(error)); + + auto xyz = source.tileUrl(0, 3, 2, 1, &error); + QVERIFY2(xyz.isValid(), qPrintable(error)); + QCOMPARE( + xyz.toString(QUrl::FullyEncoded), + QStringLiteral("https://tiles-a.example.test/aerial/3/2/1.png?style=base") + ); + + source.row_scheme = imagery::TileRowScheme::Tms; + auto tms = source.tileUrl(1, 3, 2, 1, &error); + QVERIFY2(tms.isValid(), qPrintable(error)); + QCOMPARE( + tms.toString(QUrl::FullyEncoded), + QStringLiteral("https://tiles-b.example.test/aerial/3/2/6.png?style=base") + ); + + QVERIFY(source.tileUrl(0, 3, 0, 1, &error).isEmpty()); + QVERIFY(error.contains(QStringLiteral("limits"))); + QVERIFY(source.tileUrl(0, 21, 2, 1, &error).isEmpty()); + QVERIFY(error.contains(QStringLiteral("usable range"))); +} + +void ImageryCoreTest::rejectsUnsafeUrlTemplates() +{ + auto accepts = [](const QString& text) { + QString error; + return imagery::TileUrlTemplate { text }.validate(&error); + }; + + QVERIFY(accepts(QStringLiteral("https://tiles.example.test/{z}/{x}/{y}.png"))); + QVERIFY(accepts(QStringLiteral("http://tiles.example.test/tiles?z={z}&x={x}&y={y}"))); + QVERIFY(!accepts(QStringLiteral("ftp://tiles.example.test/{z}/{x}/{y}.png"))); + QVERIFY(!accepts(QStringLiteral("https://user:secret@tiles.example.test/{z}/{x}/{y}.png"))); + QVERIFY(!accepts(QStringLiteral("https://tiles.example.test/{z}/{x}.png"))); + QVERIFY(!accepts(QStringLiteral("https://tiles.example.test/{z}/{x}/{y}/{y}.png"))); + QVERIFY(!accepts(QStringLiteral("https://tiles.example.test/${z}/${x}/${y}.png"))); + QVERIFY(!accepts(QStringLiteral("https://tiles.example.test/{z}/{x}/{y}/{s}.png"))); + QVERIFY(!accepts(QStringLiteral("https://{z}.example.test/tiles/{x}/{y}.png"))); + QVERIFY(!accepts(QStringLiteral("https://tiles.example.test/{z}/{x}/{y}.png#fragment"))); + QVERIFY(!accepts(QStringLiteral("https://tiles.example.test/{z}/{x}/{y}.png\nInjected: value"))); +} + +void ImageryCoreTest::validatesTileLimits() +{ + auto source = sourceFixture(); + QString error; + QVERIFY2(source.validate(&error), qPrintable(error)); + + source.tile_limits.push_back({ 3, 0, 1, 0, 1 }); + QVERIFY(!source.validate(&error)); + QVERIFY(error.contains(QStringLiteral("duplicate zoom"))); + + source = sourceFixture(); + source.tile_limits = { { 3, 0, 8, 0, 7 } }; + QVERIFY(!source.validate(&error)); + QVERIFY(error.contains(QStringLiteral("outside"))); + + source = sourceFixture(); + source.tile_matrix_set.matrices[4].cell_size *= 0.9; + QVERIFY(!source.validate(&error)); + QVERIFY(error.contains(QStringLiteral("halve"))); +} + +void ImageryCoreTest::requestPolicyHasExplicitDefaults() +{ + imagery::ImageryRequestPolicy policy; + QCOMPARE(policy.empty_http_status_codes, QVector({ 204, 404 })); + + auto source = sourceFixture(); + QString error; + source.catalog_provenance->catalog_sha256 = QByteArray(64, 'A'); + QVERIFY(!source.validate(&error)); + QVERIFY(error.contains(QStringLiteral("lowercase SHA-256"))); +} + +void ImageryCoreTest::snapshotRoundTripsDeterministically() +{ + auto const source = sourceFixture(); + QString error; + auto const first = imagery::ImagerySourceSnapshotCodec::encode(source, &error); + QVERIFY2(first, qPrintable(error)); + auto const second = imagery::ImagerySourceSnapshotCodec::encode(source, &error); + QVERIFY2(second, qPrintable(error)); + QCOMPARE(first->canonical_json, second->canonical_json); + QCOMPARE(first->sha256, second->sha256); + QCOMPARE(first->sha256.size(), 64); + QCOMPARE( + first->sha256, + QCryptographicHash::hash( + first->canonical_json, QCryptographicHash::Sha256 + ).toHex() + ); + QVERIFY(!first->canonical_json.contains('\n')); + QVERIFY(QJsonDocument::fromJson(first->canonical_json).isObject()); + + auto const decoded = imagery::ImagerySourceSnapshotCodec::decode( + first->canonical_json, &error + ); + QVERIFY2(decoded, qPrintable(error)); + QCOMPARE(decoded->canonical_json, first->canonical_json); + QCOMPARE(decoded->sha256, first->sha256); + QCOMPARE(decoded->source, source); + QVERIFY(decoded->source.catalog_provenance); + QCOMPARE(decoded->source.catalog_provenance->catalog_revision, 7); + QCOMPARE(decoded->source.catalog_provenance->operational_fingerprint, QByteArray(64, 'c')); + + auto const round_trip = imagery::ImagerySourceSnapshotCodec::encode( + decoded->source, &error + ); + QVERIFY2(round_trip, qPrintable(error)); + QCOMPARE(round_trip->canonical_json, first->canonical_json); +} + +void ImageryCoreTest::snapshotRejectsUnsupportedRegistrations() +{ + QString error; + auto const encoded = imagery::ImagerySourceSnapshotCodec::encode( + sourceFixture(), &error + ); + QVERIFY2(encoded, qPrintable(error)); + + auto affine = encoded->canonical_json; + QVERIFY(affine.contains("\"translation2d\"")); + affine.replace("\"translation2d\"", "\"affine2d\""); + QVERIFY(!imagery::ImagerySourceSnapshotCodec::decode(affine, &error)); + QVERIFY(error.contains(QStringLiteral("does not support affine2d"))); + + auto grid = encoded->canonical_json; + QVERIFY(grid.contains("\"translation2d\"")); + grid.replace("\"translation2d\"", "\"gridShift\""); + QVERIFY(!imagery::ImagerySourceSnapshotCodec::decode(grid, &error)); + QVERIFY(error.contains(QStringLiteral("does not support gridShift"))); +} + +QTEST_MAIN(ImageryCoreTest) diff --git a/test/imagery_core_t.h b/test/imagery_core_t.h new file mode 100644 index 000000000..817df41ed --- /dev/null +++ b/test/imagery_core_t.h @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#ifndef OPENORIENTEERING_IMAGERY_CORE_T_H +#define OPENORIENTEERING_IMAGERY_CORE_T_H + +#include + +class ImageryCoreTest : public QObject +{ +Q_OBJECT + +private slots: + void webMercatorQuadIsDyadic(); + void expandsXyzAndTmsRows(); + void rejectsUnsafeUrlTemplates(); + void validatesTileLimits(); + void requestPolicyHasExplicitDefaults(); + void snapshotRoundTripsDeterministically(); + void snapshotRejectsUnsupportedRegistrations(); +}; + +#endif From c651eb4feb9455ac0e670bf8d984a73c8ea9c962 Mon Sep 17 00:00:00 2001 From: Ethan O'Connor Date: Thu, 16 Jul 2026 12:02:44 -0700 Subject: [PATCH 05/39] imagery: add bounded shared tile transport --- src/CMakeLists.txt | 1 + src/imagery/CMakeLists.txt | 17 + src/imagery/tile_network_manager.cpp | 1139 ++++++++++++++++++++++++++ src/imagery/tile_network_manager.h | 166 ++++ test/CMakeLists.txt | 2 + test/tile_network_manager_t.cpp | 420 ++++++++++ test/tile_network_manager_t.h | 26 + 7 files changed, 1771 insertions(+) create mode 100644 src/imagery/tile_network_manager.cpp create mode 100644 src/imagery/tile_network_manager.h create mode 100644 test/tile_network_manager_t.cpp create mode 100644 test/tile_network_manager_t.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 415db9887..e5bb2ff49 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -351,6 +351,7 @@ target_link_libraries(Mapper_Common PUBLIC Clipper2::Clipper2 Mapper::ImageryCore + Mapper::ImageryNetwork Mapper::RenderIR ${PROJ_LIBRARIES} Qt6::Widgets diff --git a/src/imagery/CMakeLists.txt b/src/imagery/CMakeLists.txt index 9476d63a2..d84ca2e70 100644 --- a/src/imagery/CMakeLists.txt +++ b/src/imagery/CMakeLists.txt @@ -30,3 +30,20 @@ set_target_properties(mapper-imagery-core PROPERTIES ) mapper_translations_sources(${MAPPER_IMAGERY_CORE_SOURCES}) + +add_library(mapper-imagery-network STATIC + tile_network_manager.cpp + tile_network_manager.h +) +add_library(Mapper::ImageryNetwork ALIAS mapper-imagery-network) +mapper_target_defaults(mapper-imagery-network) +target_include_directories(mapper-imagery-network PUBLIC "${PROJECT_SOURCE_DIR}/src") +target_link_libraries(mapper-imagery-network PUBLIC Qt6::Core Qt6::Network) +target_compile_definitions(mapper-imagery-network PRIVATE + QT_NO_CAST_FROM_ASCII + QT_NO_CAST_TO_ASCII + QT_USE_QSTRINGBUILDER +) +set_target_properties(mapper-imagery-network PROPERTIES PREFIX "") + +mapper_translations_sources(tile_network_manager.cpp tile_network_manager.h) diff --git a/src/imagery/tile_network_manager.cpp b/src/imagery/tile_network_manager.cpp new file mode 100644 index 000000000..4ebfb0e21 --- /dev/null +++ b/src/imagery/tile_network_manager.cpp @@ -0,0 +1,1139 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +#include "imagery/tile_network_manager.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +class RejectingCookieJar final : public QNetworkCookieJar +{ +public: + using QNetworkCookieJar::QNetworkCookieJar; + + QList cookiesForUrl(const QUrl&) const override + { + return {}; + } + + bool setCookiesFromUrl(const QList&, const QUrl&) override + { + return false; + } +}; + +QByteArray defaultUserAgent() +{ + auto name = QCoreApplication::applicationName(); + if (name.isEmpty()) + name = QStringLiteral("OpenOrienteering-Mapper"); + auto version = QCoreApplication::applicationVersion(); + if (version.isEmpty()) + version = QStringLiteral("development"); + return (name + QLatin1Char('/') + version + + QLatin1String(" (+https://www.openorienteering.org/)")).toUtf8(); +} + +bool isTransientNetworkError(QNetworkReply::NetworkError error) +{ + switch (error) + { + case QNetworkReply::ConnectionRefusedError: + case QNetworkReply::RemoteHostClosedError: + case QNetworkReply::HostNotFoundError: + case QNetworkReply::TimeoutError: + case QNetworkReply::TemporaryNetworkFailureError: + case QNetworkReply::NetworkSessionFailedError: + case QNetworkReply::ProxyConnectionClosedError: + case QNetworkReply::ProxyNotFoundError: + case QNetworkReply::ProxyTimeoutError: + case QNetworkReply::ServiceUnavailableError: + case QNetworkReply::UnknownNetworkError: + case QNetworkReply::UnknownProxyError: + return true; + default: + return false; + } +} + +bool isTransientHttpStatus(int status) +{ + switch (status) + { + case 408: + case 425: + case 429: + case 500: + case 502: + case 503: + case 504: + return true; + default: + return false; + } +} + +QString hostKey(const QUrl& url) +{ + auto const default_port = url.scheme() == QLatin1String("https") ? 443 : 80; + return url.scheme().toLower() + QLatin1String("://") + + QString::fromLatin1(QUrl::toAce(url.host()).toLower()) + + QLatin1Char(':') + QString::number(url.port(default_port)); +} + +bool isLocalHostname(QString host) +{ + host = host.toLower(); + return host == QLatin1String("localhost") + || host == QLatin1String("localhost.localdomain") + || host.endsWith(QLatin1String(".localhost")) + || host.endsWith(QLatin1String(".local")) + || host.endsWith(QLatin1String(".internal")) + || host.endsWith(QLatin1String(".home.arpa")); +} + +QString validateHttpUrl( + const QUrl& url, + const TileNetworkManager::Config& config) +{ + if (!url.isValid() || url.isRelative()) + return TileNetworkManager::tr("The imagery URL is invalid."); + auto const scheme = url.scheme().toLower(); + if (scheme != QLatin1String("http") && scheme != QLatin1String("https")) + return TileNetworkManager::tr("Only HTTP and HTTPS imagery URLs are allowed."); + if (url.host().isEmpty()) + return TileNetworkManager::tr("The imagery URL has no host."); + if (!url.userInfo().isEmpty()) + return TileNetworkManager::tr("Credentials embedded in imagery URLs are not allowed."); + if (url.hasFragment()) + return TileNetworkManager::tr("Imagery URLs must not contain fragments."); + auto const encoded = url.toEncoded(); + if (encoded.contains('\r') || encoded.contains('\n') || encoded.contains('\0')) + return TileNetworkManager::tr("The imagery URL contains unsafe control characters."); + auto const port = url.port(); + if (port == 0 || port > 65535) + return TileNetworkManager::tr("The imagery URL has an invalid port."); + + if (config.allow_private_networks + || config.approved_private_origins.contains(hostKey(url))) + return {}; + + QHostAddress address; + if (address.setAddress(url.host())) + { + if (address.isNull() || address.isLoopback() || address.isLinkLocal() + || address.isMulticast() || address.isPrivateUse()) + { + return TileNetworkManager::tr( + "Private, local, and link-local imagery hosts require explicit permission."); + } + } + else if (isLocalHostname(url.host())) + { + return TileNetworkManager::tr( + "Private, local, and link-local imagery hosts require explicit permission."); + } + return {}; +} + +QString validateRequest( + const TileNetworkRequest& request, + const TileNetworkManager::Config& config) +{ + if (request.client_id == 0) + return TileNetworkManager::tr("The imagery request has no client identity."); + if (!std::isfinite(request.distance_priority)) + return TileNetworkManager::tr("The imagery request priority is invalid."); + if (auto const error = validateHttpUrl(request.url, config); + !error.isEmpty()) + { + return error; + } + if (!request.referer.isEmpty()) + { + auto const referer = QUrl(request.referer); + if (auto const error = validateHttpUrl(referer, config); + !error.isEmpty()) + { + return TileNetworkManager::tr("The imagery Referer is invalid: %1").arg(error); + } + } + if (request.empty_http_status_codes.size() > 32) + return TileNetworkManager::tr("Too many empty-tile HTTP status codes."); + QSet statuses; + for (auto const status : request.empty_http_status_codes) + { + if (status < 100 || status > 599 || statuses.contains(status)) + return TileNetworkManager::tr("The empty-tile HTTP status list is invalid."); + statuses.insert(status); + } + return {}; +} + +bool isBetterEntry(const TileNetworkRequest& lhs, quint64 lhs_sequence, + const TileNetworkRequest& rhs, quint64 rhs_sequence) +{ + if (lhs.priority != rhs.priority) + return lhs.priority < rhs.priority; + if (lhs.distance_priority != rhs.distance_priority) + return lhs.distance_priority < rhs.distance_priority; + return lhs_sequence < rhs_sequence; +} + +} // namespace + +class TileNetworkManager::Worker final : public QObject +{ +public: + Worker(Config config, + QPointer facade, + std::atomic_bool* offline) + : config_(std::move(config)) + , facade_(std::move(facade)) + , offline_(offline) + {} + + void initialize() + { + Q_ASSERT(QThread::currentThread() == thread()); + clock_.start(); + wake_timer_ = new QTimer(this); + wake_timer_->setSingleShot(true); + connect(wake_timer_, &QTimer::timeout, this, [this] { dispatch(); }); + + network_ = new QNetworkAccessManager(this); + network_->setCookieJar(new RejectingCookieJar(network_)); + network_->setRedirectPolicy(QNetworkRequest::ManualRedirectPolicy); + network_->setTransferTimeout(config_.transfer_timeout); + connect( + network_, &QNetworkAccessManager::authenticationRequired, + this, [this](QNetworkReply* reply, QAuthenticator*) { + auto const found = active_replies_.constFind(reply); + if (found == active_replies_.cend()) + return; + (*found)->authentication_rejected = true; + reply->abort(); + }); + connect( + network_, &QNetworkAccessManager::proxyAuthenticationRequired, + this, [](const QNetworkProxy&, QAuthenticator*) { + // Catalogs and sources never supply proxy credentials. + }); + + auto* cache = new QNetworkDiskCache(network_); + cache->setCacheDirectory(config_.cache_directory); + cache->setMaximumCacheSize(config_.disk_cache_bytes); + network_->setCache(cache); + } + + void shutdown() + { + Q_ASSERT(QThread::currentThread() == thread()); + shutting_down_ = true; + if (wake_timer_) + wake_timer_->stop(); + auto const active_replies = active_replies_.keys(); + for (auto* reply : active_replies) + { + disconnect(reply, nullptr, this, nullptr); + reply->abort(); + } + for (auto const lookup_id : std::as_const(destination_lookups_)) + QHostInfo::abortHostLookup(lookup_id); + destination_lookups_.clear(); + destination_waiters_.clear(); + queue_.clear(); + entries_.clear(); + active_replies_.clear(); + active_hosts_.clear(); + active_clients_.clear(); + active_total_ = 0; + if (network_) + { + delete network_; + network_ = nullptr; + } + } + + void submit(Token token, TileNetworkRequest request) + { + Q_ASSERT(QThread::currentThread() == thread()); + if (shutting_down_) + return; + + if (auto const error = validateRequest(request, config_); + !error.isEmpty()) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Rejected; + result.error_string = error; + deliver(token, request, std::move(result)); + return; + } + + auto url = request.url; + auto const negative = negative_cache_.constFind(url); + if (negative != negative_cache_.cend()) + { + if (*negative > now()) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::EmptyTile; + result.from_cache = true; + deliver(token, request, std::move(result)); + return; + } + negative_cache_.erase(negative); + } + + auto const pending_total = std::max( + 0, entries_.size() - active_total_); + auto const pending_for_client = std::ranges::count_if( + entries_, [&request](auto const& entry) { + return !entry->reply + && entry->request.client_id == request.client_id; + }); + if (pending_total >= config_.max_pending_total + || pending_for_client >= config_.max_pending_per_client) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Rejected; + result.error_string = TileNetworkManager::tr( + "The imagery request queue is full."); + deliver(token, request, std::move(result)); + return; + } + + auto entry = std::make_shared(); + entry->token = token; + entry->request = std::move(request); + entry->current_url = std::move(url); + entry->sequence = next_sequence_++; + entries_.insert(token, entry); + queueAfterDestinationCheck(entry); + } + + void cancel(Token token) + { + Q_ASSERT(QThread::currentThread() == thread()); + auto const found = entries_.constFind(token); + if (found == entries_.cend()) + return; + auto const entry = *found; + entry->cancelled = true; + if (entry->reply) + { + entry->reply->abort(); + return; + } + eraseQueued(entry); + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Cancelled; + finish(entry, std::move(result)); + } + + void cancelClient(quint64 client_id, quint64 through_generation) + { + Q_ASSERT(QThread::currentThread() == thread()); + QVector tokens; + for (auto it = entries_.cbegin(); it != entries_.cend(); ++it) + { + auto const& request = (*it)->request; + if (request.client_id == client_id + && request.generation <= through_generation) + { + tokens.push_back(it.key()); + } + } + for (auto const token : tokens) + cancel(token); + } + +private: + struct Entry + { + Token token = 0; + TileNetworkRequest request; + QUrl current_url; + quint64 sequence = 0; + int redirects = 0; + int retries = 0; + qint64 not_before = 0; + QPointer reply; + QByteArray body; + QString active_host; + bool cancelled = false; + bool too_large = false; + bool absolute_timeout = false; + bool authentication_rejected = false; + bool received_metadata = false; + }; + + struct DestinationDecision + { + bool allowed = false; + bool transient_failure = false; + QString error; + qint64 expires = 0; + }; + + qint64 now() const + { + return clock_.elapsed(); + } + + void enqueue(const std::shared_ptr& entry) + { + if (!entries_.contains(entry->token) || entry->cancelled) + return; + entry->not_before = std::max(entry->not_before, now()); + queue_.push_back(entry); + dispatch(); + } + + void destinationFailure( + const std::shared_ptr& entry, + const DestinationDecision& decision) + { + if (!entries_.contains(entry->token)) + return; + TileNetworkResult result; + result.outcome = decision.transient_failure + ? TileNetworkResult::Outcome::TransientError + : TileNetworkResult::Outcome::Rejected; + result.error_string = decision.error; + finish(entry, std::move(result)); + } + + void queueAfterDestinationCheck(const std::shared_ptr& entry) + { + auto const origin = hostKey(entry->current_url); + if (config_.allow_private_networks + || config_.approved_private_origins.contains(origin)) + { + enqueue(entry); + return; + } + + QHostAddress literal; + if (literal.setAddress(entry->current_url.host())) + { + // validateHttpUrl() already rejected non-global literals. + enqueue(entry); + return; + } + + auto const cached = destination_cache_.constFind(origin); + if (cached != destination_cache_.cend() && cached->expires > now()) + { + if (cached->allowed) + enqueue(entry); + else + destinationFailure(entry, *cached); + return; + } + if (cached != destination_cache_.cend()) + destination_cache_.erase(cached); + + auto& waiters = destination_waiters_[origin]; + waiters.push_back(entry); + if (destination_lookups_.contains(origin)) + return; + + auto const lookup_id = QHostInfo::lookupHost( + entry->current_url.host(), this, + [this, origin](QHostInfo info) { + destination_lookups_.remove(origin); + auto waiters = destination_waiters_.take(origin); + DestinationDecision decision; + if (info.error() != QHostInfo::NoError || info.addresses().isEmpty()) + { + decision.transient_failure = true; + decision.error = TileNetworkManager::tr( + "The imagery host could not be resolved."); + decision.expires = now() + 10'000; + } + else + { + decision.allowed = std::ranges::all_of( + info.addresses(), + [](auto const& address) { return address.isGlobal(); }); + if (!decision.allowed) + { + decision.error = TileNetworkManager::tr( + "The imagery host resolved to a private or non-global address."); + } + decision.expires = now() + 5 * 60 * 1000; + } + destination_cache_.insert(origin, decision); + for (auto const& waiter : std::as_const(waiters)) + { + if (decision.allowed) + enqueue(waiter); + else + destinationFailure(waiter, decision); + } + dispatch(); + }); + destination_lookups_.insert(origin, lookup_id); + } + + bool eligible(const std::shared_ptr& entry, qint64 current) const + { + if (entry->cancelled || entry->not_before > current) + return false; + if (active_clients_.value(entry->request.client_id) + >= config_.max_active_per_client) + { + return false; + } + auto const host = hostKey(entry->current_url); + if (active_hosts_.value(host) >= config_.max_active_per_host) + return false; + return host_not_before_.value(host) <= current; + } + + std::optional chooseNext() const + { + auto const current = now(); + QHash best_for_client; + for (int index = 0; index < queue_.size(); ++index) + { + auto const& entry = queue_.at(index); + if (!eligible(entry, current)) + continue; + auto const client = entry->request.client_id; + auto const found = best_for_client.constFind(client); + if (found == best_for_client.cend()) + { + best_for_client.insert(client, index); + continue; + } + auto const& current_best = queue_.at(*found); + if (isBetterEntry( + entry->request, entry->sequence, + current_best->request, current_best->sequence)) + { + best_for_client[client] = index; + } + } + if (best_for_client.isEmpty()) + return std::nullopt; + + std::optional selected; + quint64 selected_service = 0; + quint64 selected_sequence = 0; + for (auto const index : std::as_const(best_for_client)) + { + auto const& entry = queue_.at(index); + auto const service = client_last_service_.value(entry->request.client_id); + if (!selected || service < selected_service + || (service == selected_service + && entry->sequence < selected_sequence)) + { + selected = index; + selected_service = service; + selected_sequence = entry->sequence; + } + } + return selected; + } + + void dispatch() + { + Q_ASSERT(QThread::currentThread() == thread()); + if (shutting_down_ || !network_) + return; + if (wake_timer_) + wake_timer_->stop(); + + while (active_total_ < config_.max_active_total) + { + auto const selected = chooseNext(); + if (!selected) + break; + auto entry = queue_.takeAt(*selected); + client_last_service_[entry->request.client_id] = next_service_++; + start(entry); + } + scheduleWake(); + } + + void scheduleWake() + { + if (!wake_timer_ || queue_.isEmpty() + || active_total_ >= config_.max_active_total) + return; + auto const current = now(); + auto earliest = std::numeric_limits::max(); + for (auto const& entry : std::as_const(queue_)) + { + auto const ready = std::max( + entry->not_before, + host_not_before_.value(hostKey(entry->current_url))); + if (ready > current) + earliest = std::min(earliest, ready); + } + if (earliest == std::numeric_limits::max()) + return; + auto const delay = int(std::clamp( + earliest - current, 1, 60'000)); + wake_timer_->start(delay); + } + + void start(const std::shared_ptr& entry) + { + if (offline_->load() && !entry->request.referer.isEmpty()) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::OfflineMiss; + result.error_string = TileNetworkManager::tr( + "Referer-dependent imagery is not stored in the offline HTTP cache."); + finish(entry, std::move(result)); + return; + } + + entry->body.clear(); + entry->too_large = false; + entry->absolute_timeout = false; + entry->authentication_rejected = false; + entry->received_metadata = false; + entry->active_host = hostKey(entry->current_url); + ++active_total_; + ++active_hosts_[entry->active_host]; + ++active_clients_[entry->request.client_id]; + + QNetworkRequest request(entry->current_url); + request.setHeader( + QNetworkRequest::UserAgentHeader, + QString::fromUtf8(config_.user_agent)); + if (!entry->request.referer.isEmpty()) + { + request.setRawHeader( + QByteArrayLiteral("Referer"), + entry->request.referer.toUtf8()); + } + request.setRawHeader(QByteArrayLiteral("Accept"), QByteArrayLiteral("image/*")); + request.setPriority( + entry->request.priority == TileRequestPriority::Coverage + ? QNetworkRequest::HighPriority + : entry->request.priority == TileRequestPriority::Background + ? QNetworkRequest::LowPriority + : QNetworkRequest::NormalPriority); + request.setAttribute( + QNetworkRequest::RedirectPolicyAttribute, + QNetworkRequest::ManualRedirectPolicy); + request.setAttribute( + QNetworkRequest::CookieLoadControlAttribute, + QNetworkRequest::Manual); + request.setAttribute( + QNetworkRequest::CookieSaveControlAttribute, + QNetworkRequest::Manual); + request.setAttribute( + QNetworkRequest::AuthenticationReuseAttribute, + QNetworkRequest::Manual); + request.setAttribute(QNetworkRequest::UseCredentialsAttribute, false); + auto const referer_dependent = !entry->request.referer.isEmpty(); + request.setAttribute( + QNetworkRequest::CacheLoadControlAttribute, + referer_dependent + ? QNetworkRequest::AlwaysNetwork + : offline_->load() + ? QNetworkRequest::AlwaysCache + : QNetworkRequest::PreferNetwork); + request.setAttribute( + QNetworkRequest::CacheSaveControlAttribute, + !referer_dependent); + request.setMaximumRedirectsAllowed(config_.max_redirects); + request.setTransferTimeout(config_.transfer_timeout); + request.setDecompressedSafetyCheckThreshold(config_.max_response_bytes); + + auto* reply = network_->get(request); + entry->reply = reply; + active_replies_.insert(reply, entry); + connect(reply, &QNetworkReply::metaDataChanged, this, [this, entry] { + if (!entry->reply) + return; + entry->received_metadata = true; + auto const length = entry->reply->header( + QNetworkRequest::ContentLengthHeader).toLongLong(); + if (length > config_.max_response_bytes) + { + entry->too_large = true; + entry->reply->abort(); + } + }); + connect(reply, &QIODevice::readyRead, this, [this, entry] { + if (!entry->reply || entry->too_large) + return; + auto chunk = entry->reply->readAll(); + if (chunk.size() > config_.max_response_bytes - entry->body.size()) + { + entry->too_large = true; + entry->body.clear(); + entry->reply->abort(); + return; + } + entry->body += chunk; + }); + connect(reply, &QNetworkReply::finished, this, [this, entry] { + replyFinished(entry); + }); + QTimer::singleShot( + config_.first_byte_timeout, + reply, + [entry] { + if (entry->reply && entry->reply->isRunning() + && !entry->received_metadata) + { + entry->absolute_timeout = true; + entry->reply->abort(); + } + }); + QTimer::singleShot( + config_.absolute_timeout, + reply, + [entry] { + if (entry->reply && entry->reply->isRunning()) + { + entry->absolute_timeout = true; + entry->reply->abort(); + } + }); + } + + void releaseActive(const std::shared_ptr& entry) + { + auto* reply = entry->reply.data(); + if (reply) + active_replies_.remove(reply); + entry->reply = nullptr; + --active_total_; + if (--active_hosts_[entry->active_host] <= 0) + active_hosts_.remove(entry->active_host); + if (--active_clients_[entry->request.client_id] <= 0) + active_clients_.remove(entry->request.client_id); + entry->active_host.clear(); + if (reply) + reply->deleteLater(); + } + + void replyFinished(const std::shared_ptr& entry) + { + if (!entry->reply) + return; + auto* reply = entry->reply.data(); + if (!entry->too_large) + { + auto tail = reply->readAll(); + if (tail.size() > config_.max_response_bytes - entry->body.size()) + { + entry->too_large = true; + entry->body.clear(); + } + else + { + entry->body += tail; + } + } + + auto const network_error = reply->error(); + auto const network_error_string = reply->errorString(); + auto const status = reply->attribute( + QNetworkRequest::HttpStatusCodeAttribute).toInt(); + auto const redirect = reply->attribute( + QNetworkRequest::RedirectionTargetAttribute).toUrl(); + auto const content_type = reply->header( + QNetworkRequest::ContentTypeHeader).toString(); + auto const from_cache = reply->attribute( + QNetworkRequest::SourceIsFromCacheAttribute).toBool(); + auto const retry_after = reply->rawHeader(QByteArrayLiteral("Retry-After")); + releaseActive(entry); + + if (entry->cancelled) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Cancelled; + finish(entry, std::move(result)); + dispatch(); + return; + } + if (entry->too_large) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::PermanentError; + result.error_string = TileNetworkManager::tr( + "The imagery response exceeded the %1 MB safety limit.") + .arg(config_.max_response_bytes / (1024 * 1024)); + finish(entry, std::move(result)); + dispatch(); + return; + } + if (entry->authentication_rejected) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::PermanentError; + result.error_string = TileNetworkManager::tr( + "Imagery sources requiring HTTP authentication are not supported."); + finish(entry, std::move(result)); + dispatch(); + return; + } + + if (!redirect.isEmpty() && status >= 300 && status < 400) + { + auto const next = entry->current_url.resolved(redirect) + .adjusted(QUrl::RemoveFragment); + auto error = validateHttpUrl(next, config_); + if (error.isEmpty() + && entry->current_url.scheme() == QLatin1String("https") + && next.scheme() == QLatin1String("http") + && !config_.allow_https_downgrade) + { + error = TileNetworkManager::tr( + "An imagery redirect attempted to downgrade HTTPS to HTTP."); + } + if (error.isEmpty() && entry->redirects >= config_.max_redirects) + { + error = TileNetworkManager::tr( + "The imagery server redirected too many times."); + } + if (!error.isEmpty()) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::PermanentError; + result.http_status = status; + result.error_string = error; + finish(entry, std::move(result)); + dispatch(); + return; + } + ++entry->redirects; + entry->current_url = next; + entry->not_before = now(); + queueAfterDestinationCheck(entry); + return; + } + + if (entry->request.empty_http_status_codes.contains(status)) + { + negative_cache_.insert( + entry->request.url.adjusted(QUrl::RemoveFragment), + now() + config_.negative_cache_ttl_ms); + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::EmptyTile; + result.http_status = status; + result.content_type = content_type; + result.from_cache = from_cache; + finish(entry, std::move(result)); + dispatch(); + return; + } + + if (network_error == QNetworkReply::NoError && status >= 200 && status < 300) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Success; + result.body = std::move(entry->body); + result.http_status = status; + result.content_type = content_type; + result.from_cache = from_cache; + finish(entry, std::move(result)); + dispatch(); + return; + } + + if (offline_->load() + && (network_error == QNetworkReply::ContentNotFoundError + || network_error == QNetworkReply::ProtocolUnknownError)) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::OfflineMiss; + result.http_status = status; + result.error_string = TileNetworkManager::tr( + "The imagery tile is not available in the offline cache."); + finish(entry, std::move(result)); + dispatch(); + return; + } + + auto const transient = entry->absolute_timeout + || isTransientNetworkError(network_error) + || isTransientHttpStatus(status); + if (transient && entry->retries < config_.max_retries) + { + auto delay = retryDelay(entry, retry_after); + if (status == 429 || status == 503) + { + auto const host = hostKey(entry->current_url); + host_not_before_[host] = std::max( + host_not_before_.value(host), now() + delay); + } + ++entry->retries; + entry->not_before = now() + delay; + enqueue(entry); + return; + } + + TileNetworkResult result; + result.outcome = transient + ? TileNetworkResult::Outcome::TransientError + : TileNetworkResult::Outcome::PermanentError; + result.http_status = status; + result.content_type = content_type; + result.from_cache = from_cache; + result.error_string = entry->absolute_timeout + ? TileNetworkManager::tr("The imagery request timed out.") + : network_error_string; + if (result.error_string.isEmpty()) + { + result.error_string = TileNetworkManager::tr( + "The imagery server returned HTTP status %1.").arg(status); + } + finish(entry, std::move(result)); + dispatch(); + } + + int retryDelay(const std::shared_ptr& entry, const QByteArray& header) const + { + bool seconds_ok = false; + auto const seconds = header.trimmed().toInt(&seconds_ok); + qint64 delay = 0; + if (seconds_ok && seconds >= 0) + { + delay = qint64(seconds) * 1000; + } + else if (!header.isEmpty()) + { + auto const date = QDateTime::fromString( + QString::fromLatin1(header), Qt::RFC2822Date); + if (date.isValid()) + delay = QDateTime::currentDateTimeUtc().msecsTo(date.toUTC()); + } + if (delay <= 0) + { + delay = qint64(config_.retry_base_delay_ms) + << std::min(entry->retries, 20); + auto const jitter_percent = int( + (entry->token * 1103515245u + quint64(entry->retries) * 12345u) % 21u) - 10; + delay += delay * jitter_percent / 100; + } + return int(std::clamp( + delay, 1, std::max(1, config_.retry_max_delay_ms))); + } + + void eraseQueued(const std::shared_ptr& entry) + { + queue_.erase( + std::remove_if( + queue_.begin(), queue_.end(), + [&entry](auto const& queued) { return queued == entry; }), + queue_.end()); + } + + void finish(const std::shared_ptr& entry, TileNetworkResult result) + { + eraseQueued(entry); + entries_.remove(entry->token); + deliver(entry->token, entry->request, std::move(result)); + } + + void deliver(Token token, const TileNetworkRequest& request, TileNetworkResult result) + { + if (shutting_down_) + return; + result.client_id = request.client_id; + result.generation = request.generation; + result.user_data = request.user_data; + auto facade = facade_; + if (!facade) + return; + QMetaObject::invokeMethod( + facade, + [facade, token, result = std::move(result)] { + if (facade) + emit facade->finished(token, result); + }, + Qt::QueuedConnection); + } + + Config config_; + QPointer facade_; + std::atomic_bool* offline_ = nullptr; + QElapsedTimer clock_; + QNetworkAccessManager* network_ = nullptr; + QTimer* wake_timer_ = nullptr; + bool shutting_down_ = false; + quint64 next_sequence_ = 1; + quint64 next_service_ = 1; + int active_total_ = 0; + QHash> entries_; + QVector> queue_; + QHash> active_replies_; + QHash active_hosts_; + QHash active_clients_; + QHash client_last_service_; + QHash host_not_before_; + QHash negative_cache_; + QHash destination_cache_; + QHash destination_lookups_; + QHash>> destination_waiters_; +}; + +TileNetworkManager::TileNetworkManager(QObject* parent) + : TileNetworkManager(Config {}, parent) +{} + +TileNetworkManager::TileNetworkManager(Config config, QObject* parent) + : QObject(parent) + , config_(std::move(config)) +{ + if (config_.cache_directory.isEmpty()) + { + config_.cache_directory = QDir( + QStandardPaths::writableLocation(QStandardPaths::CacheLocation)) + .filePath(QStringLiteral("online-imagery")); + } + if (config_.user_agent.isEmpty()) + config_.user_agent = defaultUserAgent(); + config_.max_active_total = std::max(1, config_.max_active_total); + config_.max_active_per_host = std::max(1, config_.max_active_per_host); + config_.max_active_per_client = std::max(1, config_.max_active_per_client); + config_.max_pending_total = std::max(1, config_.max_pending_total); + config_.max_pending_per_client = std::max(1, config_.max_pending_per_client); + config_.max_redirects = std::max(0, config_.max_redirects); + config_.max_retries = std::max(0, config_.max_retries); + config_.max_response_bytes = std::max(1, config_.max_response_bytes); + config_.disk_cache_bytes = std::max(0, config_.disk_cache_bytes); + + qRegisterMetaType(); + network_thread_ = new QThread(this); + network_thread_->setObjectName(QStringLiteral("Mapper imagery network")); + worker_ = new Worker(config_, this, &offline_); + worker_->moveToThread(network_thread_); + network_thread_->start(); + QMetaObject::invokeMethod( + worker_, [worker = worker_] { worker->initialize(); }, + Qt::BlockingQueuedConnection); +} + +TileNetworkManager::~TileNetworkManager() +{ + if (!network_thread_ || !worker_) + return; + QMetaObject::invokeMethod( + worker_, [worker = worker_] { worker->shutdown(); }, + Qt::BlockingQueuedConnection); + QMetaObject::invokeMethod(worker_, &QObject::deleteLater, Qt::QueuedConnection); + network_thread_->quit(); + network_thread_->wait(); + worker_ = nullptr; +} + +Q_APPLICATION_STATIC(TileNetworkManager, application_tile_network_manager) + +TileNetworkManager& TileNetworkManager::instance() +{ + auto* application = QCoreApplication::instance(); + Q_ASSERT(application); + Q_ASSERT(QThread::currentThread() == application->thread()); + return *application_tile_network_manager; +} + +quint64 TileNetworkManager::nextClientId() +{ + static std::atomic next { 1 }; + auto const id = next.fetch_add(1); + if (id == 0) + qFatal("Imagery network client identity space exhausted"); + return id; +} + +QString TileNetworkManager::canonicalOrigin(const QUrl& url) +{ + return hostKey(url); +} + +TileNetworkManager::Token TileNetworkManager::submit(TileNetworkRequest request) +{ + auto const token = next_token_.fetch_add(1); + if (token == 0) + qFatal("Imagery network token space exhausted"); + QMetaObject::invokeMethod( + worker_, + [worker = worker_, token, request = std::move(request)]() mutable { + worker->submit(token, std::move(request)); + }, + Qt::QueuedConnection); + return token; +} + +void TileNetworkManager::cancel(Token token) +{ + QMetaObject::invokeMethod( + worker_, + [worker = worker_, token] { worker->cancel(token); }, + Qt::QueuedConnection); +} + +void TileNetworkManager::cancelClient( + quint64 client_id, quint64 through_generation) +{ + QMetaObject::invokeMethod( + worker_, + [worker = worker_, client_id, through_generation] { + worker->cancelClient(client_id, through_generation); + }, + Qt::QueuedConnection); +} + +void TileNetworkManager::setOfflineMode(bool offline) +{ + offline_.store(offline); +} + +bool TileNetworkManager::offlineMode() const noexcept +{ + return offline_.load(); +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/tile_network_manager.h b/src/imagery/tile_network_manager.h new file mode 100644 index 000000000..a6854f907 --- /dev/null +++ b/src/imagery/tile_network_manager.h @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +#ifndef OPENORIENTEERING_TILE_NETWORK_MANAGER_H +#define OPENORIENTEERING_TILE_NETWORK_MANAGER_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +class QThread; + +namespace OpenOrienteering::imagery { + +enum class TileRequestPriority +{ + Coverage, + Visible, + Background, +}; + +struct TileNetworkRequest +{ + QUrl url; + quint64 client_id = 0; + quint64 generation = 0; + quint64 user_data = 0; + TileRequestPriority priority = TileRequestPriority::Visible; + double distance_priority = 0; + QString referer; + QVector empty_http_status_codes = { 204, 404 }; +}; + +struct TileNetworkResult +{ + enum class Outcome + { + Success, + EmptyTile, + Cancelled, + OfflineMiss, + TransientError, + PermanentError, + Rejected, + }; + + Outcome outcome = Outcome::PermanentError; + QByteArray body; + QString content_type; + QString error_string; + int http_status = 0; + bool from_cache = false; + quint64 client_id = 0; + quint64 generation = 0; + quint64 user_data = 0; +}; + +/** + * Application-scoped, bounded HTTP scheduler for tiled imagery. + * + * A single QNetworkAccessManager and QNetworkDiskCache live on a dedicated + * event-loop thread. Public methods are thread-safe. Results are emitted on + * this object's thread and retain the caller's client/generation/user fields. + * + * Requests are owner-fair: clients rotate before priority is considered, and + * each client's coverage, visible, then background work is ordered by distance. + * The manager enforces total, per-host, per-client, and pending limits. + * + * Only HTTP(S) URLs without embedded credentials are accepted. Cookies and + * HTTP authentication are disabled. Redirects are validated explicitly and + * HTTPS downgrades are rejected by default. Response bodies and time are + * bounded before image decoding. + */ +class TileNetworkManager final : public QObject +{ +Q_OBJECT + +public: + using Token = quint64; + + struct Config + { + QString cache_directory; +#ifdef Q_OS_ANDROID + qint64 disk_cache_bytes = qint64(128) << 20; + int max_active_total = 6; + int max_active_per_client = 4; +#else + qint64 disk_cache_bytes = qint64(512) << 20; + int max_active_total = 12; + int max_active_per_client = 6; +#endif + int max_active_per_host = 6; + int max_pending_total = 2048; + int max_pending_per_client = 256; + int max_redirects = 5; + int max_retries = 2; + int retry_base_delay_ms = 400; + int retry_max_delay_ms = 15'000; + int negative_cache_ttl_ms = 5 * 60 * 1000; + qint64 max_response_bytes = 20 * 1024 * 1024; + std::chrono::milliseconds first_byte_timeout = std::chrono::seconds(10); + std::chrono::milliseconds transfer_timeout = std::chrono::seconds(20); + std::chrono::milliseconds absolute_timeout = std::chrono::seconds(45); + QByteArray user_agent; + QSet approved_private_origins; + /** Intended for deterministic tests and explicitly trusted deployments. */ + bool allow_private_networks = false; + bool allow_https_downgrade = false; + }; + + explicit TileNetworkManager(QObject* parent = nullptr); + explicit TileNetworkManager(Config config, QObject* parent = nullptr); + ~TileNetworkManager() override; + + TileNetworkManager(const TileNetworkManager&) = delete; + TileNetworkManager& operator=(const TileNetworkManager&) = delete; + + static TileNetworkManager& instance(); + static quint64 nextClientId(); + static QString canonicalOrigin(const QUrl& url); + + Token submit(TileNetworkRequest request); + void cancel(Token token); + void cancelClient( + quint64 client_id, + quint64 through_generation = std::numeric_limits::max()); + + void setOfflineMode(bool offline); + bool offlineMode() const noexcept; + +signals: + void finished( + OpenOrienteering::imagery::TileNetworkManager::Token token, + const OpenOrienteering::imagery::TileNetworkResult& result); + +private: + class Worker; + + Config config_; + QThread* network_thread_ = nullptr; + Worker* worker_ = nullptr; + std::atomic next_token_ { 1 }; + std::atomic_bool offline_ { false }; +}; + +} // namespace OpenOrienteering::imagery + +Q_DECLARE_METATYPE(OpenOrienteering::imagery::TileNetworkResult) + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 45a837605..05423278f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -190,6 +190,8 @@ add_unit_test(util_t ../src/util/util # Low-level imagery contracts stay independent of the full Mapper runtime. add_test_helper(imagery_core_t) target_link_libraries(imagery_core_t PRIVATE Mapper::ImageryCore) +add_test_helper(tile_network_manager_t) +target_link_libraries(tile_network_manager_t PRIVATE Mapper::ImageryNetwork Qt6::Network) # Benchmarks add_system_test(coord_xml_t MANUAL) diff --git a/test/tile_network_manager_t.cpp b/test/tile_network_manager_t.cpp new file mode 100644 index 000000000..9a1463591 --- /dev/null +++ b/test/tile_network_manager_t.cpp @@ -0,0 +1,420 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "tile_network_manager_t.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/tile_network_manager.h" + +using OpenOrienteering::imagery::TileNetworkManager; +using OpenOrienteering::imagery::TileNetworkRequest; +using OpenOrienteering::imagery::TileNetworkResult; +using OpenOrienteering::imagery::TileRequestPriority; + +namespace { + +class MiniHttpServer final : public QObject +{ +public: + explicit MiniHttpServer(QObject* parent = nullptr) + : QObject(parent) + { + connect(&server_, &QTcpServer::newConnection, this, [this] { + while (auto* socket = server_.nextPendingConnection()) + { + connect(socket, &QTcpSocket::readyRead, this, [this, socket] { + buffers_[socket] += socket->readAll(); + if (!buffers_[socket].contains("\r\n\r\n")) + return; + auto const request = buffers_.take(socket); + auto const first_line = request.left(request.indexOf("\r\n")); + auto const parts = first_line.split(' '); + if (parts.size() < 2) + { + respond(socket, 400, "Bad Request", {}); + return; + } + auto const path = QString::fromUtf8(parts.at(1)); + paths_.push_back(path); + handle(socket, path); + }); + connect(socket, &QTcpSocket::disconnected, this, [this, socket] { + buffers_.remove(socket); + socket->deleteLater(); + }); + } + }); + QVERIFY(server_.listen(QHostAddress::LocalHost)); + } + + QUrl url(const QString& path) const + { + return QUrl( + QStringLiteral("http://127.0.0.1:%1%2") + .arg(server_.serverPort()) + .arg(path)); + } + + QStringList paths() const + { + return paths_; + } + + int retryRequests() const + { + return retry_requests_; + } + + int heldCount() const + { + return int(std::ranges::count_if( + held_, [](auto const& socket) { return !socket.isNull(); })); + } + + void releaseOne() + { + for (auto& socket : held_) + { + if (!socket) + continue; + respond(socket, 200, "OK", QByteArrayLiteral("held")); + socket.clear(); + return; + } + } + + void releaseAll() + { + while (heldCount() > 0) + releaseOne(); + } + +private: + static QByteArray reason(int status) + { + switch (status) + { + case 200: return QByteArrayLiteral("OK"); + case 302: return QByteArrayLiteral("Found"); + case 404: return QByteArrayLiteral("Not Found"); + case 503: return QByteArrayLiteral("Service Unavailable"); + default: return QByteArrayLiteral("Error"); + } + } + + void respond( + QTcpSocket* socket, + int status, + QByteArray status_text, + QByteArray body, + QByteArray extra_headers = {}) + { + if (!socket) + return; + if (status_text.isEmpty()) + status_text = reason(status); + QByteArray response = QByteArrayLiteral("HTTP/1.1 ") + + QByteArray::number(status) + ' ' + status_text + + QByteArrayLiteral("\r\nContent-Length: ") + + QByteArray::number(body.size()) + + QByteArrayLiteral("\r\nConnection: close\r\n") + + extra_headers + + QByteArrayLiteral("\r\n") + + body; + socket->write(response); + socket->disconnectFromHost(); + } + + void handle(QTcpSocket* socket, const QString& path) + { + if (path == QLatin1String("/ok")) + { + respond( + socket, 200, {}, QByteArrayLiteral("tile"), + QByteArrayLiteral("Content-Type: image/png\r\n")); + } + else if (path == QLatin1String("/empty")) + { + respond(socket, 404, {}, {}); + } + else if (path == QLatin1String("/large")) + { + respond(socket, 200, {}, QByteArray(256, 'x')); + } + else if (path == QLatin1String("/redirect")) + { + respond( + socket, 302, {}, {}, + QByteArrayLiteral("Location: /ok\r\n")); + } + else if (path == QLatin1String("/retry")) + { + ++retry_requests_; + if (retry_requests_ == 1) + { + respond( + socket, 503, {}, {}, + QByteArrayLiteral("Retry-After: 0\r\n")); + } + else + { + respond(socket, 200, {}, QByteArrayLiteral("retry-ok")); + } + } + else if (path == QLatin1String("/cache")) + { + respond( + socket, 200, {}, QByteArrayLiteral("cached"), + QByteArrayLiteral( + "Content-Type: image/png\r\n" + "Cache-Control: public, max-age=3600\r\n")); + } + else if (path == QLatin1String("/slow")) + { + held_.push_back(socket); + } + else if (path.startsWith(QLatin1String("/hold/"))) + { + held_.push_back(socket); + } + else if (path.startsWith(QLatin1String("/order/"))) + { + respond(socket, 200, {}, path.toUtf8()); + } + else + { + respond(socket, 404, {}, {}); + } + } + + QTcpServer server_; + QHash buffers_; + QVector> held_; + QStringList paths_; + int retry_requests_ = 0; +}; + +TileNetworkManager::Config configFor( + const QTemporaryDir& directory, + qint64 max_response_bytes = 1024) +{ + TileNetworkManager::Config config; + config.cache_directory = directory.filePath(QStringLiteral("cache")); + config.disk_cache_bytes = 1024 * 1024; + config.max_active_total = 4; + config.max_active_per_host = 4; + config.max_active_per_client = 4; + config.max_pending_total = 32; + config.max_pending_per_client = 16; + config.max_response_bytes = max_response_bytes; + config.max_retries = 0; + config.retry_base_delay_ms = 5; + config.retry_max_delay_ms = 50; + config.transfer_timeout = std::chrono::milliseconds(250); + config.absolute_timeout = std::chrono::milliseconds(1000); + config.allow_private_networks = true; + return config; +} + +TileNetworkRequest request( + QUrl url, + quint64 client, + quint64 generation = 1, + quint64 user_data = 0) +{ + TileNetworkRequest result; + result.url = std::move(url); + result.client_id = client; + result.generation = generation; + result.user_data = user_data; + return result; +} + +TileNetworkResult resultAt(const QSignalSpy& spy, int index) +{ + return qvariant_cast(spy.at(index).at(1)); +} + +} // namespace + +void TileNetworkManagerTest::rejectsUnsafeUrls() +{ + QTemporaryDir directory; + QVERIFY(directory.isValid()); + auto config = configFor(directory); + config.allow_private_networks = false; + TileNetworkManager manager(config); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + + manager.submit(request(QUrl(QStringLiteral("file:///tmp/tile.png")), 1)); + manager.submit(request(QUrl(QStringLiteral("http://127.0.0.1/tile.png")), 1)); + manager.submit(request( + QUrl(QStringLiteral("https://user:secret@example.test/tile.png")), 1)); + + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 3, 2000); + for (int index = 0; index < spy.size(); ++index) + QCOMPARE(resultAt(spy, index).outcome, TileNetworkResult::Outcome::Rejected); +} + +void TileNetworkManagerTest::handlesRedirectsEmptyTilesAndBodyLimits() +{ + MiniHttpServer server; + QTemporaryDir directory; + QVERIFY(directory.isValid()); + TileNetworkManager manager(configFor(directory, 32)); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + + manager.submit(request(server.url(QStringLiteral("/redirect")), 1, 1, 10)); + manager.submit(request(server.url(QStringLiteral("/empty")), 1, 1, 20)); + manager.submit(request(server.url(QStringLiteral("/large")), 1, 1, 30)); + + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 3, 3000); + QHash by_user_data; + for (int index = 0; index < spy.size(); ++index) + by_user_data.insert(resultAt(spy, index).user_data, resultAt(spy, index)); + QCOMPARE(by_user_data[10].outcome, TileNetworkResult::Outcome::Success); + QCOMPARE(by_user_data[10].body, QByteArray("tile")); + QCOMPARE(by_user_data[20].outcome, TileNetworkResult::Outcome::EmptyTile); + QCOMPARE(by_user_data[20].http_status, 404); + QCOMPARE(by_user_data[30].outcome, TileNetworkResult::Outcome::PermanentError); + QVERIFY(by_user_data[30].error_string.contains(QStringLiteral("safety"))); +} + +void TileNetworkManagerTest::retriesTransientFailures() +{ + MiniHttpServer server; + QTemporaryDir directory; + QVERIFY(directory.isValid()); + auto config = configFor(directory); + config.max_retries = 1; + TileNetworkManager manager(config); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + + manager.submit(request(server.url(QStringLiteral("/retry")), 1)); + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 1, 3000); + QCOMPARE(resultAt(spy, 0).outcome, TileNetworkResult::Outcome::Success); + QCOMPARE(resultAt(spy, 0).body, QByteArray("retry-ok")); + QCOMPARE(server.retryRequests(), 2); +} + +void TileNetworkManagerTest::cancelsClientGenerations() +{ + MiniHttpServer server; + QTemporaryDir directory; + QVERIFY(directory.isValid()); + TileNetworkManager manager(configFor(directory)); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + + manager.submit(request(server.url(QStringLiteral("/hold/old")), 42, 3)); + QTRY_COMPARE_WITH_TIMEOUT(server.heldCount(), 1, 2000); + manager.cancelClient(42, 3); + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 1, 2000); + QCOMPARE(resultAt(spy, 0).outcome, TileNetworkResult::Outcome::Cancelled); + QCOMPARE(resultAt(spy, 0).client_id, quint64(42)); + QCOMPARE(resultAt(spy, 0).generation, quint64(3)); +} + +void TileNetworkManagerTest::enforcesFairnessAndQueueBounds() +{ + MiniHttpServer server; + QTemporaryDir directory; + QVERIFY(directory.isValid()); + auto config = configFor(directory); + config.max_active_total = 1; + config.max_active_per_host = 1; + config.max_active_per_client = 1; + config.max_pending_total = 3; + config.max_pending_per_client = 2; + TileNetworkManager manager(config); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + + manager.submit(request(server.url(QStringLiteral("/hold/first")), 1, 1, 1)); + QTRY_COMPARE_WITH_TIMEOUT(server.heldCount(), 1, 2000); + + auto client_one_coverage = request( + server.url(QStringLiteral("/order/client-one-coverage")), 1, 1, 2); + client_one_coverage.priority = TileRequestPriority::Coverage; + manager.submit(std::move(client_one_coverage)); + auto client_one_visible = request( + server.url(QStringLiteral("/order/client-one-visible")), 1, 1, 3); + client_one_visible.priority = TileRequestPriority::Visible; + manager.submit(std::move(client_one_visible)); + auto client_two_background = request( + server.url(QStringLiteral("/order/client-two-background")), 2, 1, 4); + client_two_background.priority = TileRequestPriority::Background; + manager.submit(std::move(client_two_background)); + manager.submit(request(server.url(QStringLiteral("/order/rejected")), 3, 1, 5)); + + QTRY_VERIFY_WITH_TIMEOUT(spy.size() >= 1, 2000); + QCOMPARE(resultAt(spy, 0).user_data, quint64(5)); + QCOMPARE(resultAt(spy, 0).outcome, TileNetworkResult::Outcome::Rejected); + + server.releaseOne(); + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 5, 4000); + auto const paths = server.paths(); + QCOMPARE(paths.at(0), QStringLiteral("/hold/first")); + QCOMPARE(paths.at(1), QStringLiteral("/order/client-two-background")); + QCOMPARE(paths.at(2), QStringLiteral("/order/client-one-coverage")); + QCOMPARE(paths.at(3), QStringLiteral("/order/client-one-visible")); +} + +void TileNetworkManagerTest::servesFreshDiskCacheOffline() +{ + MiniHttpServer server; + QTemporaryDir directory; + QVERIFY(directory.isValid()); + TileNetworkManager manager(configFor(directory)); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + auto const url = server.url(QStringLiteral("/cache")); + + manager.submit(request(url, 1, 1, 1)); + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 1, 3000); + QCOMPARE(resultAt(spy, 0).outcome, TileNetworkResult::Outcome::Success); + + manager.setOfflineMode(true); + manager.submit(request(url, 1, 2, 2)); + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 2, 3000); + auto const cached = resultAt(spy, 1); + QCOMPARE(cached.outcome, TileNetworkResult::Outcome::Success); + QVERIFY(cached.from_cache); + QCOMPARE(server.paths().count(QStringLiteral("/cache")), 1); +} + +void TileNetworkManagerTest::timesOutWithoutBlockingTheCaller() +{ + MiniHttpServer server; + QTemporaryDir directory; + QVERIFY(directory.isValid()); + auto config = configFor(directory); + config.transfer_timeout = std::chrono::milliseconds(40); + config.absolute_timeout = std::chrono::milliseconds(80); + TileNetworkManager manager(config); + QSignalSpy spy(&manager, &TileNetworkManager::finished); + QElapsedTimer elapsed; + elapsed.start(); + + manager.submit(request(server.url(QStringLiteral("/slow")), 1)); + QVERIFY(elapsed.elapsed() < 20); + QTRY_COMPARE_WITH_TIMEOUT(spy.size(), 1, 2000); + QCOMPARE( + resultAt(spy, 0).outcome, + TileNetworkResult::Outcome::TransientError); + QVERIFY(resultAt(spy, 0).error_string.contains(QStringLiteral("timed out"))); + server.releaseAll(); +} + +QTEST_GUILESS_MAIN(TileNetworkManagerTest) diff --git a/test/tile_network_manager_t.h b/test/tile_network_manager_t.h new file mode 100644 index 000000000..1bba245a1 --- /dev/null +++ b/test/tile_network_manager_t.h @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#ifndef OPENORIENTEERING_TILE_NETWORK_MANAGER_T_H +#define OPENORIENTEERING_TILE_NETWORK_MANAGER_T_H + +#include + +class TileNetworkManagerTest : public QObject +{ +Q_OBJECT + +private slots: + void rejectsUnsafeUrls(); + void handlesRedirectsEmptyTilesAndBodyLimits(); + void retriesTransientFailures(); + void cancelsClientGenerations(); + void enforcesFairnessAndQueueBounds(); + void servesFreshDiskCacheOffline(); + void timesOutWithoutBlockingTheCaller(); +}; + +#endif From a1c57bc301fe5d27af1fccdfc48365706dadc94d Mon Sep 17 00:00:00 2001 From: Ethan O'Connor Date: Thu, 16 Jul 2026 16:41:04 -0700 Subject: [PATCH 06/39] imagery: complete native online sources and OIC catalogs --- doc/manual/pages/online-imagery.md | 126 + doc/manual/pages/templates-index.md | 4 +- doc/manual/pages/templates.md | 4 +- doc/manual/pages/templates_menu.md | 26 +- doc/online-imagery-architecture.md | 163 + src/CMakeLists.txt | 7 + src/core/map_printer.cpp | 213 +- src/core/map_printer.h | 37 + src/gdal/kmz_groundoverlay_export.cpp | 534 ++- src/gdal/kmz_groundoverlay_export.h | 30 +- src/gui/imagery/catalog_import_dialog.cpp | 599 +++ src/gui/imagery/catalog_import_dialog.h | 93 + src/gui/imagery/catalog_manager_dialog.cpp | 398 ++ src/gui/imagery/catalog_manager_dialog.h | 64 + .../imagery_network_permissions_dialog.cpp | 254 + .../imagery_network_permissions_dialog.h | 56 + src/gui/imagery/imagery_source_model.cpp | 361 ++ src/gui/imagery/imagery_source_model.h | 83 + src/gui/imagery/online_imagery_dialog.cpp | 1158 +++++ src/gui/imagery/online_imagery_dialog.h | 144 + src/gui/map/map_editor.cpp | 145 + src/gui/map/map_editor.h | 12 + src/gui/print_progress_dialog.cpp | 20 +- src/gui/print_widget.cpp | 868 +++- src/gui/print_widget.h | 28 +- src/gui/widgets/template_list_widget.cpp | 8 + src/imagery/CMakeLists.txt | 24 +- src/imagery/arcgis_tile_service.cpp | 868 ++++ src/imagery/arcgis_tile_service.h | 82 + src/imagery/imagery_catalog_repository.cpp | 659 +++ src/imagery/imagery_catalog_repository.h | 191 + src/imagery/imagery_catalog_store.cpp | 1257 +++++ src/imagery/imagery_catalog_store.h | 174 + src/imagery/imagery_network_permissions.cpp | 219 + src/imagery/imagery_network_permissions.h | 62 + src/imagery/imagery_source.cpp | 11 + src/imagery/imagery_source.h | 20 + src/imagery/imagery_source_snapshot.h | 2 + src/imagery/manual_imagery_source.cpp | 470 ++ src/imagery/manual_imagery_source.h | 120 + src/imagery/oic_catalog.cpp | 4153 +++++++++++++++++ src/imagery/oic_catalog.h | 238 + src/imagery/tile_matrix_set.cpp | 28 + src/imagery/tile_network_manager.cpp | 1213 ++++- src/imagery/tile_network_manager.h | 83 +- src/render/overlay_scene.cpp | 9 +- src/render/render_ir.h | 2 + src/render/template_layer_planner.cpp | 202 +- src/render/template_layer_planner.h | 3 + src/templates/online_raster_template.cpp | 3890 +++++++++++++++ src/templates/online_raster_template.h | 486 ++ src/templates/raster_resource_manager.cpp | 28 +- src/templates/raster_resource_manager.h | 14 +- src/templates/template.cpp | 134 + src/templates/template.h | 106 + src/templates/template_image.h | 20 +- src/templates/template_map.cpp | 75 + src/templates/template_map.h | 6 + src/templates/template_table_model.cpp | 186 +- src/templates/template_table_model.h | 9 + test/CMakeLists.txt | 20 + test/arcgis_tile_service_t.cpp | 557 +++ test/arcgis_tile_service_t.h | 28 + .../valid/custom-dyadic-epsg2927.oic | 125 + test/data/imagery-catalogs/valid/minimal.oic | 17 + test/imagery_catalog_repository_t.cpp | 620 +++ test/imagery_catalog_repository_t.h | 25 + test/imagery_catalog_store_t.cpp | 634 +++ test/imagery_catalog_store_t.h | 28 + test/imagery_core_t.cpp | 19 + test/imagery_core_t.h | 1 + test/imagery_network_permissions_t.cpp | 215 + test/imagery_network_permissions_t.h | 21 + test/imagery_source_model_t.cpp | 373 ++ test/imagery_source_model_t.h | 24 + test/manual_imagery_source_t.cpp | 348 ++ test/manual_imagery_source_t.h | 26 + test/map_printer_t.cpp | 901 +++- test/oic_catalog_t.cpp | 1224 +++++ test/oic_catalog_t.h | 42 + test/online_imagery_dialog_t.cpp | 323 ++ test/online_imagery_dialog_t.h | 24 + test/online_raster_template_t.cpp | 1399 ++++++ test/raster_resource_manager_t.cpp | 43 + test/render_ir_t.cpp | 3 +- test/template_layer_planner_t.cpp | 153 + test/template_layer_planner_t.h | 3 + test/template_t.cpp | 111 + test/tile_network_manager_t.cpp | 626 ++- test/tile_network_manager_t.h | 13 + test/vello_renderer_t.cpp | 5 +- 91 files changed, 28183 insertions(+), 247 deletions(-) create mode 100644 doc/manual/pages/online-imagery.md create mode 100644 doc/online-imagery-architecture.md create mode 100644 src/gui/imagery/catalog_import_dialog.cpp create mode 100644 src/gui/imagery/catalog_import_dialog.h create mode 100644 src/gui/imagery/catalog_manager_dialog.cpp create mode 100644 src/gui/imagery/catalog_manager_dialog.h create mode 100644 src/gui/imagery/imagery_network_permissions_dialog.cpp create mode 100644 src/gui/imagery/imagery_network_permissions_dialog.h create mode 100644 src/gui/imagery/imagery_source_model.cpp create mode 100644 src/gui/imagery/imagery_source_model.h create mode 100644 src/gui/imagery/online_imagery_dialog.cpp create mode 100644 src/gui/imagery/online_imagery_dialog.h create mode 100644 src/imagery/arcgis_tile_service.cpp create mode 100644 src/imagery/arcgis_tile_service.h create mode 100644 src/imagery/imagery_catalog_repository.cpp create mode 100644 src/imagery/imagery_catalog_repository.h create mode 100644 src/imagery/imagery_catalog_store.cpp create mode 100644 src/imagery/imagery_catalog_store.h create mode 100644 src/imagery/imagery_network_permissions.cpp create mode 100644 src/imagery/imagery_network_permissions.h create mode 100644 src/imagery/manual_imagery_source.cpp create mode 100644 src/imagery/manual_imagery_source.h create mode 100644 src/imagery/oic_catalog.cpp create mode 100644 src/imagery/oic_catalog.h create mode 100644 src/templates/online_raster_template.cpp create mode 100644 src/templates/online_raster_template.h create mode 100644 test/arcgis_tile_service_t.cpp create mode 100644 test/arcgis_tile_service_t.h create mode 100644 test/data/imagery-catalogs/valid/custom-dyadic-epsg2927.oic create mode 100644 test/data/imagery-catalogs/valid/minimal.oic create mode 100644 test/imagery_catalog_repository_t.cpp create mode 100644 test/imagery_catalog_repository_t.h create mode 100644 test/imagery_catalog_store_t.cpp create mode 100644 test/imagery_catalog_store_t.h create mode 100644 test/imagery_network_permissions_t.cpp create mode 100644 test/imagery_network_permissions_t.h create mode 100644 test/imagery_source_model_t.cpp create mode 100644 test/imagery_source_model_t.h create mode 100644 test/manual_imagery_source_t.cpp create mode 100644 test/manual_imagery_source_t.h create mode 100644 test/oic_catalog_t.cpp create mode 100644 test/oic_catalog_t.h create mode 100644 test/online_imagery_dialog_t.cpp create mode 100644 test/online_imagery_dialog_t.h create mode 100644 test/online_raster_template_t.cpp diff --git a/doc/manual/pages/online-imagery.md b/doc/manual/pages/online-imagery.md new file mode 100644 index 000000000..c172e06c2 --- /dev/null +++ b/doc/manual/pages/online-imagery.md @@ -0,0 +1,126 @@ +--- +title: Online imagery +keywords: Templates, Imagery, OIC, XYZ, TMS, ArcGIS +parent: Templates and Data +nav_order: 0.2 +last_modified_date: 16 July 2026 +--- + +Online imagery is a georeferenced raster template which loads only the tiles +needed for the current view or output. The map must be georeferenced, but it +does not need to be saved before imagery is added. + +Open **Templates → Add online imagery…**, or use **Add template… → Add online +imagery…** in the template setup window. + +## Choosing a source + +The source browser lists sources from installed OpenOrienteering Imagery +Catalog (OIC) files. Search matches source names, descriptions, identifiers and +categories. Unsupported catalog entries remain visible with an explanation. + +Selecting a source shows its catalog revision, imagery dates, coordinate +reference system, zoom range, request hosts, attribution and terms. The +template name may be changed without changing the catalog source identity. + +When the source is added, Mapper stores a complete, checksummed source snapshot +inside the map. Updating or removing the installed catalog does not silently +change an existing map. + +## Entering a URL + +Choose **Enter a tile or service URL…** to add a source which is not in a +catalog. + +Direct tiled sources use an HTTP or HTTPS URL containing all three +`{z}`, `{x}` and `{y}` placeholders. The documented `${z}`, `${x}` and `${y}` +spellings are accepted and normalized. Advanced settings make the row scheme +(XYZ or TMS), zoom range, 256 or 512 pixel tile size, HTTP Referer, empty-tile +status codes and attribution explicit. + +Cached ArcGIS MapServer and ImageServer links are checked by reading their +published `f=pjson` metadata. Mapper derives the tile grid, levels, origin, +coordinate reference system, format and endpoint from the service rather than +assuming Web Mercator or a fixed zoom range. + +WMS, WMTS and remote GeoTIFF links are recognized but are not currently +executed by the native tiled raster template. Mapper reports them as +unsupported instead of guessing an incompatible source configuration. + +## Catalogs + +Choose **Manage catalogs…** to import, update or remove OIC catalogs. + +Before installation, Mapper shows: + +- catalog identity, revision, publisher, origin and document SHA-256; +- added, removed, operationally changed and metadata-only source counts; +- usable, invalid and unsupported source counts; +- request hosts and duplicate-source counts; +- warnings for downgrade, republished revisions, HTTP, local-network + endpoints, registration corrections and credential-like query parameters. + +Catalog updates use ETag and Last-Modified validators when available. An +interrupted update cannot replace the current catalog with a partial file, and +the previous snapshot is retained for recovery. + +## Network privacy and credentials + +Cookies and HTTP authentication are not used for imagery requests. Redirects, +response sizes and timeouts are bounded. HTTPS-to-HTTP redirects are rejected. + +Private and local-network origins require an explicit permission stored only +on the current installation. A catalog or map file cannot grant this +permission. This includes private, shared, link-local, documentation, +benchmarking and other special-purpose address ranges, whether written +directly in a URL or returned by DNS. + +Choose **Templates → Imagery network permissions…** to review or revoke saved +permissions. The same dialog lists origins from requests Mapper blocked after +DNS resolution; approving one is always a deliberate user action. Merely +opening a map may add a blocked origin to the review list, but never grants it +network access or opens a permission prompt. + +Mapper's hostname preflight is defense in depth: Qt's network stack performs +its own connection-time resolution, so operating-system firewall and network +policy remain the hard isolation boundary. + +URLs containing token-like query parameters are not added to a recent-sources +list. Mapper warns before embedding the complete endpoint in the map. Anyone +who can read that map file may be able to read the endpoint, including its +query parameters. + +## Offline use + +Enable **Templates → Work offline for imagery** to prevent network access and +use only tiles already present in the local HTTP cache. Turning offline mode +off allows missing tiles to be requested again. + +Referer-dependent imagery is not retained in the shared offline cache because +serving it later without the original request context can be incorrect. + +## Printing and export + +Screen display may temporarily use a coarser cached parent tile while a sharper +tile loads. Print, PDF, image and KML/KMZ output do not use provisional parents. +Mapper prepares the exact requested source level before opening the output +paint engine. A missing or permanently failed tile stops exact output with an +error instead of silently producing an incomplete map. + +Large translucent output is prepared as bounded, non-overlapping atlas chunks. +Chunk sizes account for both source and reprojected output geometry, and +neighbor pixels are sampled before each chunk is cropped to its own coverage. +This retains seam-free alpha blending without requiring one page-sized raster +allocation. Exact preparation also reserves the renderer snapshot memory it +will need before reporting that output is ready. + +Exact-imagery preparation and multi-tile KML/KMZ export can be canceled. A +user cancellation is reported as canceled rather than as a rendering failure. +Incomplete staged PDF, image and KMZ files are discarded. Image exports with +a world file use a destination lock, recovery journal and old-image backup; if +the two-file commit is interrupted, Mapper restores the matching old pair or +finishes cleanup before the next export. Direct KML image sidecars are +published through a unique asset directory; unreferenced directories left by +a process interruption are recovered on the next export. Interrupted KML/KMZ +staging files are scoped to their absolute destination and safely removed when +that destination is exported again. diff --git a/doc/manual/pages/templates-index.md b/doc/manual/pages/templates-index.md index d07487738..09a4f9308 100644 --- a/doc/manual/pages/templates-index.md +++ b/doc/manual/pages/templates-index.md @@ -9,6 +9,9 @@ has_children: true [Introduction to Templates](templates.md){: .subpage} Types of templates, loading and positioning +[Online imagery](online-imagery.md){: .subpage} +Streaming tiled imagery and OIC catalogs + [Adjusting template positions](template_adjust.md){: .subpage} For non-georeferenced templates @@ -17,4 +20,3 @@ Vectorizing line features in raster graphics templates. [Geospatial data](gdal.md){: .subpage} Geospatial raster data and vector data support based on GDAL. - diff --git a/doc/manual/pages/templates.md b/doc/manual/pages/templates.md index df42e2224..f79991dc0 100644 --- a/doc/manual/pages/templates.md +++ b/doc/manual/pages/templates.md @@ -20,7 +20,9 @@ OpenOrienteering Mapper supports the following file formats to be loaded as temp - [Raster images](#image-templates) (bmp, jpg, png, gif and [formats supported by GDAL](gdal.md)) - Geospatial vector data (cf. [geospatial data support with GDAL](gdal.md)) - [GPX tracks](#track-templates) (gpx) - - [Map files](#map-templates) (omap, xmap, ocd) + - [Map files](#map-templates) (omap, xmap, ocd) + - [Online imagery](online-imagery.md) (OIC catalog sources, XYZ/TMS tiles and + cached ArcGIS services) Additionally, templates can be classified into **georeferenced** and **non-georeferenced** templates. For georeferenced templates, information about the exact positioning of the template in a known world coordinate system is available - see [georeferencing](georeferencing.md). This way, they can be positioned on the map automatically provided that the map is georeferenced too. For non-georeferenced templates, this information is not available, so they have to be [positioned manually](#positioning). diff --git a/doc/manual/pages/templates_menu.md b/doc/manual/pages/templates_menu.md index 441c52081..19f3732ae 100644 --- a/doc/manual/pages/templates_menu.md +++ b/doc/manual/pages/templates_menu.md @@ -17,9 +17,29 @@ last_modified_date: 20 January 2018 Switches the display of the [template setup window](templates.md#setup) on the map screen. This provides access to the functions required to open, scale, and position a template, and to close it. ---- - -#### Open template... +--- + +#### Add online imagery... + +Opens the [online imagery source browser](online-imagery.md). Sources may come +from installed OIC catalogs, a direct XYZ/TMS URL, or a cached ArcGIS service. + + +#### Manage imagery catalogs... + +Imports, reviews, updates and removes OIC catalogs. Existing maps keep their +embedded source snapshots when a catalog changes. + + +#### Work offline for imagery + +Prevents imagery network access and uses only tiles already present in the +local cache. + + +--- + +#### Open template... Opens a template file directly without going through the setup window. diff --git a/doc/online-imagery-architecture.md b/doc/online-imagery-architecture.md new file mode 100644 index 000000000..ae06a8b03 --- /dev/null +++ b/doc/online-imagery-architecture.md @@ -0,0 +1,163 @@ +# Online imagery architecture + +This feature intentionally uses a native tiled raster path rather than +regenerating GDAL WMS XML sidecars. + +GDAL remains the right abstraction for local geospatial files, broad raster +format support and export. Current GDAL releases also offer thread-safe raster +dataset wrappers for read-only access. Those capabilities do not provide the +application contracts needed here: view-driven request cancellation, source +fairness, provisional parent tiles, alpha-seam prevention, renderer resource +admission, exact-output preflight, catalog identity or installation-local +network permissions. + +References: + +- [GDAL WMS driver](https://gdal.org/en/stable/drivers/raster/wms.html) +- [GDAL WMTS driver](https://gdal.org/en/stable/drivers/raster/wmts.html) +- [GDAL multithreading guidance](https://gdal.org/en/stable/user/multithreading.html) +- [GDAL RFC 101 thread-safe raster datasets](https://gdal.org/en/stable/development/rfc/rfc101_raster_dataset_threadsafety.html) +- [Qt 6 QNetworkAccessManager](https://doc.qt.io/qt-6/qnetworkaccessmanager.html) +- [Qt 6 QHostAddress classification](https://doc.qt.io/qt-6/qhostaddress.html) +- [IANA IPv4 special-purpose registry](https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml) +- [IANA IPv6 special-purpose registry](https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml) +- [OGC Two Dimensional Tile Matrix Set](https://docs.ogc.org/is/17-083r4/17-083r4.pdf) +- [OGC API - Tiles - Part 1: Core](https://docs.ogc.org/is/20-057/20-057.pdf) + +## Layering + +```text +OIC/manual/ArcGIS definitions + | + v +ResolvedImagerySource -> checksummed embedded snapshot + | + v +OnlineRasterTemplate + | requests | immutable render descriptions + v v +TileNetworkManager TemplateLayerPlanner + | | + v v +bounded HTTP/decode Qt reference / Vello screen backend +``` + +Installed catalogs are an input index, not a runtime dependency of a map. +`ImageryCatalogRepository` publishes immutable snapshots and stable +`catalog-id/source-id/catalog-sha256` handles. Selecting a source copies its +resolved definition into an `ImagerySourceSnapshot`; later catalog updates do +not mutate the map. + +## Catalog durability + +Catalog bytes are immutable and addressed by document SHA-256: + +```text +imagery-catalogs// + current.json + snapshots// + catalog.oic + snapshot.json +``` + +`QLockFile` serializes writers across Mapper processes. A `QSaveFile` update of +`current.json` is the only activation step, so catalog bytes and transport +metadata cannot come from different revisions after a crash. The previous +snapshot is retained and used if the active snapshot is damaged. Legacy +mapper-coc `catalog.oic`/`state.json` installations are read and migrated on +the next write. + +## HTTP ownership and policy + +One application-scoped `QNetworkAccessManager` and `QNetworkDiskCache` live on +a dedicated event-loop thread. Requests are bounded globally, per host and per +source, and are owner-fair before priority and distance ordering. + +The manager performs asynchronous DNS checks before contacting a destination. +Literal and resolved addresses share one conservative public-destination +predicate. It supplements Qt's broad `isGlobal()` category with the IANA +special-purpose ranges, including RFC 1918, ULA, shared, benchmarking, +documentation and reserved space. IPv6 authorities are canonicalized through +`QUrl`, preserving brackets and exact permission scope. Cookies, embedded +credentials and HTTP authentication are disabled. Every redirect is +revalidated; HTTPS downgrade and unapproved private destinations are rejected. +Response size, decompression, first-byte, transfer and absolute time limits are +enforced while streaming. + +The DNS check deliberately has a narrow claim. `QNetworkAccessManager` +resolves the connection independently, so Mapper shortens but cannot eliminate +the rebinding interval without replacing Qt's TLS, HTTP and cache stack. +Endpoint trust and operating-system network policy remain the hard security +boundary. + +Tile images use the shared disk cache. Catalog and service JSON uses +conditional requests but bypasses the tile cache so HTTP 304 remains visible +to the catalog repository. Empty-tile cache keys retain only a fixed-size +digest of the URL, Referer and representation policy. Offline and +private-permission transitions advance facade generations which are checked +again immediately before queued results reach the UI, closing worker/event +ordering races while still allowing genuine cache-only successes. +Private-origin approvals are stored in local settings and can only be created +by explicit UI action. + +## Rendering + +Opaque tiles become direct image-to-map primitives and can remain independent. +This gives the screen backend granular resource admission and avoids building +a viewport-sized CPU mosaic. + +Independent translucent tiles would blend their shared edges more than once. +A translucent screen window is therefore composed once into a retained +premultiplied atlas. Exact output divides large translucent extents into +non-overlapping, tile-aligned atlas chunks. Chunk subdivision accounts for both +source pixels and the projected destination geometry. For cross-CRS output, +each worker reads a true one-pixel strip from adjacent chunks before bilinear +resampling, then masks the result back to its half-open core domain. The +neighbor pixels are filter support only, preventing both transparent sampling +seams and double-alpha overlap. Bounded workers build adaptive inverse-warp +grids and resample those chunks off the UI thread. + +The screen may use an exact cropped parent tile as a provisional fallback. +Output sessions pin the requested zoom and derived resources and reject +provisional imagery. Preparation finishes before a printer/PDF/image/KMZ paint +engine is opened, so cancellation or missing exact imagery cannot leave a +nominally successful partial export. + +File output is committed transactionally. PDF and standalone image writers +target `QSaveFile`. An image plus world-file export stages both files under a +per-destination process lock; a bounded journal and sibling image backup make +the unavoidable two-file commit recoverable. Rollback restores the world file +before republishing its matching image, so an interruption never exposes old +pixels with new georeferencing. + +KMZ archives are completed in sibling staging files before the selected +destination is atomically replaced. Direct KML sidecars are written into a +unique, destination-scoped exporter-owned asset directory and the staged KML +document is published last. A per-destination process lock serializes writers. +Staging filenames include the SHA-256 of the absolute destination; while +holding that lock, the next export removes only matching regular, non-symlink +staging files left by an interrupted writer. The published KML identifies the +live asset directory, so unreferenced directories for that destination are +likewise recovered without touching another KML document's assets. + +## Memory and cancellation + +Network bodies, DNS waiters, decode jobs, completion delivery, decoded tiles +and derived atlases have separate bounded queues and working-set checks. +Retained raster memory is budgeted across all online templates. Accounting +leases follow shared pixels into queued atlas workers and immutable renderer +snapshots, so cache eviction never reports memory as released while another +owner still holds it. Once exact translucent atlases are complete, their +source-tile pins are released and renderer snapshot memory is reserved before +preflight reports Ready. Admission uses application-wide least-recently-used +eviction before applying backpressure, continuing past externally pinned +entries when reclaimable cache entries remain. View generations remove +superseded network, decode and warp work from their bounded queues. Cached +source tiles survive pans and map georeferencing changes; only derived +map-space atlases are invalidated. + +Empty tiles use a compact typed cache state rather than a full transparent +image. Retry state records terminal and policy failures per equivalent +endpoint: transient failures rotate without poisoning a backup, while +approving any previously blocked endpoint revives affected tiles. Offline +misses wait for offline mode to end instead of forming a retry loop. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e5bb2ff49..b8bea3936 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,6 +195,12 @@ set(Mapper_Common_SRCS gui/text_browser_dialog.cpp gui/touch_cursor.cpp gui/util_gui.cpp + + gui/imagery/catalog_import_dialog.cpp + gui/imagery/catalog_manager_dialog.cpp + gui/imagery/imagery_network_permissions_dialog.cpp + gui/imagery/imagery_source_model.cpp + gui/imagery/online_imagery_dialog.cpp gui/map/new_map_dialog.cpp gui/map/map_dialog_scale.cpp @@ -249,6 +255,7 @@ set(Mapper_Common_SRCS templates/paint_on_template_feature.cpp templates/paint_on_template_tool.cpp + templates/online_raster_template.cpp templates/raster_resource_manager.cpp templates/template.cpp templates/template_adjust.cpp diff --git a/src/core/map_printer.cpp b/src/core/map_printer.cpp index 247950d84..17609838a 100644 --- a/src/core/map_printer.cpp +++ b/src/core/map_printer.cpp @@ -28,6 +28,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -488,7 +492,10 @@ MapPrinter::MapPrinter(Map& map, const MapView* view, QObject* parent) connect(&map.getGeoreferencing(), &Georeferencing::transformationChanged, this, &MapPrinter::mapScaleChanged); } -MapPrinter::~MapPrinter() = default; +MapPrinter::~MapPrinter() +{ + finishOutput(true); +} void MapPrinter::saveConfig() const @@ -583,26 +590,39 @@ std::unique_ptr MapPrinter::makePrinter() const std::unique_ptr MapPrinter::makePdfWriter(const QString& filename) const { auto writer = std::make_unique(filename); + configurePdfWriter(*writer); + return writer; +} + +std::unique_ptr MapPrinter::makePdfWriter(QIODevice* device) const +{ + Q_ASSERT(device); + auto writer = std::make_unique(device); + configurePdfWriter(*writer); + return writer; +} + +void MapPrinter::configurePdfWriter(QPdfWriter& writer) const +{ if (separationsModeSelected()) - writer->setColorModel(QPdfWriter::ColorModel::Grayscale); + writer.setColorModel(QPdfWriter::ColorModel::Grayscale); else if (options.color_mode == MapPrinterOptions::DeviceCmyk) - writer->setColorModel(QPdfWriter::ColorModel::CMYK); + writer.setColorModel(QPdfWriter::ColorModel::CMYK); else - writer->setColorModel(QPdfWriter::ColorModel::RGB); + writer.setColorModel(QPdfWriter::ColorModel::RGB); - writer->setResolution(options.resolution); + writer.setResolution(options.resolution); if (page_format.page_size == QPageSize::Custom) { - writer->setPageSize(QPageSize{page_format.paper_dimensions, QPageSize::Millimeter}); - writer->setPageOrientation(QPageLayout::Portrait); + writer.setPageSize(QPageSize{page_format.paper_dimensions, QPageSize::Millimeter}); + writer.setPageOrientation(QPageLayout::Portrait); } else { - writer->setPageSize(QPageSize{page_format.page_size}); - writer->setPageOrientation(page_format.orientation); + writer.setPageSize(QPageSize{page_format.page_size}); + writer.setPageOrientation(page_format.orientation); } - writer->setPageMargins(QMarginsF{}, QPageLayout::Millimeter); - return writer; + writer.setPageMargins(QMarginsF{}, QPageLayout::Millimeter); } bool MapPrinter::isPrinter() const noexcept @@ -976,6 +996,135 @@ void MapPrinter::takePrinterSettings(const QPrinter* printer) } +bool MapPrinter::prepareOutput() +{ + return prepareOutput(print_area); +} + +bool MapPrinter::prepareOutput(const QRectF& map_extent) +{ + if (output_preparation_active) + { + if (output_preparation_extent.contains(map_extent)) + return true; + output_error = tr( + "Exact output is already prepared for a different map area."); + return false; + } + output_error.clear(); + output_templates.clear(); + cancel_print_map = false; + if (!map_extent.isValid() || map_extent.isEmpty()) + { + output_error = tr("The exact output area is invalid."); + return false; + } + if (!options.show_templates || separationsModeSelected()) + { + output_preparation_extent = map_extent; + output_preparation_active = true; + return true; + } + + for (int index = 0; index < map.getNumTemplates(); ++index) + { + auto* source = map.getTemplate(index); + if (source->getTemplateState() != Template::Loaded) + continue; + if (view) + { + auto const visibility = view->getTemplateVisibility(source); + if (!visibility.visible || visibility.opacity <= 0) + continue; + } + output_templates.push_back(source); + } + if (output_templates.empty()) + { + output_preparation_extent = map_extent; + output_preparation_active = true; + return true; + } + + auto const pixels_per_map_unit = + (options.resolution / 25.4) * scale_adjustment; + QElapsedTimer elapsed; + elapsed.start(); + constexpr qint64 timeout_ms = 120'000; + while (true) + { + qsizetype ready = 0; + qsizetype total = 0; + bool pending = false; + for (auto* source : output_templates) + { + auto const preparation = source->prepareForOutput( + map_extent, pixels_per_map_unit); + ready += preparation.ready_resources; + total += preparation.total_resources; + if (preparation.state + == OutputRenderPreparation::State::Failed) + { + output_error = preparation.error.isEmpty() + ? tr("Failed to prepare an exact template resource.") + : preparation.error; + finishOutput(true); + emit printProgress(100, tr("Error")); + return false; + } + pending |= preparation.state + == OutputRenderPreparation::State::Pending; + } + if (!pending) + { + output_preparation_extent = map_extent; + output_preparation_active = true; + return true; + } + if (cancel_print_map) + { + output_error = tr("Canceled"); + finishOutput(true); + emit printProgress(100, tr("Canceled")); + return false; + } + if (elapsed.elapsed() >= timeout_ms) + { + output_error = tr( + "Timed out while preparing exact online imagery."); + finishOutput(true); + emit printProgress(100, tr("Error")); + return false; + } + + auto const progress = total > 0 + ? std::clamp( + int(15 * ready / total), 1, 14) + : 1; + emit printProgress( + progress, + tr("Preparing online imagery (%1 of %2 tiles)...") + .arg(ready) + .arg(total)); + QCoreApplication::processEvents( + QEventLoop::AllEvents | QEventLoop::WaitForMoreEvents, + QDeadlineTimer(25)); + } +} + +void MapPrinter::finishOutput(bool cancelled) +{ + // The final page scene owns immutable raster snapshots. Release it before + // dropping the source-side output pins so exact-output memory does not remain + // charged for the lifetime of a long-lived print widget. + template_layer_planner.clear(); + for (auto* source : output_templates) + source->finishOutputPreparation(cancelled); + output_templates.clear(); + output_preparation_active = false; + output_preparation_extent = {}; +} + void MapPrinter::drawPage(QPainter* device_painter, const QRectF& page_extent, QImage* page_buffer) const { @@ -1002,6 +1151,15 @@ void MapPrinter::drawPage(QPainter* device_painter, const QRectF& page_extent, c | QPainter::SmoothPixmapTransform; const auto page_region_used = page_extent.intersected(print_area); + if (!output_templates.empty() + && !page_region_used.isEmpty() + && !output_preparation_extent.contains(page_region_used)) + { + output_error = tr( + "A page was rendered outside the prepared exact-output area."); + device_painter->end(); + return; + } const auto output_scaling = units_per_mm * scale_adjustment; const auto output_request = render::RenderRequest { render::fromQRectF(page_region_used), @@ -1018,6 +1176,8 @@ void MapPrinter::drawPage(QPainter* device_painter, const QRectF& page_extent, c ); if (!template_layers.complete) { + output_error = tr( + "An exact template resource became unavailable during rendering."); device_painter->end(); return; } @@ -1387,13 +1547,19 @@ bool MapPrinter::printMap(QPrinter* printer) // We need to use them for printing. printer->setFullPage(true); takePrinterSettings(printer); + if (!prepareOutput()) + return false; QPainter painter(printer); #if defined(Q_OS_WIN) // Invalid printer drivers (notably under Wine) may report no resolution. if (printer->resolution() == 0) + { + output_error = tr("The printer reported an invalid resolution."); + finishOutput(true); return false; + } if (printer->paintEngine()->type() == QPaintEngine::Picture) { @@ -1404,13 +1570,19 @@ bool MapPrinter::printMap(QPrinter* printer) } #endif - return printPages(printer, &painter, 1); + auto const result = printPages(printer, &painter, 1); + finishOutput(cancel_print_map || !result); + return result; } bool MapPrinter::printMap(QPdfWriter* writer, int copy_count) { + if (!prepareOutput()) + return false; QPainter painter(writer); - return printPages(writer, &painter, copy_count); + auto const result = printPages(writer, &painter, copy_count); + finishOutput(cancel_print_map || !result); + return result; } bool MapPrinter::printPages(QPagedPaintDevice* device, QPainter* painter, int copy_count) @@ -1455,7 +1627,13 @@ bool MapPrinter::printPages(QPagedPaintDevice* device, QPainter* painter, int co if (need_new_page) { - device->newPage(); + if (!device->newPage()) + { + output_error = tr( + "Could not begin a new output page."); + painter->end(); + break; + } } const QRectF page_extent{QPointF{hpos, vpos}, extent_size}; @@ -1475,10 +1653,17 @@ bool MapPrinter::printPages(QPagedPaintDevice* device, QPainter* painter, int co if (cancel_print_map) { + output_error = ::OpenOrienteering::MapPrinter::tr("Canceled"); emit printProgress(100, ::OpenOrienteering::MapPrinter::tr("Canceled")); + return false; } else if (!painter->isActive()) { + if (output_error.isEmpty()) + { + output_error = ::OpenOrienteering::MapPrinter::tr( + "The output paint device stopped during rendering."); + } emit printProgress(100, ::OpenOrienteering::MapPrinter::tr("Error")); return false; } diff --git a/src/core/map_printer.h b/src/core/map_printer.h index d83d048cb..921999b6f 100644 --- a/src/core/map_printer.h +++ b/src/core/map_printer.h @@ -46,6 +46,7 @@ template class QHash; class QImage; +class QIODevice; class QPainter; class QPagedPaintDevice; class QPdfWriter; @@ -348,10 +349,39 @@ Q_OBJECT /** Creates a PDF writer configured according to the current settings. */ std::unique_ptr makePdfWriter(const QString& filename) const; + + /** + * Creates a PDF writer which writes to an already-open device. + * + * The caller retains ownership of the device and must keep it alive until + * the writer is destroyed. + */ + std::unique_ptr makePdfWriter(QIODevice* device) const; /** Takes the settings from the given printer, * and generates signals for changing properties. */ void takePrinterSettings(const QPrinter* printer); + + /** + * Resolves all exact template resources needed by the configured output. + * + * This bounded, cancellable preflight runs before any print paint engine is + * opened. Call finishOutput() after the final drawPage(), or rely on + * printMap() which manages the pair automatically. + */ + bool prepareOutput(); + + /** + * Resolves exact template resources for an explicit map extent. + * + * Use this overload when an exporter renders an area different from the + * configured print area. The supplied extent must cover every map clip + * rendered before finishOutput() is called. + */ + bool prepareOutput(const QRectF& map_extent); + void finishOutput(bool cancelled = false); + const QString& outputError() const noexcept { return output_error; } + bool outputWasCanceled() const noexcept { return cancel_print_map; } /** Prints the map to the given printer. * @@ -493,6 +523,9 @@ public slots: /** Updates the scale adjustment and page breaks. */ void mapScaleChanged(); + /** Applies this printer's PDF output settings to an existing writer. */ + void configurePdfWriter(QPdfWriter& writer) const; + /** Renders all configured pages to an active painter. */ bool printPages(QPagedPaintDevice* device, QPainter* painter, int copy_count); @@ -504,6 +537,10 @@ public slots: std::vector h_page_pos; std::vector v_page_pos; bool cancel_print_map = false; + bool output_preparation_active = false; + QRectF output_preparation_extent; + mutable QString output_error; + std::vector output_templates; mutable render::FramePlanner frame_planner; mutable render::TemplateLayerPlanner template_layer_planner; }; diff --git a/src/gdal/kmz_groundoverlay_export.cpp b/src/gdal/kmz_groundoverlay_export.cpp index a4a2736b6..5f0086232 100644 --- a/src/gdal/kmz_groundoverlay_export.cpp +++ b/src/gdal/kmz_groundoverlay_export.cpp @@ -32,16 +32,23 @@ #include #include #include +#include +#include #include +#include #include #include #include #include #include +#include #include #include #include #include +#include +#include +#include #include #include "mapper_config.h" @@ -130,7 +137,32 @@ QRectF boundingBoxMap(const Georeferencing& georef, const QRectF& extent_lonlat) ); } - +QByteArray directAssetPrefix( + const QByteArray& output_filepath) +{ + return QByteArray(".mapper-kml-") + + QCryptographicHash::hash( + output_filepath, + QCryptographicHash::Sha256) + .toHex() + .left(16) + + QByteArray("-assets-"); +} + +QByteArray stagingFilePrefix( + const QByteArray& output_filepath, + bool kmz) +{ + return QByteArray(".mapper-") + + (kmz ? QByteArray("kmz-") : QByteArray("kml-")) + + QCryptographicHash::hash( + output_filepath, + QCryptographicHash::Sha256) + .toHex() + + QByteArray("-stage-"); +} + + } // namespace @@ -160,16 +192,9 @@ KmzGroundOverlayExport::KmzGroundOverlayExport(const QString& path, const Map& m , is_kmz(path.endsWith(QLatin1String(".kmz"), Qt::CaseInsensitive)) { auto const fileinfo = QFileInfo(path); - if (is_kmz) - { - basepath_utf8 = "/vsizip/" + fileinfo.absoluteFilePath().toUtf8(); - doc_filepath_utf8 = basepath_utf8 + "/doc.kml"; - } - else - { + output_filepath_utf8 = fileinfo.absoluteFilePath().toUtf8(); + if (!is_kmz) basepath_utf8 = fileinfo.absolutePath().toUtf8(); - doc_filepath_utf8 = fileinfo.absoluteFilePath().toUtf8(); - } } @@ -183,11 +208,31 @@ QString KmzGroundOverlayExport::errorString() const return error_message; } +bool KmzGroundOverlayExport::wasCanceled() const +{ + return cancelled + || (progress_observer && progress_observer->wasCanceled()); +} -bool KmzGroundOverlayExport::doExport(const MapPrinter& map_printer, int tile_width_px) + +bool KmzGroundOverlayExport::doExport(MapPrinter& map_printer, int tile_width_px) { error_message.clear(); - + cancelled = false; + staging_filepath_utf8.clear(); + direct_assets_relative_utf8.clear(); + tile_progress_maximum = 1; + QLockFile output_lock( + QString::fromUtf8(output_filepath_utf8) + + QStringLiteral(".mapper-export.lock")); + if (!output_lock.tryLock()) + { + error_message = tr( + "Another process is already exporting to this destination."); + return false; + } + pruneStagedOutputFiles(); + #ifdef QT_PRINTSUPPORT_LIB auto const& georef = map.getGeoreferencing(); if (georef.getState() != Georeferencing::Geospatial) @@ -207,20 +252,34 @@ bool KmzGroundOverlayExport::doExport(const MapPrinter& map_printer, int tile_wi auto const bounding_box_lonlat = boundingBoxLonLat(georef, map_printer.getPrintArea()); auto const bounding_box_map = boundingBoxMap(georef, bounding_box_lonlat); auto const metrics = makeMetrics(bounding_box_map.size(), resolution, tile_width_px); - auto const tiles = makeTiles(map_printer.getPrintArea(), metrics); - - if (!is_kmz) + auto tiles = makeTiles(map_printer.getPrintArea(), metrics); + + auto const exact_output_extent = + renderExtent(tiles).intersected( + map_printer.getPrintArea()); + if (!map_printer.prepareOutput( + exact_output_extent.isEmpty() + ? map_printer.getPrintArea() + : exact_output_extent)) { - // When not creating a KMZ container file, do not overwrite existing image files. - for (auto const& tile : tiles) - { - auto const filepath_utf8 = QByteArray(basepath_utf8 + '/' + tile.filepath); - if (GdalFile::exists(filepath_utf8)) - { - error_message = tr("%1 already exists.").arg(QString::fromUtf8(filepath_utf8)); - return false; - } - } + cancelled = + map_printer.outputWasCanceled() || wasCanceled(); + error_message = cancelled + ? tr("Canceled") + : map_printer.outputError(); + return false; + } + if (wasCanceled()) + { + cancelled = true; + error_message = tr("Canceled"); + map_printer.finishOutput(true); + return false; + } + if (!beginStagedOutput(tiles)) + { + map_printer.finishOutput(true); + return false; } VSIErrorReset(); @@ -230,24 +289,48 @@ bool KmzGroundOverlayExport::doExport(const MapPrinter& map_printer, int tile_wi if (is_kmz && !kmz_file) { error_message = QString::fromUtf8(VSIGetLastErrorMsg()); + map_printer.finishOutput(true); + cleanupPartialOutput(tiles); return false; } // Create KML document and image files. setMaximumProgress(int(tiles.size())); - auto const result = doExport(map_printer, metrics, tiles); + auto result = doExport(map_printer, metrics, tiles); + cancelled = + wasCanceled() || map_printer.outputWasCanceled(); + map_printer.finishOutput(!result || cancelled); if (is_kmz) { - VSIFCloseL(kmz_file); - if (wasCanceled()) - VSIUnlink(basepath_utf8); + if (VSIFCloseL(kmz_file) != 0) + { + if (result && !cancelled) + { + error_message = tr( + "Failed to finish the KMZ archive."); + } + result = false; + } } - else if (wasCanceled()) + + if (result && !cancelled) { - VSIUnlink(doc_filepath_utf8); + result = commitStagedOutput(); } - setProgress(maximumProgress()); - return result; + if (!result || cancelled) + { + cleanupPartialOutput(tiles); + if (cancelled) + { + error_message = tr("Canceled"); + } + } + else + { + if (progress_observer) + progress_observer->setValue(100); + } + return result && !cancelled; #else Q_UNUSED(map_printer) Q_UNUSED(tile_width_px) @@ -255,10 +338,16 @@ bool KmzGroundOverlayExport::doExport(const MapPrinter& map_printer, int tile_wi #endif // QT_PRINTSUPPORT_LIB } -bool KmzGroundOverlayExport::doExport(const MapPrinter& map_printer, const Metrics& metrics, const std::vector& tiles) +bool KmzGroundOverlayExport::doExport(MapPrinter& map_printer, const Metrics& metrics, const std::vector& tiles) try { #ifdef QT_PRINTSUPPORT_LIB + if (wasCanceled()) + { + error_message = tr("Canceled"); + return false; + } + // Reusing the same QByteArray allocation for KML document and for tiles. QByteArray byte_array; byte_array.reserve(100000); @@ -272,20 +361,42 @@ try QImage image(metrics.tile_size_px, QImage::Format_ARGB32_Premultiplied); QImage buffer(metrics.tile_size_px, QImage::Format_RGB32); + if (image.isNull() || buffer.isNull()) + { + error_message = tr("Failed to allocate a raster export tile."); + return false; + } auto progress = 0; for (auto const& tile : tiles) { + if (wasCanceled()) + { + error_message = tr("Canceled"); + return false; + } image.fill(Qt::white); buffer.fill(Qt::white); QPainter painter(&image); const auto tile_transform = makeTileTransform(tile.rect_map, metrics, map.getGeoreferencing().getDeclination()); - map_printer.drawPage(&painter, tile.rect_map.adjusted(-5, -5, 5, 5), tile_transform, &buffer); + map_printer.drawPage( + &painter, renderClip(tile), tile_transform, &buffer); + if (!painter.isActive()) + { + error_message = map_printer.outputError().isEmpty() + ? tr("Failed to render exact template imagery.") + : map_printer.outputError(); + return false; + } + painter.end(); saveToBuffer(image, byte_array); writeToVSI(basepath_utf8 + '/' + tile.filepath, byte_array); setProgress(++progress); if (wasCanceled()) - break; + { + error_message = tr("Canceled"); + return false; + } } return true; #else @@ -352,6 +463,25 @@ std::vector KmzGroundOverlayExport::makeTiles(cons } +QRectF KmzGroundOverlayExport::renderClip(const Tile& tile) noexcept +{ + // Include nearby objects and template pixels which can contribute through + // antialiasing at the edge of the geographic tile. + return tile.rect_map.adjusted(-5, -5, 5, 5); +} + +QRectF KmzGroundOverlayExport::renderExtent( + const std::vector& tiles) noexcept +{ + if (tiles.empty()) + return {}; + auto extent = renderClip(tiles.front()); + for (auto tile = std::next(tiles.cbegin()); tile != tiles.cend(); ++tile) + extent = extent.united(renderClip(*tile)); + return extent; +} + + void KmzGroundOverlayExport::writeKml(QByteArray& buffer, const std::vector& tiles) const { buffer.append( @@ -400,9 +530,13 @@ QTransform KmzGroundOverlayExport::makeTileTransform(const QRectF& tile_map, con void KmzGroundOverlayExport::saveToBuffer(const QImage& image, QByteArray& data) { QBuffer buffer(&data); - buffer.open(QIODevice::WriteOnly | QIODevice::Truncate); - image.save(&buffer, format, quality); - buffer.close(); + if (!buffer.open(QIODevice::WriteOnly | QIODevice::Truncate) + || !image.save(&buffer, format, quality)) + { + throw FileFormatException( + KmzGroundOverlayExport::tr( + "Failed to encode a raster export tile.")); + } } // static @@ -411,8 +545,31 @@ void KmzGroundOverlayExport::writeToVSI(const QByteArray& filepath_utf8, const Q auto* file = VSIFOpenL(filepath_utf8, "wb"); if (!file) throw FileFormatException(QString::fromUtf8(VSIGetLastErrorMsg())); - VSIFWriteL(data, 1, data.size(), file); - VSIFCloseL(file); + qsizetype offset = 0; + bool write_failed = false; + while (offset < data.size()) + { + auto const written = VSIFWriteL( + data.constData() + offset, + 1, + std::size_t(data.size() - offset), + file); + if (written == 0) + { + write_failed = true; + break; + } + offset += qsizetype(written); + } + auto const close_failed = VSIFCloseL(file) != 0; + if (write_failed || close_failed) + { + auto error = QString::fromUtf8(VSIGetLastErrorMsg()); + if (error.isEmpty()) + error = KmzGroundOverlayExport::tr( + "Failed to write an export file."); + throw FileFormatException(error); + } } void KmzGroundOverlayExport::mkdir(const QByteArray& path_utf8) const @@ -426,31 +583,306 @@ void KmzGroundOverlayExport::mkdir(const QByteArray& path_utf8) const } -void KmzGroundOverlayExport::setMaximumProgress(int value) const +void KmzGroundOverlayExport::setMaximumProgress(int value) +{ + tile_progress_maximum = std::max(1, value); + if (progress_observer) + progress_observer->setRange(0, 100); +} + +void KmzGroundOverlayExport::setProgress(int value) const { if (progress_observer) { - progress_observer->setMaximum(value); + auto const percent = + 15 + + (84 * std::clamp( + value, 0, tile_progress_maximum)) + / tile_progress_maximum; + progress_observer->setValue(percent); + QApplication::processEvents( + QEventLoop::AllEvents, 100 /* ms */); } } -int KmzGroundOverlayExport::maximumProgress() const +bool KmzGroundOverlayExport::beginStagedOutput( + std::vector& tiles) { - return progress_observer ? progress_observer->maximum() : 100; + auto const output_path = + QString::fromUtf8(output_filepath_utf8); + auto const fileinfo = QFileInfo(output_path); + auto const staging_prefix = QString::fromLatin1( + stagingFilePrefix(output_filepath_utf8, is_kmz)); + QTemporaryFile staging( + fileinfo.absolutePath() + + QLatin1Char('/') + + staging_prefix + + QStringLiteral("XXXXXX") + // /vsizip/ treats the first .zip/.kmz component as the archive + // boundary, so only the final suffix may contain .kmz. + + (is_kmz + ? QStringLiteral(".kmz") + : QStringLiteral(".part"))); + staging.setAutoRemove(true); + if (!staging.open()) + { + error_message = tr("Could not create a temporary export file: %1") + .arg(staging.errorString()); + return false; + } + staging_filepath_utf8 = + staging.fileName().toUtf8(); + staging.close(); + + if (is_kmz) + { + if (!QFile::remove( + QString::fromUtf8(staging_filepath_utf8))) + { + error_message = tr( + "Could not initialize the temporary KMZ archive."); + return false; + } + basepath_utf8 = + "/vsizip/" + staging_filepath_utf8; + doc_filepath_utf8 = + basepath_utf8 + "/doc.kml"; + files_directory_existed = false; + } + else + { + basepath_utf8 = fileinfo.absolutePath().toUtf8(); + doc_filepath_utf8 = staging_filepath_utf8; + files_directory_existed = + GdalFile::isDir(basepath_utf8 + "/files"); + auto const files_path = + fileinfo.absolutePath() + + QStringLiteral("/files"); + if (!QDir().mkpath(files_path)) + { + error_message = tr( + "Could not create the KML image directory."); + staging_filepath_utf8.clear(); + return false; + } + + // Direct KML is a multi-file format. Keep every new sidecar in an + // exporter-owned transaction directory and publish the KML document + // last. On the next run, the currently published KML identifies the + // one live transaction directory; any other such directories are + // crash leftovers and can be removed safely. + pruneDirectAssetDirectories(); + auto const asset_prefix = + directAssetPrefix(output_filepath_utf8); + QTemporaryDir assets( + files_path + + QLatin1Char('/') + + QString::fromLatin1(asset_prefix) + + QStringLiteral("XXXXXX")); + assets.setAutoRemove(true); + if (!assets.isValid()) + { + error_message = tr( + "Could not create a temporary KML image directory."); + staging_filepath_utf8.clear(); + return false; + } + auto const assets_name = + QFileInfo(assets.path()).fileName(); + direct_assets_relative_utf8 = + QByteArray("files/") + + assets_name.toUtf8(); + for (auto& tile : tiles) + tile.filepath = + direct_assets_relative_utf8 + + '/' + tile.name; + assets.setAutoRemove(false); + staging.setAutoRemove(false); + } + return true; } -void KmzGroundOverlayExport::setProgress(int value) const +bool KmzGroundOverlayExport::commitStagedOutput() { - if (progress_observer) + QFile source(QString::fromUtf8(staging_filepath_utf8)); + if (!source.open(QIODevice::ReadOnly)) + { + error_message = tr("Could not read the completed temporary export: %1") + .arg(source.errorString()); + return false; + } + QSaveFile destination( + QString::fromUtf8(output_filepath_utf8)); + if (!destination.open(QIODevice::WriteOnly)) + { + error_message = tr("Could not open the export destination: %1") + .arg(destination.errorString()); + return false; + } + + QByteArray buffer(256 * 1024, Qt::Uninitialized); + while (true) + { + auto const count = + source.read(buffer.data(), buffer.size()); + if (count == 0) + break; + if (count < 0) + { + error_message = tr("Could not read the completed temporary export: %1") + .arg(source.errorString()); + destination.cancelWriting(); + return false; + } + if (destination.write(buffer.constData(), count) + != count) + { + error_message = tr("Could not write the export destination: %1") + .arg(destination.errorString()); + destination.cancelWriting(); + return false; + } + QApplication::processEvents( + QEventLoop::AllEvents, 25); + if (wasCanceled()) + { + cancelled = true; + error_message = tr("Canceled"); + destination.cancelWriting(); + return false; + } + } + if (!destination.commit()) + { + error_message = tr("Could not finish the export destination: %1") + .arg(destination.errorString()); + return false; + } + source.close(); + QFile::remove( + QString::fromUtf8(staging_filepath_utf8)); + staging_filepath_utf8.clear(); + if (!is_kmz) { - progress_observer->setValue(value); - QApplication::processEvents(QEventLoop::ExcludeUserInputEvents, 100 /* ms */); // Drawing and Cancel events + pruneDirectAssetDirectories( + direct_assets_relative_utf8); + direct_assets_relative_utf8.clear(); } + return true; } -bool KmzGroundOverlayExport::wasCanceled() const +void KmzGroundOverlayExport::cleanupPartialOutput( + const std::vector& tiles) { - return progress_observer && progress_observer->wasCanceled(); + if (!staging_filepath_utf8.isEmpty()) + { + QFile::remove( + QString::fromUtf8(staging_filepath_utf8)); + staging_filepath_utf8.clear(); + } + if (is_kmz) + return; + Q_UNUSED(tiles) + if (!direct_assets_relative_utf8.isEmpty()) + { + QDir( + QString::fromUtf8( + basepath_utf8 + '/' + + direct_assets_relative_utf8)) + .removeRecursively(); + direct_assets_relative_utf8.clear(); + } + if (!files_directory_existed) + { + auto const files_path = QByteArray(basepath_utf8 + "/files"); + VSIRmdir(files_path.constData()); + } +} + +void KmzGroundOverlayExport::pruneStagedOutputFiles() +{ + auto const output_path = + QString::fromUtf8(output_filepath_utf8); + auto const fileinfo = QFileInfo(output_path); + QDir directory(fileinfo.absolutePath()); + if (!directory.exists()) + return; + + auto const prefix = QString::fromLatin1( + stagingFilePrefix(output_filepath_utf8, is_kmz)); + auto const suffix = is_kmz + ? QStringLiteral(".kmz") + : QStringLiteral(".part"); + auto const entries = directory.entryInfoList( + { prefix + QLatin1Char('*') + suffix }, + QDir::Files | QDir::Hidden | QDir::NoSymLinks); + for (auto const& entry : entries) + { + // Never follow or unlink a caller-provided symlink or special file. + if (entry.isFile() && !entry.isSymLink()) + QFile::remove(entry.absoluteFilePath()); + } +} + +void KmzGroundOverlayExport::pruneDirectAssetDirectories( + const QByteArray& keep_relative_path) +{ + if (is_kmz) + return; + auto const files_path = + QString::fromUtf8(basepath_utf8 + "/files"); + QDir files(files_path); + if (!files.exists()) + return; + + QByteArray published_kml; + auto can_identify_published_assets = + !keep_relative_path.isEmpty(); + if (!can_identify_published_assets) + { + QFile current( + QString::fromUtf8(output_filepath_utf8)); + constexpr qint64 maximum_kml_recovery_bytes = + qint64(16) * 1024 * 1024; + if (!current.exists()) + { + can_identify_published_assets = true; + } + else if (current.size() + <= maximum_kml_recovery_bytes + && current.open(QIODevice::ReadOnly)) + { + published_kml = + current.read( + maximum_kml_recovery_bytes + 1); + can_identify_published_assets = + published_kml.size() + <= maximum_kml_recovery_bytes; + } + } + if (!can_identify_published_assets) + return; + + auto const entries = files.entryInfoList( + { QString::fromLatin1( + directAssetPrefix(output_filepath_utf8)) + + QLatin1Char('*') }, + QDir::Dirs | QDir::Hidden + | QDir::NoDotAndDotDot + | QDir::NoSymLinks); + for (auto const& entry : entries) + { + QByteArray const relative = + QByteArray("files/") + + entry.fileName().toUtf8(); + auto const keep = + !keep_relative_path.isEmpty() + ? relative == keep_relative_path + : published_kml.contains(relative); + if (!keep) + QDir(entry.absoluteFilePath()) + .removeRecursively(); + } } diff --git a/src/gdal/kmz_groundoverlay_export.h b/src/gdal/kmz_groundoverlay_export.h index e16b11014..7d200940d 100644 --- a/src/gdal/kmz_groundoverlay_export.h +++ b/src/gdal/kmz_groundoverlay_export.h @@ -81,14 +81,21 @@ class KmzGroundOverlayExport void setProgressObserver(QProgressDialog* observer) noexcept; QString errorString() const; + bool wasCanceled() const; - bool doExport(const MapPrinter& map_printer, int tile_width_px = 512); + bool doExport(MapPrinter& map_printer, int tile_width_px = 512); protected: - bool doExport(const MapPrinter& map_printer, const Metrics& metrics, const std::vector& tiles); + bool doExport(MapPrinter& map_printer, const Metrics& metrics, const std::vector& tiles); std::vector makeTiles(const QRectF& extent_map, const Metrics& metrics) const; + + /** Returns the actual overscanned map clip painted for a tile. */ + static QRectF renderClip(const Tile& tile) noexcept; + + /** Returns the union of all map clips painted by the export. */ + static QRectF renderExtent(const std::vector& tiles) noexcept; void writeKml(QByteArray& buffer, const std::vector& tiles) const; @@ -102,23 +109,32 @@ class KmzGroundOverlayExport void mkdir(const QByteArray& path) const; - void setMaximumProgress(int value) const; - - int maximumProgress() const; + void setMaximumProgress(int value); void setProgress(int value) const; - - bool wasCanceled() const; + + bool beginStagedOutput(std::vector& tiles); + bool commitStagedOutput(); + void cleanupPartialOutput(const std::vector& tiles); + void pruneStagedOutputFiles(); + void pruneDirectAssetDirectories( + const QByteArray& keep_relative_path = {}); private: const Map& map; QProgressDialog* progress_observer = nullptr; + QByteArray output_filepath_utf8; QByteArray basepath_utf8; QByteArray doc_filepath_utf8; + QByteArray staging_filepath_utf8; + QByteArray direct_assets_relative_utf8; QString error_message; qreal overlap = 0.000000000001; int precision = 13; + int tile_progress_maximum = 1; bool is_kmz = false; + bool cancelled = false; + bool files_directory_existed = false; }; diff --git a/src/gui/imagery/catalog_import_dialog.cpp b/src/gui/imagery/catalog_import_dialog.cpp new file mode 100644 index 000000000..501f4d387 --- /dev/null +++ b/src/gui/imagery/catalog_import_dialog.cpp @@ -0,0 +1,599 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "gui/imagery/catalog_import_dialog.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/manual_imagery_source.h" +#include "imagery/imagery_network_permissions.h" +#include "imagery/tile_network_manager.h" + +namespace OpenOrienteering { + +namespace { + +QString updateKindText( + imagery::ImageryCatalogAnalysis::UpdateKind kind) +{ + using Kind = imagery::ImageryCatalogAnalysis::UpdateKind; + switch (kind) + { + case Kind::NewCatalog: + return CatalogImportDialog::tr("New catalog"); + case Kind::ExactReimport: + return CatalogImportDialog::tr( + "Same catalog snapshot"); + case Kind::HigherRevision: + return CatalogImportDialog::tr( + "Newer revision"); + case Kind::LowerRevision: + return CatalogImportDialog::tr( + "Older revision"); + case Kind::SameRevisionConflict: + return CatalogImportDialog::tr( + "Republished revision"); + } + Q_UNREACHABLE(); +} + +QString displayUrl(const QString& value) +{ + auto url = QUrl(value); + if (!url.isValid()) + return value; + url.setPassword(QString {}); + url.setUserName(QString {}); + auto query = url.query(); + if (!query.isEmpty()) + url.setQuery(QStringLiteral("…")); + return url.toDisplayString( + QUrl::RemovePassword | QUrl::DecodeReserved); +} + +QStringList requestHosts( + const imagery::OicCatalogReadResult& catalog) +{ + QSet hosts; + for (auto const& source : catalog.catalog.sources) + { + for (auto const& tile : source.tile_urls) + { + auto text = tile.value; + text.replace(QStringLiteral("${z}"), QStringLiteral("0")); + text.replace(QStringLiteral("${x}"), QStringLiteral("0")); + text.replace(QStringLiteral("${y}"), QStringLiteral("0")); + text.replace(QStringLiteral("{z}"), QStringLiteral("0")); + text.replace(QStringLiteral("{x}"), QStringLiteral("0")); + text.replace(QStringLiteral("{y}"), QStringLiteral("0")); + auto const url = QUrl(text); + if (!url.host().isEmpty()) + hosts.insert(url.host().toLower()); + } + } + auto list = hosts.values(); + std::sort(list.begin(), list.end()); + return list; +} + +QVector requestUrls( + const imagery::OicCatalogReadResult& catalog) +{ + QVector urls; + for (auto const& source : catalog.catalog.sources) + { + for (auto const& tile : source.tile_urls) + { + auto text = tile.value; + text.replace(QStringLiteral("${z}"), QStringLiteral("0")); + text.replace(QStringLiteral("${x}"), QStringLiteral("0")); + text.replace(QStringLiteral("${y}"), QStringLiteral("0")); + text.replace(QStringLiteral("{z}"), QStringLiteral("0")); + text.replace(QStringLiteral("{x}"), QStringLiteral("0")); + text.replace(QStringLiteral("{y}"), QStringLiteral("0")); + urls.push_back(QUrl(text)); + } + } + return urls; +} + +} // namespace + + +CatalogImportDialog::CatalogImportDialog( + imagery::ImageryCatalogRepository& repository, + QWidget* parent, + imagery::ImageryNetworkPermissions* permissions) + : QDialog(parent) + , repository_(repository) +{ + setWindowTitle(tr("Review imagery catalog")); + resize(620, 560); + + status_icon_ = new QLabel(this); + status_icon_->setAccessibleName( + tr("Catalog review status")); + status_text_ = new QLabel(this); + status_text_->setTextFormat(Qt::PlainText); + status_text_->setWordWrap(true); + auto* status_layout = new QHBoxLayout(); + status_layout->addWidget(status_icon_, 0, Qt::AlignTop); + status_layout->addWidget(status_text_, 1); + + progress_ = new QProgressBar(this); + progress_->setRange(0, 0); + progress_->setTextVisible(false); + + auto* details = new QWidget(this); + auto* form = new QFormLayout(details); + form->setFieldGrowthPolicy( + QFormLayout::AllNonFixedFieldsGrow); + identity_ = new QLabel(details); + origin_ = new QLabel(details); + publisher_ = new QLabel(details); + hash_ = new QLabel(details); + changes_ = new QLabel(details); + sources_ = new QLabel(details); + hosts_ = new QLabel(details); + warnings_ = new QLabel(details); + for (auto* label : { + identity_, origin_, publisher_, hash_, changes_, + sources_, hosts_, warnings_ }) + { + label->setTextFormat(Qt::PlainText); + label->setWordWrap(true); + label->setTextInteractionFlags( + Qt::TextSelectableByMouse); + } + form->addRow(tr("Catalog:"), identity_); + form->addRow(tr("Origin:"), origin_); + form->addRow(tr("Publisher:"), publisher_); + form->addRow(tr("Document SHA-256:"), hash_); + form->addRow(tr("Update:"), changes_); + form->addRow(tr("Sources:"), sources_); + form->addRow(tr("Request hosts:"), hosts_); + form->addRow(tr("Review:"), warnings_); + + auto* scroll = new QScrollArea(this); + scroll->setWidgetResizable(true); + scroll->setFrameShape(QFrame::NoFrame); + scroll->setWidget(details); + + confirmation_ = new QCheckBox(this); + confirmation_->hide(); + + buttons_ = new QDialogButtonBox( + QDialogButtonBox::Cancel, + this); + approve_private_button_ = buttons_->addButton( + tr("Allow local network and retry"), + QDialogButtonBox::ActionRole); + approve_private_button_->hide(); + install_button_ = buttons_->addButton( + tr("Install"), + QDialogButtonBox::AcceptRole); + install_button_->setEnabled(false); + + auto* layout = new QVBoxLayout(this); + layout->addLayout(status_layout); + layout->addWidget(progress_); + layout->addWidget(scroll, 1); + layout->addWidget(confirmation_); + layout->addWidget(buttons_); + + connect( + buttons_, + &QDialogButtonBox::rejected, + this, + &QDialog::reject); + connect( + install_button_, + &QPushButton::clicked, + this, + &CatalogImportDialog::install); + connect( + approve_private_button_, + &QPushButton::clicked, + this, + &CatalogImportDialog::approvePrivateFetch); + connect( + confirmation_, + &QCheckBox::toggled, + this, + &CatalogImportDialog::updateInstallButton); + connect( + &repository_, + &imagery::ImageryCatalogRepository::operationFinished, + this, + &CatalogImportDialog::operationFinished); + permissions_ = permissions + ? permissions + : new imagery::ImageryNetworkPermissions( + repository_.networkManager(), + this); +} + + +CatalogImportDialog::~CatalogImportDialog() +{ + if (operation_id_) + repository_.cancel(operation_id_); +} + + +void CatalogImportDialog::startFile( + const QString& path) +{ + fetch_request_.reset(); + private_network_approval_url_.clear(); + startOperation( + repository_.readCatalogFile(path), + tr("Reading and validating the catalog…")); +} + + +void CatalogImportDialog::startFetch( + const imagery::ImageryCatalogFetchRequest& request) +{ + fetch_request_ = request; + private_network_approval_url_.clear(); + startOperation( + repository_.fetchCatalog(request), + request.installed_catalog_id.isEmpty() + ? tr("Downloading and validating the catalog…") + : tr("Checking for a catalog update…")); +} + + +void CatalogImportDialog::startOperation( + imagery::ImageryCatalogRepository::OperationId operation_id, + const QString& activity) +{ + if (operation_id_) + repository_.cancel(operation_id_); + operation_id_ = operation_id; + candidate_.clear(); + approve_private_button_->hide(); + progress_->show(); + showStatus(activity, QStyle::SP_BrowserReload); + updateInstallButton(); +} + + +void CatalogImportDialog::operationFinished( + imagery::ImageryCatalogRepository::OperationId operation_id, + const imagery::ImageryCatalogOperationResult& result) +{ + if (operation_id != operation_id_) + return; + operation_id_ = 0; + progress_->hide(); + switch (result.kind) + { + case imagery::ImageryCatalogOperationKind::CandidateReady: + showCandidate(result.candidate); + return; + + case imagery::ImageryCatalogOperationKind::NotModified: + showStatus( + tr("This catalog is already up to date."), + QStyle::SP_DialogApplyButton); + install_button_->setText(tr("Close")); + install_button_->setEnabled(true); + disconnect( + install_button_, + &QPushButton::clicked, + this, + &CatalogImportDialog::install); + connect( + install_button_, + &QPushButton::clicked, + this, + &QDialog::accept); + return; + + case imagery::ImageryCatalogOperationKind::Installed: + showStatus( + tr("The imagery catalog was installed."), + QStyle::SP_DialogApplyButton); + accept(); + return; + + case imagery::ImageryCatalogOperationKind::Cancelled: + showStatus( + tr("Catalog operation cancelled."), + QStyle::SP_MessageBoxInformation); + return; + + case imagery::ImageryCatalogOperationKind::Failed: + if (fetch_request_ + && !result.private_network_approval_url.isEmpty()) + { + private_network_approval_url_ = + result.private_network_approval_url; + showStatus( + tr("This catalog is on a local or private " + "network. Contacting it requires an explicit, " + "installation-local approval."), + QStyle::SP_MessageBoxWarning); + approve_private_button_->show(); + updateInstallButton(); + return; + } + showStatus( + result.error.isEmpty() + ? tr("The catalog operation failed.") + : result.error, + QStyle::SP_MessageBoxCritical); + updateInstallButton(); + return; + + case imagery::ImageryCatalogOperationKind::Removed: + break; + } +} + + +void CatalogImportDialog::approvePrivateFetch() +{ + if (!fetch_request_ + || private_network_approval_url_.isEmpty()) + return; + auto const origin = + imagery::TileNetworkManager::canonicalOrigin( + private_network_approval_url_); + auto const answer = QMessageBox::warning( + this, + tr("Allow local-network catalog"), + tr("Allow Mapper to contact %1 from this installation?\n\n" + "This permission is stored only on this device. Catalogs " + "and map files cannot grant it.") + .arg(origin), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes + || !permissions_->approve(private_network_approval_url_)) + return; + auto const request = *fetch_request_; + startFetch(request); +} + + +void CatalogImportDialog::showCandidate( + imagery::ImageryCatalogCandidatePtr candidate) +{ + candidate_ = std::move(candidate); + confirmation_required_ = false; + install_options_ = {}; + if (!candidate_) + { + showStatus( + tr("The catalog could not be read."), + QStyle::SP_MessageBoxCritical); + return; + } + + auto const& read = candidate_->read_result; + auto const& catalog = read.catalog; + auto const& analysis = candidate_->analysis; + identity_->setText( + tr("%1\nID: %2\nRevision: %3") + .arg( + catalog.name.isEmpty() ? catalog.id : catalog.name, + catalog.id) + .arg(catalog.revision)); + origin_->setText(displayUrl(candidate_->metadata.origin)); + publisher_->setText( + catalog.publisher + ? catalog.publisher->name + : tr("Not specified")); + hash_->setText(QString::fromLatin1(catalog.document_sha256)); + changes_->setText( + tr("%1\n%2 added, %3 removed, %4 operational, " + "%5 metadata-only") + .arg(updateKindText(analysis.update_kind)) + .arg(analysis.added) + .arg(analysis.removed) + .arg(analysis.operational_changed) + .arg(analysis.metadata_only_changed)); + sources_->setText( + tr("%1 total · %2 usable · %3 invalid · %4 unsupported") + .arg(catalog.sources.size()) + .arg(read.supportedSourceCount()) + .arg(analysis.invalid) + .arg(analysis.unsupported)); + auto const hosts = requestHosts(read); + hosts_->setText( + hosts.isEmpty() + ? tr("None") + : hosts.join(QLatin1Char('\n'))); + + QStringList warnings; + if (!read.accepted()) + { + for (auto const& diagnostic : read.diagnostics) + { + warnings.push_back(diagnostic.displayText()); + if (warnings.size() >= 8) + break; + } + } + if (analysis.update_kind + == imagery::ImageryCatalogAnalysis::UpdateKind::LowerRevision) + { + install_options_.allow_lower_revision = true; + confirmation_required_ = true; + warnings.push_back( + tr("This would replace a newer installed revision.")); + } + if (analysis.update_kind + == imagery::ImageryCatalogAnalysis::UpdateKind:: + SameRevisionConflict) + { + install_options_.allow_same_revision_conflict = true; + confirmation_required_ = true; + warnings.push_back( + tr("The publisher reused a revision number for different contents.")); + } + if (catalog.original_bytes.size() > 1024 * 1024 + || catalog.sources.size() > 100) + { + confirmation_required_ = true; + warnings.push_back( + tr("This is a large catalog; review its publisher and request hosts.")); + } + if (QUrl(candidate_->metadata.origin).scheme() + == QLatin1String("http")) + { + confirmation_required_ = true; + warnings.push_back( + tr("The catalog was downloaded over unencrypted HTTP.")); + } + if (analysis.exact_duplicates > 0 + || analysis.potential_duplicates > 0) + { + warnings.push_back( + tr("%1 exact and %2 operational duplicate source(s) " + "also exist in other catalogs.") + .arg(analysis.exact_duplicates) + .arg(analysis.potential_duplicates)); + } + bool insecure_service = false; + bool local_service = false; + QStringList secret_parameters; + for (auto const& url : requestUrls(read)) + { + insecure_service = insecure_service + || url.scheme().toLower() == QLatin1String("http"); + auto const host = url.host().toLower(); + QHostAddress address; + local_service = local_service + || (address.setAddress(host) + && !imagery::TileNetworkManager:: + isPublicDestinationAddress(address)) + || host == QLatin1String("localhost") + || host.endsWith(QLatin1String(".localhost")) + || host.endsWith(QLatin1String(".local")) + || host.endsWith(QLatin1String(".internal")) + || host.endsWith(QLatin1String(".home.arpa")); + secret_parameters.append( + imagery::ManualImagerySource:: + likelySecretQueryParameters(url)); + } + secret_parameters.removeDuplicates(); + if (insecure_service) + { + warnings.push_back( + tr("One or more imagery services use unencrypted HTTP.")); + } + if (local_service) + { + warnings.push_back( + tr("One or more services target a local or private-network " + "origin. Installing this catalog does not grant access.")); + } + if (!secret_parameters.isEmpty()) + { + warnings.push_back( + tr("Service endpoints contain credential-like query " + "parameter(s): %1.") + .arg(secret_parameters.join(QStringLiteral(", ")))); + } + for (auto const& source : catalog.sources) + { + if (source.registration.kind + != imagery::OicRegistrationKind::None) + { + warnings.push_back( + tr("One or more sources include surveyed registration corrections.")); + break; + } + } + warnings.removeDuplicates(); + warnings_->setText( + warnings.isEmpty() + ? tr("No additional warnings.") + : QStringLiteral("• ") + + warnings.join(QStringLiteral("\n• "))); + + if (read.accepted()) + { + showStatus( + tr("Review the catalog before installing it. " + "No imagery service has been contacted."), + warnings.isEmpty() + ? QStyle::SP_MessageBoxInformation + : QStyle::SP_MessageBoxWarning); + } + else + { + showStatus( + tr("This file is not an installable OIC catalog."), + QStyle::SP_MessageBoxCritical); + } + + confirmation_->setText( + tr("I reviewed the warnings and want to install this catalog.")); + confirmation_->setChecked(false); + confirmation_->setVisible( + confirmation_required_ && read.accepted()); + updateInstallButton(); +} + + +void CatalogImportDialog::showStatus( + const QString& text, + QStyle::StandardPixmap icon) +{ + status_icon_->setPixmap( + style()->standardIcon(icon).pixmap(24, 24)); + status_text_->setText(text); +} + + +void CatalogImportDialog::updateInstallButton() +{ + auto const accepted = + candidate_ + && candidate_->read_result.accepted() + && (!confirmation_required_ + || confirmation_->isChecked()); + install_button_->setEnabled( + accepted && operation_id_ == 0); +} + + +void CatalogImportDialog::install() +{ + if (!candidate_ || operation_id_) + return; + startOperation( + repository_.installCandidate( + candidate_, + install_options_), + tr("Installing the catalog snapshot…")); +} + +} // namespace OpenOrienteering diff --git a/src/gui/imagery/catalog_import_dialog.h b/src/gui/imagery/catalog_import_dialog.h new file mode 100644 index 000000000..c3d40bdc0 --- /dev/null +++ b/src/gui/imagery/catalog_import_dialog.h @@ -0,0 +1,93 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_GUI_CATALOG_IMPORT_DIALOG_H +#define OPENORIENTEERING_GUI_CATALOG_IMPORT_DIALOG_H + +#include + +#include +#include + +#include "imagery/imagery_catalog_repository.h" + +class QCheckBox; +class QDialogButtonBox; +class QLabel; +class QProgressBar; +class QPushButton; + +namespace OpenOrienteering { + +namespace imagery { +class ImageryNetworkPermissions; +} + +class CatalogImportDialog final : public QDialog +{ +Q_OBJECT + +public: + explicit CatalogImportDialog( + imagery::ImageryCatalogRepository& repository, + QWidget* parent = nullptr, + imagery::ImageryNetworkPermissions* permissions = nullptr); + ~CatalogImportDialog() override; + + void startFile(const QString& path); + void startFetch( + const imagery::ImageryCatalogFetchRequest& request); + +private: + void startOperation( + imagery::ImageryCatalogRepository::OperationId operation_id, + const QString& activity); + void operationFinished( + imagery::ImageryCatalogRepository::OperationId operation_id, + const imagery::ImageryCatalogOperationResult& result); + void showCandidate( + imagery::ImageryCatalogCandidatePtr candidate); + void showStatus( + const QString& text, + QStyle::StandardPixmap icon); + void approvePrivateFetch(); + void install(); + void updateInstallButton(); + + imagery::ImageryCatalogRepository& repository_; + imagery::ImageryNetworkPermissions* permissions_ = nullptr; + imagery::ImageryCatalogRepository::OperationId operation_id_ = 0; + std::optional fetch_request_; + QUrl private_network_approval_url_; + imagery::ImageryCatalogCandidatePtr candidate_; + imagery::ImageryCatalogInstallOptions install_options_; + bool confirmation_required_ = false; + + QLabel* status_icon_ = nullptr; + QLabel* status_text_ = nullptr; + QLabel* identity_ = nullptr; + QLabel* origin_ = nullptr; + QLabel* publisher_ = nullptr; + QLabel* hash_ = nullptr; + QLabel* changes_ = nullptr; + QLabel* sources_ = nullptr; + QLabel* hosts_ = nullptr; + QLabel* warnings_ = nullptr; + QProgressBar* progress_ = nullptr; + QCheckBox* confirmation_ = nullptr; + QDialogButtonBox* buttons_ = nullptr; + QPushButton* approve_private_button_ = nullptr; + QPushButton* install_button_ = nullptr; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/src/gui/imagery/catalog_manager_dialog.cpp b/src/gui/imagery/catalog_manager_dialog.cpp new file mode 100644 index 000000000..0912f9aa1 --- /dev/null +++ b/src/gui/imagery/catalog_manager_dialog.cpp @@ -0,0 +1,398 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "gui/imagery/catalog_manager_dialog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gui/file_dialog.h" +#include "gui/imagery/catalog_import_dialog.h" + +namespace OpenOrienteering { + +namespace { + +constexpr int catalog_id_role = Qt::UserRole; + +QString checkedText(const QDateTime& value) +{ + return value.isValid() + ? QLocale().toString( + value.toLocalTime(), + QLocale::ShortFormat) + : CatalogManagerDialog::tr("Never"); +} + +QString plainTextToolTip(const QString& value) +{ + auto escaped = value.toHtmlEscaped(); + escaped.replace( + QLatin1Char('\n'), + QStringLiteral("
")); + return QStringLiteral("%1").arg(escaped); +} + +} // namespace + + +CatalogManagerDialog::CatalogManagerDialog( + imagery::ImageryCatalogRepository& repository, + QWidget* parent, + imagery::ImageryNetworkPermissions* permissions) + : QDialog(parent) + , repository_(repository) + , permissions_(permissions) +{ + setWindowTitle(tr("Imagery catalogs")); + resize(780, 440); + + issues_icon_ = new QLabel(this); + issues_icon_->setPixmap( + style()->standardIcon( + QStyle::SP_MessageBoxWarning).pixmap(24, 24)); + issues_icon_->setAccessibleName( + tr("Catalog storage warning")); + issues_icon_->hide(); + issues_ = new QLabel(this); + issues_->setTextFormat(Qt::PlainText); + issues_->setWordWrap(true); + issues_->hide(); + auto* issues_layout = new QHBoxLayout(); + issues_layout->addWidget( + issues_icon_, 0, Qt::AlignTop); + issues_layout->addWidget(issues_, 1); + + catalogs_ = new QTreeWidget(this); + catalogs_->setRootIsDecorated(false); + catalogs_->setSelectionMode( + QAbstractItemView::SingleSelection); + catalogs_->setHeaderLabels({ + tr("Catalog"), + tr("Revision"), + tr("Usable sources"), + tr("Last checked"), + tr("Origin"), + }); + catalogs_->setAlternatingRowColors(true); + catalogs_->setSortingEnabled(true); + catalogs_->sortByColumn(0, Qt::AscendingOrder); + + auto* import_file = new QPushButton( + tr("Import file…"), + this); + auto* import_url = new QPushButton( + tr("Import URL…"), + this); + update_button_ = new QPushButton( + tr("Check for update"), + this); + remove_button_ = new QPushButton( + tr("Remove"), + this); + auto* action_layout = new QHBoxLayout(); + action_layout->addWidget(import_file); + action_layout->addWidget(import_url); + action_layout->addStretch(); + action_layout->addWidget(update_button_); + action_layout->addWidget(remove_button_); + + auto* buttons = new QDialogButtonBox( + QDialogButtonBox::Close, + this); + + auto* layout = new QVBoxLayout(this); + layout->addLayout(issues_layout); + layout->addWidget(catalogs_, 1); + layout->addLayout(action_layout); + layout->addWidget(buttons); + + connect( + import_file, + &QPushButton::clicked, + this, + &CatalogManagerDialog::importFile); + connect( + import_url, + &QPushButton::clicked, + this, + &CatalogManagerDialog::importUrl); + connect( + update_button_, + &QPushButton::clicked, + this, + &CatalogManagerDialog::checkForUpdate); + connect( + remove_button_, + &QPushButton::clicked, + this, + &CatalogManagerDialog::removeSelected); + connect( + buttons, + &QDialogButtonBox::rejected, + this, + &QDialog::reject); + connect( + catalogs_, + &QTreeWidget::itemSelectionChanged, + this, + &CatalogManagerDialog::updateButtons); + connect( + &repository_, + &imagery::ImageryCatalogRepository::snapshotChanged, + this, + &CatalogManagerDialog::rebuild); + connect( + &repository_, + &imagery::ImageryCatalogRepository::operationFinished, + this, + &CatalogManagerDialog::operationFinished); + + rebuild(); +} + + +void CatalogManagerDialog::rebuild() +{ + auto selected_id = QString {}; + if (auto* item = catalogs_->currentItem()) + selected_id = item->data(0, catalog_id_role).toString(); + + catalogs_->clear(); + auto const snapshot = repository_.snapshot(); + if (!snapshot) + return; + for (auto const& installed : snapshot->catalogs) + { + auto const& catalog = installed.read_result.catalog; + auto* item = new QTreeWidgetItem(catalogs_); + item->setText( + 0, + catalog.name.isEmpty() ? catalog.id : catalog.name); + item->setText(1, QString::number(catalog.revision)); + item->setText( + 2, + QString::number( + installed.read_result.supportedSourceCount())); + item->setText(3, checkedText(installed.state.checked_at)); + item->setText( + 4, + QUrl(installed.state.origin) + .toDisplayString( + QUrl::RemovePassword + | QUrl::RemoveQuery)); + item->setData(0, catalog_id_role, catalog.id); + item->setToolTip( + 0, + plainTextToolTip( + tr("ID: %1\nSHA-256: %2") + .arg( + catalog.id, + QString::fromLatin1( + installed.state.sha256)))); + if (catalog.id == selected_id) + catalogs_->setCurrentItem(item); + } + catalogs_->resizeColumnToContents(0); + catalogs_->resizeColumnToContents(1); + catalogs_->resizeColumnToContents(2); + catalogs_->resizeColumnToContents(3); + + if (snapshot->issues.isEmpty()) + { + issues_icon_->hide(); + issues_->hide(); + } + else + { + issues_->setText( + tr("%n catalog store entry could not be loaded. " + "Healthy catalogs remain available.", + nullptr, + snapshot->issues.size())); + issues_icon_->show(); + issues_->show(); + } + updateButtons(); +} + + +const imagery::InstalledImageryCatalog* +CatalogManagerDialog::selectedCatalog() const +{ + auto* item = catalogs_->currentItem(); + if (!item) + return nullptr; + auto const snapshot = repository_.snapshot(); + return snapshot + ? snapshot->catalog( + item->data(0, catalog_id_role).toString()) + : nullptr; +} + + +void CatalogManagerDialog::updateButtons() +{ + auto const* catalog = selectedCatalog(); + auto const scheme = catalog + ? QUrl( + catalog->state.final_url.isEmpty() + ? catalog->state.origin + : catalog->state.final_url) + .scheme() + .toLower() + : QString {}; + update_button_->setEnabled( + catalog + && (scheme == QLatin1String("https") + || scheme == QLatin1String("http")) + && !remove_operation_); + remove_button_->setEnabled(catalog && !remove_operation_); +} + + +void CatalogManagerDialog::importFile() +{ + auto const path = FileDialog::getOpenFileName( + this, + tr("Import imagery catalog"), + QString {}, + tr("Imagery catalogs (*.oic);;All files (*.*)")); + if (path.isEmpty()) + return; + CatalogImportDialog dialog(repository_, this, permissions_); + dialog.startFile(path); + dialog.exec(); +} + + +void CatalogManagerDialog::importUrl() +{ + bool accepted = false; + auto const value = QInputDialog::getText( + this, + tr("Import imagery catalog"), + tr("Catalog URL:"), + QLineEdit::Normal, + QStringLiteral("https://"), + &accepted); + if (!accepted || value.trimmed().isEmpty()) + return; + imagery::ImageryCatalogFetchRequest request; + request.url = QUrl::fromUserInput(value.trimmed()); + if (request.url.scheme().toLower() == QLatin1String("http")) + { + auto const answer = QMessageBox::warning( + this, + tr("Unencrypted catalog download"), + tr("This catalog URL uses plain HTTP. Its contents and " + "publisher identity cannot be trusted in transit. Continue?"), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes) + return; + request.allow_insecure_http = true; + } + CatalogImportDialog dialog(repository_, this, permissions_); + dialog.startFetch(request); + dialog.exec(); +} + + +void CatalogManagerDialog::checkForUpdate() +{ + auto const* installed = selectedCatalog(); + if (!installed) + return; + imagery::ImageryCatalogFetchRequest request; + request.url = QUrl( + installed->state.final_url.isEmpty() + ? installed->state.origin + : installed->state.final_url); + request.etag = installed->state.etag; + request.last_modified = installed->state.last_modified; + request.installed_catalog_id = + installed->read_result.catalog.id; + if (request.url.scheme().toLower() == QLatin1String("http")) + { + auto const answer = QMessageBox::warning( + this, + tr("Unencrypted catalog update"), + tr("Checking this catalog uses plain HTTP. Continue?"), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes) + return; + request.allow_insecure_http = true; + } + CatalogImportDialog dialog(repository_, this, permissions_); + dialog.startFetch(request); + dialog.exec(); +} + + +void CatalogManagerDialog::removeSelected() +{ + auto const* installed = selectedCatalog(); + if (!installed || remove_operation_) + return; + auto const& catalog = installed->read_result.catalog; + QMessageBox confirmation( + QMessageBox::Question, + tr("Remove imagery catalog"), + tr("Remove “%1” from this installation?\n\n" + "Existing maps keep their embedded source snapshots.") + .arg(catalog.name.isEmpty() ? catalog.id : catalog.name), + QMessageBox::NoButton, + this); + auto* remove_button = confirmation.addButton( + tr("Remove"), + QMessageBox::DestructiveRole); + confirmation.addButton(QMessageBox::Cancel); + confirmation.exec(); + if (confirmation.clickedButton() != remove_button) + return; + remove_operation_ = + repository_.removeCatalog(catalog.id); + updateButtons(); +} + + +void CatalogManagerDialog::operationFinished( + imagery::ImageryCatalogRepository::OperationId id, + const imagery::ImageryCatalogOperationResult& result) +{ + if (id != remove_operation_) + return; + remove_operation_ = 0; + if (result.kind == imagery::ImageryCatalogOperationKind::Failed) + { + QMessageBox::warning( + this, + tr("Could not remove catalog"), + result.error); + } + updateButtons(); +} + +} // namespace OpenOrienteering diff --git a/src/gui/imagery/catalog_manager_dialog.h b/src/gui/imagery/catalog_manager_dialog.h new file mode 100644 index 000000000..5e9425bae --- /dev/null +++ b/src/gui/imagery/catalog_manager_dialog.h @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_GUI_CATALOG_MANAGER_DIALOG_H +#define OPENORIENTEERING_GUI_CATALOG_MANAGER_DIALOG_H + +#include + +#include "imagery/imagery_catalog_repository.h" + +class QLabel; +class QPushButton; +class QTreeWidget; +class QTreeWidgetItem; + +namespace OpenOrienteering { + +namespace imagery { +class ImageryNetworkPermissions; +} + +class CatalogManagerDialog final : public QDialog +{ +Q_OBJECT + +public: + explicit CatalogManagerDialog( + imagery::ImageryCatalogRepository& repository, + QWidget* parent = nullptr, + imagery::ImageryNetworkPermissions* permissions = nullptr); + +private: + void rebuild(); + const imagery::InstalledImageryCatalog* selectedCatalog() const; + void updateButtons(); + void importFile(); + void importUrl(); + void checkForUpdate(); + void removeSelected(); + void operationFinished( + imagery::ImageryCatalogRepository::OperationId id, + const imagery::ImageryCatalogOperationResult& result); + + imagery::ImageryCatalogRepository& repository_; + imagery::ImageryNetworkPermissions* permissions_ = nullptr; + imagery::ImageryCatalogRepository::OperationId remove_operation_ = 0; + QTreeWidget* catalogs_ = nullptr; + QLabel* issues_icon_ = nullptr; + QLabel* issues_ = nullptr; + QPushButton* update_button_ = nullptr; + QPushButton* remove_button_ = nullptr; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/src/gui/imagery/imagery_network_permissions_dialog.cpp b/src/gui/imagery/imagery_network_permissions_dialog.cpp new file mode 100644 index 000000000..079582ef7 --- /dev/null +++ b/src/gui/imagery/imagery_network_permissions_dialog.cpp @@ -0,0 +1,254 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "gui/imagery/imagery_network_permissions_dialog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/imagery_network_permissions.h" + +namespace OpenOrienteering { + +namespace { + +constexpr int origin_role = Qt::UserRole; +constexpr int pending_role = Qt::UserRole + 1; + +} // namespace + + +ImageryNetworkPermissionsDialog::ImageryNetworkPermissionsDialog( + imagery::ImageryNetworkPermissions& permissions, + QWidget* parent) + : QDialog(parent) + , permissions_(permissions) +{ + setWindowTitle(tr("Imagery network permissions")); + resize(660, 390); + + auto* explanation = new QLabel( + tr("Mapper blocks imagery requests to private and non-global " + "destinations by default. Allow only origins you recognize. " + "These permissions are stored on this device; catalogs and map " + "files cannot grant them."), + this); + explanation->setWordWrap(true); + explanation->setTextFormat(Qt::PlainText); + + origins_ = new QTreeWidget(this); + origins_->setObjectName( + QStringLiteral("imagery_network_permission_origins")); + origins_->setRootIsDecorated(false); + origins_->setSelectionMode( + QAbstractItemView::SingleSelection); + origins_->setAlternatingRowColors(true); + origins_->setHeaderLabels({ tr("Origin"), tr("Access") }); + origins_->setSortingEnabled(true); + origins_->sortByColumn(0, Qt::AscendingOrder); + + empty_state_ = new QLabel(this); + empty_state_->setTextFormat(Qt::PlainText); + empty_state_->setWordWrap(true); + empty_state_->setAlignment(Qt::AlignCenter); + empty_state_->setText( + tr("No private-network imagery origins need review.")); + + approve_button_ = new QPushButton(tr("Allow"), this); + approve_button_->setObjectName( + QStringLiteral("approve_imagery_network_origin")); + revoke_button_ = new QPushButton(tr("Revoke"), this); + revoke_button_->setObjectName( + QStringLiteral("revoke_imagery_network_origin")); + dismiss_button_ = new QPushButton(tr("Dismiss"), this); + dismiss_button_->setObjectName( + QStringLiteral("dismiss_imagery_network_origin")); + auto* action_layout = new QHBoxLayout(); + action_layout->addWidget(approve_button_); + action_layout->addWidget(revoke_button_); + action_layout->addWidget(dismiss_button_); + action_layout->addStretch(); + + auto* buttons = new QDialogButtonBox( + QDialogButtonBox::Close, + this); + auto* layout = new QVBoxLayout(this); + layout->addWidget(explanation); + layout->addWidget(origins_, 1); + layout->addWidget(empty_state_); + layout->addLayout(action_layout); + layout->addWidget(buttons); + + connect( + buttons, + &QDialogButtonBox::rejected, + this, + &QDialog::reject); + connect( + origins_, + &QTreeWidget::itemSelectionChanged, + this, + &ImageryNetworkPermissionsDialog::updateButtons); + connect( + approve_button_, + &QPushButton::clicked, + this, + &ImageryNetworkPermissionsDialog::approveSelected); + connect( + revoke_button_, + &QPushButton::clicked, + this, + &ImageryNetworkPermissionsDialog::revokeSelected); + connect( + dismiss_button_, + &QPushButton::clicked, + this, + &ImageryNetworkPermissionsDialog::dismissSelected); + connect( + &permissions_, + &imagery::ImageryNetworkPermissions::approvalsChanged, + this, + &ImageryNetworkPermissionsDialog::rebuild); + connect( + &permissions_, + &imagery::ImageryNetworkPermissions::pendingOriginsChanged, + this, + &ImageryNetworkPermissionsDialog::rebuild); + + rebuild(); +} + + +void ImageryNetworkPermissionsDialog::rebuild() +{ + auto selected_origin = QString {}; + if (auto* item = selectedItem()) + selected_origin = item->data(0, origin_role).toString(); + + origins_->clear(); + for (auto const& origin : permissions_.pendingOrigins()) + { + auto* item = new QTreeWidgetItem(origins_); + item->setText(0, origin); + item->setText(1, tr("Approval needed")); + item->setData(0, origin_role, origin); + item->setData(0, pending_role, true); + item->setIcon( + 0, + style()->standardIcon(QStyle::SP_MessageBoxWarning)); + item->setToolTip( + 0, + tr("A recent imagery request resolved to a private or " + "non-global destination and was blocked.")); + if (origin == selected_origin) + origins_->setCurrentItem(item); + } + for (auto const& origin : permissions_.approvedOrigins()) + { + auto* item = new QTreeWidgetItem(origins_); + item->setText(0, origin); + item->setText(1, tr("Allowed on this device")); + item->setData(0, origin_role, origin); + item->setData(0, pending_role, false); + item->setIcon( + 0, + style()->standardIcon(QStyle::SP_DialogApplyButton)); + item->setToolTip( + 0, + tr("Mapper may contact this private-network origin for " + "imagery requests.")); + if (origin == selected_origin) + origins_->setCurrentItem(item); + } + origins_->resizeColumnToContents(0); + origins_->resizeColumnToContents(1); + auto const empty = origins_->topLevelItemCount() == 0; + origins_->setVisible(!empty); + empty_state_->setVisible(empty); + updateButtons(); +} + + +QTreeWidgetItem* +ImageryNetworkPermissionsDialog::selectedItem() const +{ + return origins_->currentItem(); +} + + +void ImageryNetworkPermissionsDialog::updateButtons() +{ + auto* item = selectedItem(); + auto const pending = item + && item->data(0, pending_role).toBool(); + approve_button_->setEnabled(item && pending); + dismiss_button_->setEnabled(item && pending); + revoke_button_->setEnabled(item && !pending); +} + + +void ImageryNetworkPermissionsDialog::approveSelected() +{ + auto* item = selectedItem(); + if (!item || !item->data(0, pending_role).toBool()) + return; + auto const origin = item->data(0, origin_role).toString(); + auto const answer = QMessageBox::warning( + this, + tr("Allow private-network imagery"), + tr("Allow Mapper to contact %1 for imagery?\n\n" + "Only continue if you recognize and trust this origin. " + "The permission applies to this installation until revoked.") + .arg(origin), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer == QMessageBox::Yes) + permissions_.approve(QUrl(origin)); +} + + +void ImageryNetworkPermissionsDialog::revokeSelected() +{ + auto* item = selectedItem(); + if (!item || item->data(0, pending_role).toBool()) + return; + auto const origin = item->data(0, origin_role).toString(); + auto const answer = QMessageBox::question( + this, + tr("Revoke imagery permission"), + tr("Stop allowing imagery requests to %1?\n\n" + "Active requests to this origin will be cancelled.") + .arg(origin), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Yes); + if (answer == QMessageBox::Yes) + permissions_.revoke(QUrl(origin)); +} + + +void ImageryNetworkPermissionsDialog::dismissSelected() +{ + auto* item = selectedItem(); + if (!item || !item->data(0, pending_role).toBool()) + return; + permissions_.dismissPending( + QUrl(item->data(0, origin_role).toString())); +} + +} // namespace OpenOrienteering diff --git a/src/gui/imagery/imagery_network_permissions_dialog.h b/src/gui/imagery/imagery_network_permissions_dialog.h new file mode 100644 index 000000000..211dde090 --- /dev/null +++ b/src/gui/imagery/imagery_network_permissions_dialog.h @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_GUI_IMAGERY_NETWORK_PERMISSIONS_DIALOG_H +#define OPENORIENTEERING_GUI_IMAGERY_NETWORK_PERMISSIONS_DIALOG_H + +#include + +class QLabel; +class QPushButton; +class QTreeWidget; +class QTreeWidgetItem; + +namespace OpenOrienteering { + +namespace imagery { +class ImageryNetworkPermissions; +} + +/** Reviews installation-local exceptions to the private-network imagery policy. */ +class ImageryNetworkPermissionsDialog final : public QDialog +{ +Q_OBJECT + +public: + explicit ImageryNetworkPermissionsDialog( + imagery::ImageryNetworkPermissions& permissions, + QWidget* parent = nullptr); + +private: + void rebuild(); + void updateButtons(); + void approveSelected(); + void revokeSelected(); + void dismissSelected(); + QTreeWidgetItem* selectedItem() const; + + imagery::ImageryNetworkPermissions& permissions_; + QTreeWidget* origins_ = nullptr; + QLabel* empty_state_ = nullptr; + QPushButton* approve_button_ = nullptr; + QPushButton* revoke_button_ = nullptr; + QPushButton* dismiss_button_ = nullptr; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/src/gui/imagery/imagery_source_model.cpp b/src/gui/imagery/imagery_source_model.cpp new file mode 100644 index 000000000..100859b86 --- /dev/null +++ b/src/gui/imagery/imagery_source_model.cpp @@ -0,0 +1,361 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "gui/imagery/imagery_source_model.h" + +#include + +#include +#include +#include +#include +#include + +#include "gui/action_icon.h" + +namespace OpenOrienteering { + +namespace { + +QString displayOrigin(QString value) +{ + auto url = QUrl(value); + if (!url.isValid() || url.scheme().isEmpty()) + return value; + url.setUserName(QString {}); + url.setPassword(QString {}); + url.setQuery(QString {}); + url.setFragment(QString {}); + return url.toDisplayString(QUrl::RemovePassword); +} + +QString plainTextToolTip(const QString& value) +{ + auto escaped = value.toHtmlEscaped(); + escaped.replace( + QLatin1Char('\n'), + QStringLiteral("
")); + return QStringLiteral("%1").arg(escaped); +} + +} // namespace + +struct ImagerySourceModel::Node +{ + NodeType type = NodeType::Catalog; + Node* parent = nullptr; + int row = 0; + QString name; + QString tooltip; + QString search_text; + QString status; + bool supported = true; + std::optional handle; + std::vector> children; +}; + + +ImagerySourceModel::ImagerySourceModel( + imagery::ImageryCatalogRepository& repository, + QObject* parent) + : QAbstractItemModel(parent) + , repository_(repository) +{ + connect( + &repository_, + &imagery::ImageryCatalogRepository::snapshotChanged, + this, + [this] { rebuild(); }); + rebuild(); +} + + +ImagerySourceModel::~ImagerySourceModel() = default; + + +void ImagerySourceModel::rebuild() +{ + beginResetModel(); + snapshot_ = repository_.snapshot(); + roots_.clear(); + if (snapshot_) + { + roots_.reserve(snapshot_->catalogs.size()); + for (auto const& installed : snapshot_->catalogs) + { + auto catalog = std::make_unique(); + catalog->type = NodeType::Catalog; + catalog->row = int(roots_.size()); + catalog->name = + installed.read_result.catalog.name.isEmpty() + ? installed.read_result.catalog.id + : installed.read_result.catalog.name; + catalog->status = tr("Revision %1 · %n source(s)", nullptr, + installed.read_result.catalog.sources.size()) + .arg(installed.read_result.catalog.revision); + catalog->tooltip = tr("%1\nInstalled from %2") + .arg( + installed.read_result.catalog.id, + displayOrigin(installed.state.origin)); + catalog->search_text = + catalog->name + QLatin1Char(' ') + + installed.read_result.catalog.id + QLatin1Char(' ') + + installed.read_result.catalog.description; + + for (qsizetype source_index = 0; + source_index + < installed.read_result.catalog.sources.size(); + ++source_index) + { + auto const& definition = + installed.read_result.catalog.sources.at( + source_index); + auto source = std::make_unique(); + source->type = NodeType::Source; + source->parent = catalog.get(); + source->row = int(catalog->children.size()); + source->name = definition.metadata.name.isEmpty() + ? definition.metadata.id + : definition.metadata.name; + if (source->name.isEmpty()) + { + source->name = tr("Invalid source %1") + .arg(source_index + 1); + } + source->supported = + definition.valid && definition.supported + && definition.resolved_source.has_value(); + auto const use_index_identity = + !definition.valid + || definition.metadata.id.isEmpty(); + source->handle = imagery::ImagerySourceHandle { + installed.read_result.catalog.id, + use_index_identity + ? QString {} + : definition.metadata.id, + installed.state.sha256, + use_index_identity + ? int(source_index) + : -1, + }; + source->search_text = + source->name + QLatin1Char(' ') + + definition.metadata.id + QLatin1Char(' ') + + definition.metadata.description + QLatin1Char(' ') + + imagery::categoryName( + definition.metadata.category); + if (source->supported) + { + auto const& resolved = + *definition.resolved_source; + source->status = tr("%1 · zoom %2–%3") + .arg( + resolved.tile_matrix_set.crs) + .arg(resolved.min_zoom) + .arg(resolved.max_zoom); + source->tooltip = + definition.metadata.description; + } + else + { + source->status = tr("Not supported"); + QStringList reasons = + definition.unsupported_capabilities; + for (auto const& diagnostic + : installed.read_result.diagnostics) + { + if (diagnostic.source_index + == source->row + && (diagnostic.kind + == imagery::OicDiagnosticKind:: + UnsupportedSource + || diagnostic.kind + == imagery::OicDiagnosticKind:: + SourceError)) + { + reasons.push_back( + diagnostic.message); + } + } + reasons.removeDuplicates(); + source->tooltip = reasons.isEmpty() + ? tr("This source cannot be used by this build.") + : reasons.join(QLatin1Char('\n')); + } + catalog->children.push_back(std::move(source)); + } + std::stable_sort( + catalog->children.begin(), + catalog->children.end(), + [](auto const& left, auto const& right) { + return QString::localeAwareCompare( + left->name, + right->name) < 0; + }); + for (int row = 0; + row < int(catalog->children.size()); + ++row) + catalog->children[row]->row = row; + roots_.push_back(std::move(catalog)); + } + } + endResetModel(); +} + + +ImagerySourceModel::Node* ImagerySourceModel::node( + const QModelIndex& index) const +{ + return index.isValid() + ? static_cast(index.internalPointer()) + : nullptr; +} + + +QModelIndex ImagerySourceModel::index( + int row, + int column, + const QModelIndex& parent) const +{ + if (column != 0 || row < 0) + return {}; + auto* parent_node = node(parent); + if (!parent_node) + { + if (row >= int(roots_.size())) + return {}; + return createIndex(row, column, roots_[row].get()); + } + if (parent_node->type != NodeType::Catalog + || row >= int(parent_node->children.size())) + return {}; + return createIndex( + row, + column, + parent_node->children[row].get()); +} + + +QModelIndex ImagerySourceModel::parent( + const QModelIndex& child) const +{ + auto* child_node = node(child); + if (!child_node || !child_node->parent) + return {}; + auto* parent_node = child_node->parent; + return createIndex( + parent_node->row, + 0, + parent_node); +} + + +int ImagerySourceModel::rowCount( + const QModelIndex& parent) const +{ + auto* parent_node = node(parent); + if (!parent_node) + return int(roots_.size()); + return parent_node->type == NodeType::Catalog + ? int(parent_node->children.size()) + : 0; +} + + +int ImagerySourceModel::columnCount( + const QModelIndex&) const +{ + return 1; +} + + +QVariant ImagerySourceModel::data( + const QModelIndex& index, + int role) const +{ + auto* item = node(index); + if (!item) + return {}; + switch (role) + { + case Qt::DisplayRole: + return item->name; + case Qt::ToolTipRole: + return plainTextToolTip(item->tooltip); + case Qt::DecorationRole: + if (item->type == NodeType::Catalog) + return ActionIcon::fromName(u"folder"); + return item->supported + ? ActionIcon::fromName(u"image") + : ActionIcon::fromName(u"warning"); + case NodeTypeRole: + return int(item->type); + case SourceHandleRole: + return item->handle + ? QVariant::fromValue(*item->handle) + : QVariant {}; + case SupportedRole: + return item->supported; + case SearchTextRole: + return item->search_text; + case StatusTextRole: + return item->status; + case Qt::ForegroundRole: + if (!item->supported + && item->type == NodeType::Source) + return QPalette().brush( + QPalette::Disabled, + QPalette::Text); + break; + default: + break; + } + return {}; +} + + +Qt::ItemFlags ImagerySourceModel::flags( + const QModelIndex& index) const +{ + auto* item = node(index); + if (!item) + return Qt::NoItemFlags; + return Qt::ItemIsEnabled | Qt::ItemIsSelectable; +} + + +std::optional +ImagerySourceModel::sourceHandle( + const QModelIndex& index) const +{ + auto* item = node(index); + return item ? item->handle : std::nullopt; +} + + +QModelIndex ImagerySourceModel::indexForHandle( + const imagery::ImagerySourceHandle& handle) const +{ + for (auto const& catalog : roots_) + { + for (auto const& source : catalog->children) + { + if (source->handle && *source->handle == handle) + return createIndex( + source->row, + 0, + source.get()); + } + } + return {}; +} + +} // namespace OpenOrienteering diff --git a/src/gui/imagery/imagery_source_model.h b/src/gui/imagery/imagery_source_model.h new file mode 100644 index 000000000..7d2f02f04 --- /dev/null +++ b/src/gui/imagery/imagery_source_model.h @@ -0,0 +1,83 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_GUI_IMAGERY_SOURCE_MODEL_H +#define OPENORIENTEERING_GUI_IMAGERY_SOURCE_MODEL_H + +#include +#include +#include + +#include + +#include "imagery/imagery_catalog_repository.h" + +namespace OpenOrienteering { + +class ImagerySourceModel final : public QAbstractItemModel +{ +Q_OBJECT + +public: + enum Role + { + NodeTypeRole = Qt::UserRole + 1, + SourceHandleRole, + SupportedRole, + SearchTextRole, + StatusTextRole, + }; + + enum class NodeType + { + Catalog, + Source, + }; + + explicit ImagerySourceModel( + imagery::ImageryCatalogRepository& repository, + QObject* parent = nullptr); + ~ImagerySourceModel() override; + + QModelIndex index( + int row, + int column, + const QModelIndex& parent = {}) const override; + QModelIndex parent( + const QModelIndex& child) const override; + int rowCount( + const QModelIndex& parent = {}) const override; + int columnCount( + const QModelIndex& parent = {}) const override; + QVariant data( + const QModelIndex& index, + int role = Qt::DisplayRole) const override; + Qt::ItemFlags flags(const QModelIndex& index) const override; + + std::optional sourceHandle( + const QModelIndex& index) const; + QModelIndex indexForHandle( + const imagery::ImagerySourceHandle& handle) const; + +private: + struct Node; + + void rebuild(); + Node* node(const QModelIndex& index) const; + + imagery::ImageryCatalogRepository& repository_; + imagery::ImageryCatalogRepositorySnapshotPtr snapshot_; + std::vector> roots_; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/src/gui/imagery/online_imagery_dialog.cpp b/src/gui/imagery/online_imagery_dialog.cpp new file mode 100644 index 000000000..61b26b509 --- /dev/null +++ b/src/gui/imagery/online_imagery_dialog.cpp @@ -0,0 +1,1158 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "gui/imagery/online_imagery_dialog.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gui/file_dialog.h" +#include "gui/imagery/catalog_import_dialog.h" +#include "gui/imagery/catalog_manager_dialog.h" +#include "gui/imagery/imagery_network_permissions_dialog.h" +#include "gui/imagery/imagery_source_model.h" +#include "imagery/imagery_network_permissions.h" +#include "imagery/tile_network_manager.h" + +namespace OpenOrienteering { + +namespace { + +QString sourceUrlProbe(QString value) +{ + value.replace(QStringLiteral("${z}"), QStringLiteral("0")); + value.replace(QStringLiteral("${x}"), QStringLiteral("0")); + value.replace(QStringLiteral("${y}"), QStringLiteral("0")); + value.replace(QStringLiteral("{z}"), QStringLiteral("0")); + value.replace(QStringLiteral("{x}"), QStringLiteral("0")); + value.replace(QStringLiteral("{y}"), QStringLiteral("0")); + return value; +} + +QStringList sourceHosts( + const imagery::ResolvedImagerySource& source) +{ + QSet hosts; + for (auto const& tile : source.tile_urls) + { + auto const host = + QUrl(sourceUrlProbe(tile.value)).host().toLower(); + if (!host.isEmpty()) + hosts.insert(host); + } + auto result = hosts.values(); + std::sort(result.begin(), result.end()); + return result; +} + +bool potentiallyPrivate(const QUrl& url) +{ + auto host = url.host().toLower(); + QHostAddress address; + if (address.setAddress(host)) + return !imagery::TileNetworkManager:: + isPublicDestinationAddress(address); + return host == QLatin1String("localhost") + || host.endsWith(QLatin1String(".localhost")) + || host.endsWith(QLatin1String(".local")) + || host.endsWith(QLatin1String(".internal")) + || host.endsWith(QLatin1String(".home.arpa")); +} + +QString dateRange(const imagery::ImageryMetadata& metadata) +{ + if (metadata.start_date.isValid() + && metadata.end_date.isValid()) + { + return OnlineImageryDialog::tr("%1 to %2") + .arg( + QLocale().toString( + metadata.start_date, + QLocale::ShortFormat), + QLocale().toString( + metadata.end_date, + QLocale::ShortFormat)); + } + if (metadata.start_date.isValid()) + { + return OnlineImageryDialog::tr("From %1") + .arg(QLocale().toString( + metadata.start_date, + QLocale::ShortFormat)); + } + if (metadata.end_date.isValid()) + { + return OnlineImageryDialog::tr("Through %1") + .arg(QLocale().toString( + metadata.end_date, + QLocale::ShortFormat)); + } + return OnlineImageryDialog::tr("Not specified"); +} + +} // namespace + + +OnlineImageryDialog::OnlineImageryDialog( + imagery::ImageryCatalogRepository& repository, + imagery::TileNetworkManager& network, + QWidget* parent, + imagery::ImageryNetworkPermissions* permissions) + : QDialog(parent) + , repository_(repository) + , network_(network) + , network_client_id_( + imagery::TileNetworkManager::nextClientId()) +{ + setWindowTitle(tr("Add online imagery")); + resize(940, 640); + permissions_ = permissions + ? permissions + : new imagery::ImageryNetworkPermissions(network_, this); + + source_model_ = new ImagerySourceModel(repository_, this); + source_filter_ = new QSortFilterProxyModel(this); + source_filter_->setSourceModel(source_model_); + source_filter_->setFilterRole( + ImagerySourceModel::SearchTextRole); + source_filter_->setFilterCaseSensitivity( + Qt::CaseInsensitive); + source_filter_->setRecursiveFilteringEnabled(true); + source_filter_->setAutoAcceptChildRows(true); + + search_ = new QLineEdit(this); + search_->setObjectName(QStringLiteral("imagery_search")); + search_->setPlaceholderText(tr("Search imagery sources")); + search_->setClearButtonEnabled(true); + source_tree_ = new QTreeView(this); + source_tree_->setObjectName(QStringLiteral("imagery_source_tree")); + source_tree_->setModel(source_filter_); + source_tree_->setHeaderHidden(true); + source_tree_->setUniformRowHeights(true); + source_tree_->setSelectionMode( + QAbstractItemView::SingleSelection); + source_tree_->expandAll(); + + auto* manual_button = new QPushButton( + tr("Enter a tile or service URL…"), + this); + manual_button->setObjectName(QStringLiteral("manual_source_button")); + auto* import_button = new QPushButton( + tr("Import catalog…"), + this); + auto* manage_button = new QPushButton( + tr("Manage catalogs…"), + this); + auto* permissions_button = new QPushButton( + tr("Network access…"), + this); + auto* left_actions = new QHBoxLayout(); + left_actions->addWidget(import_button); + left_actions->addWidget(manage_button); + left_actions->addWidget(permissions_button); + + auto* left = new QWidget(this); + auto* left_layout = new QVBoxLayout(left); + left_layout->setContentsMargins(0, 0, 0, 0); + left_layout->addWidget(search_); + left_layout->addWidget(source_tree_, 1); + left_layout->addWidget(manual_button); + left_layout->addLayout(left_actions); + + pages_ = new QStackedWidget(this); + auto* catalog_page = new QWidget(this); + auto* catalog_layout = new QVBoxLayout(catalog_page); + detail_title_ = new QLabel(catalog_page); + detail_title_->setObjectName( + QStringLiteral("imagery_detail_title")); + detail_title_->setTextFormat(Qt::PlainText); + auto title_font = detail_title_->font(); + title_font.setPointSizeF(title_font.pointSizeF() * 1.25); + title_font.setBold(true); + detail_title_->setFont(title_font); + detail_description_ = new QLabel(catalog_page); + detail_description_->setObjectName( + QStringLiteral("imagery_detail_description")); + detail_description_->setTextFormat(Qt::PlainText); + detail_description_->setWordWrap(true); + auto* detail_form = new QFormLayout(); + detail_status_ = new QLabel(catalog_page); + detail_catalog_ = new QLabel(catalog_page); + detail_dates_ = new QLabel(catalog_page); + detail_crs_ = new QLabel(catalog_page); + detail_hosts_ = new QLabel(catalog_page); + detail_attribution_ = new QLabel(catalog_page); + detail_terms_ = new QLabel(catalog_page); + for (auto* label : { + detail_status_, detail_catalog_, detail_dates_, + detail_crs_, detail_hosts_, detail_attribution_, + detail_terms_ }) + { + label->setTextFormat(Qt::PlainText); + label->setWordWrap(true); + label->setTextInteractionFlags( + Qt::TextSelectableByMouse); + } + detail_form->addRow(tr("Status:"), detail_status_); + detail_form->addRow(tr("Catalog:"), detail_catalog_); + detail_form->addRow(tr("Imagery dates:"), detail_dates_); + detail_form->addRow(tr("Grid:"), detail_crs_); + detail_form->addRow(tr("Request hosts:"), detail_hosts_); + detail_form->addRow(tr("Attribution:"), detail_attribution_); + detail_form->addRow(tr("Terms:"), detail_terms_); + catalog_layout->addWidget(detail_title_); + catalog_layout->addWidget(detail_description_); + catalog_layout->addLayout(detail_form); + catalog_layout->addStretch(); + pages_->addWidget(catalog_page); + + auto* manual_page = new QWidget(this); + auto* manual_layout = new QVBoxLayout(manual_page); + auto* manual_intro = new QLabel( + tr("Enter an XYZ or TMS tile template. ArcGIS REST services " + "are resolved from their published metadata before use."), + manual_page); + manual_intro->setWordWrap(true); + manual_url_ = new QLineEdit(manual_page); + manual_url_->setObjectName(QStringLiteral("manual_imagery_url")); + manual_url_->setPlaceholderText( + QStringLiteral("https://tiles.example.org/{z}/{x}/{y}.png")); + auto* manual_form = new QFormLayout(); + manual_form->addRow(tr("URL:"), manual_url_); + + auto* advanced = new QGroupBox( + tr("Advanced tile settings"), + manual_page); + auto* advanced_form = new QFormLayout(advanced); + manual_scheme_ = new QComboBox(advanced); + manual_scheme_->setObjectName(QStringLiteral("manual_row_scheme")); + manual_scheme_->addItem( + tr("XYZ (top-origin rows)"), + int(imagery::TileRowScheme::Xyz)); + manual_scheme_->addItem( + tr("TMS (bottom-origin rows)"), + int(imagery::TileRowScheme::Tms)); + manual_min_zoom_ = new QSpinBox(advanced); + manual_min_zoom_->setObjectName(QStringLiteral("manual_min_zoom")); + manual_min_zoom_->setRange( + 0, + imagery::ManualImagerySource::maximum_zoom); + manual_max_zoom_ = new QSpinBox(advanced); + manual_max_zoom_->setObjectName(QStringLiteral("manual_max_zoom")); + manual_max_zoom_->setRange( + 0, + imagery::ManualImagerySource::maximum_zoom); + manual_max_zoom_->setValue(19); + auto* zooms = new QWidget(advanced); + auto* zoom_layout = new QHBoxLayout(zooms); + zoom_layout->setContentsMargins(0, 0, 0, 0); + zoom_layout->addWidget(manual_min_zoom_); + zoom_layout->addWidget(new QLabel(tr("to"), zooms)); + zoom_layout->addWidget(manual_max_zoom_); + manual_tile_size_ = new QComboBox(advanced); + manual_tile_size_->setObjectName(QStringLiteral("manual_tile_size")); + manual_tile_size_->addItem(tr("256 × 256 pixels"), 256); + manual_tile_size_->addItem(tr("512 × 512 pixels"), 512); + manual_referer_ = new QLineEdit(advanced); + manual_referer_->setObjectName(QStringLiteral("manual_referer")); + manual_referer_->setPlaceholderText(tr("None")); + manual_empty_statuses_ = new QLineEdit(advanced); + manual_empty_statuses_->setObjectName( + QStringLiteral("manual_empty_statuses")); + manual_empty_statuses_->setText(QStringLiteral("204, 404")); + manual_attribution_ = new QLineEdit(advanced); + manual_attribution_url_ = new QLineEdit(advanced); + advanced_form->addRow(tr("Row scheme:"), manual_scheme_); + advanced_form->addRow(tr("Zoom range:"), zooms); + advanced_form->addRow(tr("Tile size:"), manual_tile_size_); + advanced_form->addRow(tr("HTTP Referer:"), manual_referer_); + advanced_form->addRow( + tr("Empty HTTP statuses:"), + manual_empty_statuses_); + advanced_form->addRow( + tr("Attribution text:"), + manual_attribution_); + advanced_form->addRow( + tr("Attribution URL:"), + manual_attribution_url_); + + discover_button_ = new QPushButton( + tr("Read service metadata"), + manual_page); + discover_button_->setObjectName( + QStringLiteral("discover_imagery_service")); + discover_button_->hide(); + approve_private_button_ = new QPushButton( + tr("Allow this local-network origin and retry…"), + manual_page); + approve_private_button_->hide(); + auto* discovery_actions = new QHBoxLayout(); + discovery_actions->addWidget(discover_button_); + discovery_actions->addWidget(approve_private_button_); + discovery_actions->addStretch(); + manual_layout->addWidget(manual_intro); + manual_layout->addLayout(manual_form); + manual_layout->addWidget(advanced); + manual_layout->addLayout(discovery_actions); + manual_layout->addStretch(); + pages_->addWidget(manual_page); + + auto* splitter = new QSplitter(this); + splitter->addWidget(left); + splitter->addWidget(pages_); + splitter->setStretchFactor(0, 2); + splitter->setStretchFactor(1, 3); + + display_name_ = new QLineEdit(this); + display_name_->setObjectName(QStringLiteral("imagery_display_name")); + auto* name_layout = new QFormLayout(); + name_layout->addRow(tr("Template name:"), display_name_); + + status_icon_ = new QLabel(this); + status_icon_->setAccessibleName( + tr("Imagery source status")); + status_text_ = new QLabel(this); + status_text_->setObjectName( + QStringLiteral("imagery_status_text")); + status_text_->setTextFormat(Qt::PlainText); + status_text_->setWordWrap(true); + auto* status_layout = new QHBoxLayout(); + status_layout->addWidget(status_icon_, 0, Qt::AlignTop); + status_layout->addWidget(status_text_, 1); + + buttons_ = new QDialogButtonBox( + QDialogButtonBox::Cancel, + this); + add_button_ = buttons_->addButton( + tr("Add"), + QDialogButtonBox::AcceptRole); + add_button_->setObjectName(QStringLiteral("add_online_imagery")); + add_button_->setEnabled(false); + + auto* layout = new QVBoxLayout(this); + layout->addWidget(splitter, 1); + layout->addLayout(name_layout); + layout->addLayout(status_layout); + layout->addWidget(buttons_); + + classify_timer_ = new QTimer(this); + classify_timer_->setSingleShot(true); + classify_timer_->setInterval(300); + + connect( + search_, + &QLineEdit::textChanged, + source_filter_, + &QSortFilterProxyModel::setFilterFixedString); + connect( + source_tree_->selectionModel(), + &QItemSelectionModel::currentChanged, + this, + [this](const QModelIndex& current) { + selectCatalogIndex(current); + }); + connect( + source_tree_, + &QTreeView::doubleClicked, + this, + [this](const QModelIndex& index) { + selectCatalogIndex(index); + if (selected_source_) + acceptSelection(); + }); + connect( + manual_button, + &QPushButton::clicked, + this, + &OnlineImageryDialog::showManualPage); + connect( + import_button, + &QPushButton::clicked, + this, + &OnlineImageryDialog::importCatalog); + connect( + manage_button, + &QPushButton::clicked, + this, + &OnlineImageryDialog::manageCatalogs); + connect( + permissions_button, + &QPushButton::clicked, + this, + &OnlineImageryDialog::manageNetworkPermissions); + connect( + classify_timer_, + &QTimer::timeout, + this, + &OnlineImageryDialog::classifyManual); + connect( + discover_button_, + &QPushButton::clicked, + this, + &OnlineImageryDialog::discoverArcGis); + connect( + approve_private_button_, + &QPushButton::clicked, + this, + &OnlineImageryDialog::approvePrivateDiscoveryOrigin); + connect( + &network_, + &imagery::TileNetworkManager::finished, + this, + &OnlineImageryDialog::onNetworkFinished); + connect( + &repository_, + &imagery::ImageryCatalogRepository::snapshotChanged, + this, + &OnlineImageryDialog::updateCatalogSelectionAfterReload); + connect( + buttons_, + &QDialogButtonBox::rejected, + this, + &QDialog::reject); + connect( + add_button_, + &QPushButton::clicked, + this, + &OnlineImageryDialog::acceptSelection); + + connect( + manual_url_, + &QLineEdit::textEdited, + this, + &OnlineImageryDialog::scheduleManualClassification); + for (auto* edit : { + manual_referer_, manual_empty_statuses_, + manual_attribution_, manual_attribution_url_ }) + { + connect( + edit, + &QLineEdit::textEdited, + this, + &OnlineImageryDialog::scheduleManualClassification); + } + for (auto* combo : { manual_scheme_, manual_tile_size_ }) + { + connect( + combo, + qOverload(&QComboBox::currentIndexChanged), + this, + &OnlineImageryDialog::scheduleManualClassification); + } + for (auto* spin : { manual_min_zoom_, manual_max_zoom_ }) + { + connect( + spin, + qOverload(&QSpinBox::valueChanged), + this, + &OnlineImageryDialog::scheduleManualClassification); + } + + if (source_model_->rowCount() > 0) + { + source_tree_->setCurrentIndex( + source_filter_->index(0, 0)); + } + else + { + showManualPage(); + } +} + + +OnlineImageryDialog::~OnlineImageryDialog() +{ + if (discovery_token_) + network_.cancel(discovery_token_); + network_.cancelClient(network_client_id_); +} + + +const imagery::ResolvedImagerySource& +OnlineImageryDialog::selectedSource() const +{ + Q_ASSERT(selected_source_); + return *selected_source_; +} + + +QString OnlineImageryDialog::displayName() const +{ + return display_name_->text().trimmed(); +} + + +void OnlineImageryDialog::selectCatalogIndex( + const QModelIndex& proxy_index) +{ + auto const source_index = + source_filter_->mapToSource(proxy_index); + auto const handle = + source_model_->sourceHandle(source_index); + if (!handle) + { + cancelManualDiscovery(); + pages_->setCurrentIndex(0); + clearSelectedSource(); + return; + } + showCatalogSource(*handle); +} + + +void OnlineImageryDialog::cancelManualDiscovery() +{ + classify_timer_->stop(); + if (discovery_token_) + { + network_.cancel(discovery_token_); + discovery_token_ = 0; + } + ++discovery_generation_; + insecure_discovery_consent_.clear(); + private_discovery_approval_url_.clear(); + discover_button_->setEnabled(true); + discover_button_->hide(); + approve_private_button_->hide(); +} + + +void OnlineImageryDialog::showCatalogSource( + const imagery::ImagerySourceHandle& handle) +{ + cancelManualDiscovery(); + pages_->setCurrentIndex(0); + auto const snapshot = repository_.snapshot(); + const imagery::InstalledImageryCatalog* installed = nullptr; + auto const* definition = snapshot + ? snapshot->source(handle, &installed) + : nullptr; + if (!definition || !installed) + { + clearSelectedSource(); + setStatus( + tr("This source is no longer available."), + QStyle::SP_MessageBoxWarning); + return; + } + selected_handle_ = handle; + detail_title_->setText( + definition->metadata.name.isEmpty() + && definition->metadata.id.isEmpty() + ? tr("Invalid source %1").arg(handle.source_index + 1) + : definition->metadata.name.isEmpty() + ? definition->metadata.id + : definition->metadata.name); + detail_description_->setText( + definition->metadata.description.isEmpty() + ? tr("No description was provided.") + : definition->metadata.description); + detail_catalog_->setText( + tr("%1 · revision %2") + .arg( + installed->read_result.catalog.name.isEmpty() + ? installed->read_result.catalog.id + : installed->read_result.catalog.name, + QString::number( + installed->read_result.catalog.revision))); + detail_dates_->setText(dateRange(definition->metadata)); + detail_attribution_->setText( + definition->notices.attribution_text.isEmpty() + ? tr("Not specified") + : definition->notices.attribution_text); + detail_terms_->setText( + definition->notices.terms_url.isEmpty() + ? tr("Not specified") + : definition->notices.terms_url.toDisplayString()); + + if (definition->resolved_source) + { + auto const& source = *definition->resolved_source; + detail_status_->setText(tr("Ready")); + detail_crs_->setText( + tr("%1 · zoom %2–%3") + .arg(source.tile_matrix_set.crs) + .arg(source.min_zoom) + .arg(source.max_zoom)); + detail_hosts_->setText( + sourceHosts(source).join(QLatin1Char('\n'))); + setSelectedSource( + source, + definition->metadata.name); + setStatus( + tr("Review the source details, then add an immutable " + "snapshot to this map."), + QStyle::SP_MessageBoxInformation); + } + else + { + detail_status_->setText(tr("Not supported")); + detail_crs_->setText(tr("Unavailable")); + detail_hosts_->clear(); + clearSelectedSource(); + QStringList reasons = + definition->unsupported_capabilities; + auto const source_index = int( + definition + - installed->read_result.catalog.sources.data()); + for (auto const& diagnostic + : installed->read_result.diagnostics) + { + if (diagnostic.source_index == source_index + && (diagnostic.kind + == imagery::OicDiagnosticKind::UnsupportedSource + || diagnostic.kind + == imagery::OicDiagnosticKind::SourceError)) + reasons.push_back(diagnostic.message); + } + reasons.removeDuplicates(); + setStatus( + reasons.isEmpty() + ? tr("This catalog source is not supported by this build.") + : reasons.join(QLatin1Char('\n')), + QStyle::SP_MessageBoxWarning); + } +} + + +void OnlineImageryDialog::showManualPage() +{ + pages_->setCurrentIndex(1); + source_tree_->clearSelection(); + selected_handle_.reset(); + clearSelectedSource(); + manual_url_->setFocus(); + classifyManual(); +} + + +imagery::ManualTiledSourceSettings +OnlineImageryDialog::manualSettings( + QString* error) const +{ + imagery::ManualTiledSourceSettings settings; + settings.scheme = imagery::TileRowScheme( + manual_scheme_->currentData().toInt()); + settings.min_zoom = manual_min_zoom_->value(); + settings.max_zoom = manual_max_zoom_->value(); + settings.tile_size = + manual_tile_size_->currentData().toInt(); + settings.referer = + QUrl(manual_referer_->text().trimmed()); + settings.attribution_text = + manual_attribution_->text().trimmed(); + settings.attribution_url = + QUrl(manual_attribution_url_->text().trimmed()); + settings.empty_http_status_codes.clear(); + QSet seen; + for (auto const& item : + manual_empty_statuses_->text().split( + QRegularExpression(QStringLiteral("[,\\s]+")), + Qt::SkipEmptyParts)) + { + bool ok = false; + auto const status = item.toInt(&ok); + if (!ok || status < 100 || status > 599 + || seen.contains(status)) + { + if (error) + { + *error = tr( + "Empty HTTP statuses must be unique numbers " + "between 100 and 599."); + } + return {}; + } + seen.insert(status); + settings.empty_http_status_codes.push_back(status); + } + return settings; +} + + +void OnlineImageryDialog::scheduleManualClassification() +{ + if (pages_->currentIndex() != 1) + pages_->setCurrentIndex(1); + if (discovery_token_) + { + network_.cancel(discovery_token_); + discovery_token_ = 0; + } + ++discovery_generation_; + insecure_discovery_consent_.clear(); + private_discovery_approval_url_.clear(); + classify_timer_->start(); + clearSelectedSource(); + discover_button_->hide(); + approve_private_button_->hide(); +} + + +void OnlineImageryDialog::classifyManual() +{ + QString settings_error; + auto const settings = manualSettings(&settings_error); + if (!settings_error.isEmpty()) + { + manual_result_ = {}; + clearSelectedSource(); + setStatus( + settings_error, + QStyle::SP_MessageBoxWarning); + return; + } + manual_result_ = imagery::ManualImagerySource::classify( + manual_url_->text(), + settings); + discover_button_->setVisible( + manual_result_.outcome + == imagery::ManualImageryOutcome::NeedsDiscovery); + approve_private_button_->hide(); + switch (manual_result_.outcome) + { + case imagery::ManualImageryOutcome::Direct: + setSelectedSource( + *manual_result_.source, + manual_result_.suggested_name); + setStatus( + manual_result_.warnings.isEmpty() + ? tr("Recognized a direct tiled source.") + : tr("Recognized a tiled source. Its URL contains " + "credential-like query parameters that will be " + "embedded in the map."), + manual_result_.warnings.isEmpty() + ? QStyle::SP_DialogApplyButton + : QStyle::SP_MessageBoxWarning); + break; + case imagery::ManualImageryOutcome::NeedsDiscovery: + clearSelectedSource(); + if (display_name_->text().trimmed().isEmpty()) + display_name_->setText( + manual_result_.suggested_name); + setStatus( + tr("This service must publish a cached, dyadic tile " + "scheme. Read its metadata to verify it."), + QStyle::SP_MessageBoxInformation); + break; + case imagery::ManualImageryOutcome::Unsupported: + clearSelectedSource(); + setStatus( + manual_result_.detail, + QStyle::SP_MessageBoxWarning); + break; + case imagery::ManualImageryOutcome::Invalid: + clearSelectedSource(); + setStatus( + manual_result_.detail, + manual_url_->text().isEmpty() + ? QStyle::SP_MessageBoxInformation + : QStyle::SP_MessageBoxWarning); + break; + } +} + + +imagery::TileNetworkRequest +OnlineImageryDialog::arcGisDiscoveryRequest( + quint64 generation) const +{ + imagery::TileNetworkRequest request; + request.url = manual_result_.discovery_url; + request.client_id = network_client_id_; + request.generation = generation; + request.priority = imagery::TileRequestPriority::Visible; + request.payload_kind = imagery::NetworkPayloadKind::JsonDocument; + request.referer = QUrl( + manual_referer_->text().trimmed()) + .toString(QUrl::FullyEncoded); + request.empty_http_status_codes.clear(); + request.max_response_bytes = + imagery::ArcGisTileService::maximum_metadata_size; + return request; +} + + +void OnlineImageryDialog::discoverArcGis() +{ + if (manual_result_.outcome + != imagery::ManualImageryOutcome::NeedsDiscovery) + return; + if (manual_result_.discovery_url.scheme().toLower() + == QLatin1String("http") + && insecure_discovery_consent_ + != manual_result_.discovery_url) + { + auto const answer = QMessageBox::warning( + this, + tr("Unencrypted service metadata"), + tr("This service publishes metadata over plain HTTP. " + "Continue for this request?"), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes) + return; + insecure_discovery_consent_ = + manual_result_.discovery_url; + } + if (potentiallyPrivate(manual_result_.discovery_url) + && !permissions_->isApproved( + manual_result_.discovery_url)) + { + approve_private_button_->show(); + setStatus( + tr("This service is on a local or private network. " + "It requires an installation-local approval."), + QStyle::SP_MessageBoxWarning); + return; + } + if (discovery_token_) + network_.cancel(discovery_token_); + ++discovery_generation_; + discovery_token_ = network_.submit( + arcGisDiscoveryRequest(discovery_generation_)); + discover_button_->setEnabled(false); + approve_private_button_->hide(); + setStatus( + tr("Reading ArcGIS service metadata…"), + QStyle::SP_BrowserReload); +} + + +void OnlineImageryDialog::onNetworkFinished( + imagery::TileNetworkManager::Token token, + const imagery::TileNetworkResult& result) +{ + if (!discovery_token_ || token != discovery_token_ + || result.client_id != network_client_id_ + || result.generation != discovery_generation_ + || pages_->currentIndex() != 1 + || manual_result_.outcome + != imagery::ManualImageryOutcome::NeedsDiscovery) + return; + discovery_token_ = 0; + discover_button_->setEnabled(true); + if (result.outcome + != imagery::TileNetworkResult::Outcome::Success) + { + clearSelectedSource(); + if (result.private_network_rejected + && !result.private_network_rejected_url.isEmpty() + && !permissions_->isApproved( + result.private_network_rejected_url)) + { + private_discovery_approval_url_ = + result.private_network_rejected_url; + approve_private_button_->show(); + } + setStatus( + result.error_string.isEmpty() + ? tr("Could not read the service metadata.") + : result.error_string, + QStyle::SP_MessageBoxWarning); + return; + } + private_discovery_approval_url_.clear(); + + QString settings_error; + auto const manual_settings = manualSettings(&settings_error); + if (!settings_error.isEmpty()) + { + setStatus( + settings_error, + QStyle::SP_MessageBoxWarning); + return; + } + imagery::ArcGisTileServiceSettings settings; + settings.name = display_name_->text().trimmed(); + settings.referer = manual_settings.referer; + settings.empty_http_status_codes = + manual_settings.empty_http_status_codes; + settings.attribution_text = + manual_settings.attribution_text; + settings.attribution_url = + manual_settings.attribution_url; + auto const service_url = + result.final_url.isValid() && !result.final_url.isEmpty() + ? result.final_url + : manual_result_.service_url; + auto const parsed = imagery::ArcGisTileService::parse( + result.body, + service_url, + settings); + if (!parsed.resolved()) + { + clearSelectedSource(); + setStatus( + parsed.detail, + parsed.outcome + == imagery::ArcGisTileServiceOutcome::Unsupported + ? QStyle::SP_MessageBoxWarning + : QStyle::SP_MessageBoxCritical); + return; + } + setSelectedSource( + *parsed.source, + parsed.service_title); + setStatus( + parsed.likely_secret_parameters.isEmpty() + ? tr("Resolved a cached ArcGIS tile service.") + : tr("Resolved the service. Its endpoint contains " + "credential-like query parameters that will be " + "embedded in the map."), + parsed.likely_secret_parameters.isEmpty() + ? QStyle::SP_DialogApplyButton + : QStyle::SP_MessageBoxWarning); +} + + +void OnlineImageryDialog::approvePrivateDiscoveryOrigin() +{ + auto const approval_url = + private_discovery_approval_url_.isEmpty() + ? manual_result_.discovery_url + : private_discovery_approval_url_; + auto const origin = imagery::TileNetworkManager::canonicalOrigin( + approval_url); + auto const answer = QMessageBox::warning( + this, + tr("Allow local-network imagery"), + tr("Allow Mapper to contact %1 from this installation?\n\n" + "This permission is stored only on this device. Catalogs " + "and map files cannot grant it.") + .arg(origin), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes) + return; + if (permissions_->approve(approval_url)) + { + private_discovery_approval_url_.clear(); + discoverArcGis(); + } +} + + +void OnlineImageryDialog::setSelectedSource( + imagery::ResolvedImagerySource source, + const QString& suggested_name) +{ + selected_source_ = std::move(source); + if (!suggested_name.trimmed().isEmpty()) + display_name_->setText(suggested_name.trimmed()); + add_button_->setEnabled(true); +} + + +void OnlineImageryDialog::clearSelectedSource() +{ + selected_source_.reset(); + add_button_->setEnabled(false); +} + + +void OnlineImageryDialog::updateCatalogSelectionAfterReload() +{ + source_tree_->expandAll(); + if (!selected_handle_) + return; + auto const snapshot = repository_.snapshot(); + if (!snapshot) + { + clearSelectedSource(); + return; + } + if (selected_handle_->source_id.isEmpty()) + { + auto const source_index = + source_model_->indexForHandle(*selected_handle_); + auto const proxy_index = + source_filter_->mapFromSource(source_index); + if (proxy_index.isValid() + && snapshot->source(*selected_handle_)) + { + source_tree_->setCurrentIndex(proxy_index); + showCatalogSource(*selected_handle_); + return; + } + clearSelectedSource(); + setStatus( + tr("The selected invalid source is no longer available."), + QStyle::SP_MessageBoxWarning); + return; + } + auto const latest = snapshot->latestHandle( + selected_handle_->catalog_id, + selected_handle_->source_id); + if (!latest) + { + clearSelectedSource(); + setStatus( + tr("The selected source was removed by a catalog update."), + QStyle::SP_MessageBoxWarning); + return; + } + auto const changed = *latest != *selected_handle_; + selected_handle_ = *latest; + auto const source_index = + source_model_->indexForHandle(*latest); + auto const proxy_index = + source_filter_->mapFromSource(source_index); + if (proxy_index.isValid()) + source_tree_->setCurrentIndex(proxy_index); + showCatalogSource(*latest); + if (changed) + { + setStatus( + tr("The catalog changed while this dialog was open. " + "Review the updated source before adding it."), + QStyle::SP_MessageBoxWarning); + } +} + + +void OnlineImageryDialog::importCatalog() +{ + auto const path = FileDialog::getOpenFileName( + this, + tr("Import imagery catalog"), + QString {}, + tr("Imagery catalogs (*.oic);;All files (*.*)")); + if (path.isEmpty()) + return; + CatalogImportDialog dialog(repository_, this, permissions_); + dialog.startFile(path); + dialog.exec(); +} + + +void OnlineImageryDialog::manageCatalogs() +{ + CatalogManagerDialog dialog(repository_, this, permissions_); + dialog.exec(); +} + + +void OnlineImageryDialog::manageNetworkPermissions() +{ + ImageryNetworkPermissionsDialog dialog( + *permissions_, + this); + dialog.exec(); +} + + +QVector OnlineImageryDialog::selectedRequestUrls() const +{ + QVector urls; + if (!selected_source_) + return urls; + for (auto const& tile : selected_source_->tile_urls) + urls.push_back(QUrl(sourceUrlProbe(tile.value))); + return urls; +} + + +void OnlineImageryDialog::acceptSelection() +{ + if (!selected_source_) + return; + auto const urls = selectedRequestUrls(); + bool insecure = false; + QStringList secret_parameters; + for (auto const& url : urls) + { + insecure = insecure + || url.scheme().toLower() == QLatin1String("http"); + secret_parameters.append( + imagery::ManualImagerySource:: + likelySecretQueryParameters(url)); + if (potentiallyPrivate(url) + && !permissions_->isApproved(url)) + { + auto const answer = QMessageBox::warning( + this, + tr("Allow local-network imagery"), + tr("This source contacts %1 on a local or private " + "network. Allow that origin on this device?") + .arg( + imagery::TileNetworkManager:: + canonicalOrigin(url)), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes + || !permissions_->approve(url)) + return; + } + } + secret_parameters.removeDuplicates(); + QStringList warnings; + if (insecure) + warnings.push_back( + tr("Imagery tiles will be downloaded over unencrypted HTTP.")); + if (!secret_parameters.isEmpty()) + { + warnings.push_back( + tr("The complete endpoint, including credential-like " + "query parameter(s) %1, will be embedded in this map.") + .arg(secret_parameters.join( + QStringLiteral(", ")))); + } + if (!warnings.isEmpty()) + { + auto const answer = QMessageBox::warning( + this, + tr("Review imagery endpoint"), + warnings.join(QStringLiteral("\n\n")), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Cancel); + if (answer != QMessageBox::Yes) + return; + } + if (display_name_->text().trimmed().isEmpty()) + display_name_->setText(selected_source_->metadata.name); + accept(); +} + + +void OnlineImageryDialog::setStatus( + const QString& text, + QStyle::StandardPixmap icon) +{ + status_icon_->setPixmap( + style()->standardIcon(icon).pixmap(24, 24)); + status_text_->setText(text); +} + +} // namespace OpenOrienteering diff --git a/src/gui/imagery/online_imagery_dialog.h b/src/gui/imagery/online_imagery_dialog.h new file mode 100644 index 000000000..f925f10c8 --- /dev/null +++ b/src/gui/imagery/online_imagery_dialog.h @@ -0,0 +1,144 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_GUI_ONLINE_IMAGERY_DIALOG_H +#define OPENORIENTEERING_GUI_ONLINE_IMAGERY_DIALOG_H + +#include + +#include +#include +#include + +#include "imagery/arcgis_tile_service.h" +#include "imagery/imagery_catalog_repository.h" +#include "imagery/manual_imagery_source.h" + +class QComboBox; +class QDialogButtonBox; +class QGroupBox; +class QLabel; +class QLineEdit; +class QPushButton; +class QSortFilterProxyModel; +class QSpinBox; +class QStackedWidget; +class QTimer; +class QTreeView; +class OnlineImageryDialogTest; + +namespace OpenOrienteering { + +namespace imagery { +class ImageryNetworkPermissions; +} + +class ImagerySourceModel; + +class OnlineImageryDialog final : public QDialog +{ +Q_OBJECT + +public: + explicit OnlineImageryDialog( + imagery::ImageryCatalogRepository& repository, + imagery::TileNetworkManager& network, + QWidget* parent = nullptr, + imagery::ImageryNetworkPermissions* permissions = nullptr); + ~OnlineImageryDialog() override; + + const imagery::ResolvedImagerySource& selectedSource() const; + QString displayName() const; + +private: + friend class ::OnlineImageryDialogTest; + + void selectCatalogIndex(const QModelIndex& proxy_index); + void cancelManualDiscovery(); + void showCatalogSource( + const imagery::ImagerySourceHandle& handle); + void showManualPage(); + void classifyManual(); + void scheduleManualClassification(); + imagery::TileNetworkRequest arcGisDiscoveryRequest( + quint64 generation) const; + void discoverArcGis(); + void onNetworkFinished( + imagery::TileNetworkManager::Token token, + const imagery::TileNetworkResult& result); + void approvePrivateDiscoveryOrigin(); + void updateCatalogSelectionAfterReload(); + void importCatalog(); + void manageCatalogs(); + void manageNetworkPermissions(); + void acceptSelection(); + void setStatus( + const QString& text, + QStyle::StandardPixmap icon); + void setSelectedSource( + imagery::ResolvedImagerySource source, + const QString& suggested_name); + void clearSelectedSource(); + imagery::ManualTiledSourceSettings manualSettings( + QString* error) const; + QVector selectedRequestUrls() const; + + imagery::ImageryCatalogRepository& repository_; + imagery::TileNetworkManager& network_; + imagery::ImageryNetworkPermissions* permissions_ = nullptr; + quint64 network_client_id_ = 0; + imagery::TileNetworkManager::Token discovery_token_ = 0; + quint64 discovery_generation_ = 0; + QUrl insecure_discovery_consent_; + QUrl private_discovery_approval_url_; + imagery::ManualImageryDiscoveryResult manual_result_; + std::optional selected_handle_; + std::optional selected_source_; + + ImagerySourceModel* source_model_ = nullptr; + QSortFilterProxyModel* source_filter_ = nullptr; + QLineEdit* search_ = nullptr; + QTreeView* source_tree_ = nullptr; + QStackedWidget* pages_ = nullptr; + + QLabel* detail_title_ = nullptr; + QLabel* detail_description_ = nullptr; + QLabel* detail_status_ = nullptr; + QLabel* detail_catalog_ = nullptr; + QLabel* detail_dates_ = nullptr; + QLabel* detail_crs_ = nullptr; + QLabel* detail_hosts_ = nullptr; + QLabel* detail_attribution_ = nullptr; + QLabel* detail_terms_ = nullptr; + + QLineEdit* manual_url_ = nullptr; + QComboBox* manual_scheme_ = nullptr; + QSpinBox* manual_min_zoom_ = nullptr; + QSpinBox* manual_max_zoom_ = nullptr; + QComboBox* manual_tile_size_ = nullptr; + QLineEdit* manual_referer_ = nullptr; + QLineEdit* manual_empty_statuses_ = nullptr; + QLineEdit* manual_attribution_ = nullptr; + QLineEdit* manual_attribution_url_ = nullptr; + QPushButton* discover_button_ = nullptr; + QPushButton* approve_private_button_ = nullptr; + QTimer* classify_timer_ = nullptr; + + QLineEdit* display_name_ = nullptr; + QLabel* status_icon_ = nullptr; + QLabel* status_text_ = nullptr; + QDialogButtonBox* buttons_ = nullptr; + QPushButton* add_button_ = nullptr; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/src/gui/map/map_editor.cpp b/src/gui/map/map_editor.cpp index 790b7f67a..7bf846978 100644 --- a/src/gui/map/map_editor.cpp +++ b/src/gui/map/map_editor.cpp @@ -114,6 +114,9 @@ #include "gui/configure_grid_dialog.h" #include "gui/file_dialog.h" #include "gui/georeferencing_dialog.h" +#include "gui/imagery/catalog_manager_dialog.h" +#include "gui/imagery/imagery_network_permissions_dialog.h" +#include "gui/imagery/online_imagery_dialog.h" #include "gui/main_window.h" #include "gui/print_widget.h" #include "gui/simple_course_dialog.h" @@ -140,6 +143,11 @@ #include "sensors/gps_display.h" #include "sensors/gps_temporary_markers.h" #include "sensors/gps_track_recorder.h" +#include "imagery/imagery_catalog_repository.h" +#include "imagery/imagery_network_permissions.h" +#include "imagery/imagery_source_snapshot.h" +#include "imagery/tile_network_manager.h" +#include "templates/online_raster_template.h" #include "templates/paint_on_template_feature.h" #include "templates/template.h" #include "templates/template_dialog_reopen.h" @@ -1040,6 +1048,47 @@ void MapEditorController::createActions() //QAction* template_config_window_act = newCheckAction("templateconfigwindow", tr("Template configurations window"), this, SLOT(showTemplateConfigurationsWindow(bool)), "window-new", tr("Show/Hide the template configurations window")); //QAction* template_visibilities_window_act = newCheckAction("templatevisibilitieswindow", tr("Template visibilities window"), this, SLOT(showTemplateVisbilitiesWindow(bool)), "window-new", tr("Show/Hide the template visibilities window")); open_template_act = newAction("opentemplate", tr("Open template..."), this, SLOT(openTemplateClicked()), nullptr, QString{}, "templates_menu.html"); + online_imagery_act = newAction( + "openonlineimagery", + tr("Add online imagery…"), + this, + SLOT(addOnlineImageryClicked()), + "image", + tr("Browse installed OIC catalogs or enter a tiled imagery URL"), + "templates_menu.html"); + manage_imagery_catalogs_act = newAction( + "manageimagerycatalogs", + tr("Manage imagery catalogs…"), + this, + SLOT(manageImageryCatalogsClicked()), + nullptr, + QString{}, + "templates_menu.html"); + manage_imagery_network_permissions_act = newAction( + "manageimagerynetworkpermissions", + tr("Imagery network permissions…"), + this, + SLOT(manageImageryNetworkPermissionsClicked()), + nullptr, + tr("Review or revoke private-network imagery access on this device"), + "templates_menu.html"); + // Observe blocked origins before map templates can make network requests. + (void)imagery::ImageryNetworkPermissions::instance(); + offline_imagery_act = newCheckAction( + "offlineimagery", + tr("Work offline for imagery"), + this, + SLOT(setOfflineImagery(bool)), + nullptr, + tr("Use only imagery already present in the local HTTP cache"), + "templates_menu.html"); + offline_imagery_act->setChecked( + imagery::TileNetworkManager::instance().offlineMode()); + connect( + &imagery::TileNetworkManager::instance(), + &imagery::TileNetworkManager::offlineModeChanged, + offline_imagery_act, + &QAction::setChecked); reopen_template_act = newAction("reopentemplate", tr("Reopen template..."), this, SLOT(reopenTemplateClicked()), nullptr, QString{}, "templates_menu.html"); tags_window_act = newCheckAction("tagswindow", tr("Tag editor"), this, SLOT(showTagsWindow(bool)), "tag-editor", tr("Show/Hide the tag editor window"), "tag_editor.html"); @@ -1294,6 +1343,11 @@ void MapEditorController::createMenuAndToolbars() /*template_menu->addAction(template_config_window_act); template_menu->addAction(template_visibilities_window_act);*/ template_menu->addSeparator(); + template_menu->addAction(online_imagery_act); + template_menu->addAction(manage_imagery_catalogs_act); + template_menu->addAction(manage_imagery_network_permissions_act); + template_menu->addAction(offline_imagery_act); + template_menu->addSeparator(); template_menu->addAction(open_template_act); template_menu->addAction(reopen_template_act); @@ -2343,6 +2397,97 @@ void MapEditorController::openTemplateClicked() } } +void MapEditorController::addOnlineImageryClicked() +{ + if (map->getGeoreferencing().getState() + != Georeferencing::Geospatial) + { + auto const answer = QMessageBox::question( + window, + tr("Georeferencing required"), + tr("Online imagery needs a georeferenced map. " + "Open the georeferencing settings now?"), + QMessageBox::Yes | QMessageBox::Cancel, + QMessageBox::Yes); + if (answer == QMessageBox::Yes) + georeferencing_act->trigger(); + return; + } + + auto& repository = + imagery::ImageryCatalogRepository::instance(); + auto& network = imagery::TileNetworkManager::instance(); + auto& permissions = + imagery::ImageryNetworkPermissions::instance(); + OnlineImageryDialog dialog( + repository, + network, + window, + &permissions); + if (dialog.exec() != QDialog::Accepted) + return; + + QString error; + auto snapshot = imagery::ImagerySourceSnapshotCodec::encode( + dialog.selectedSource(), + &error); + if (!snapshot) + { + QMessageBox::warning( + window, + tr("Could not add online imagery"), + error); + return; + } + auto online = std::make_unique( + std::move(*snapshot), + map); + if (!dialog.displayName().isEmpty()) + online->setDisplayName(dialog.displayName()); + if (!online->setupAndLoad(window, map_widget->getMapView())) + { + QMessageBox::warning( + window, + tr("Could not add online imagery"), + online->errorString()); + return; + } + map->addTemplate(-1, std::move(online)); + hideAllTemplates(false); + showTemplateWindow(true); +} + + +void MapEditorController::manageImageryCatalogsClicked() +{ + CatalogManagerDialog dialog( + imagery::ImageryCatalogRepository::instance(), + window, + &imagery::ImageryNetworkPermissions::instance()); + dialog.exec(); +} + + +void MapEditorController::manageImageryNetworkPermissionsClicked() +{ + ImageryNetworkPermissionsDialog dialog( + imagery::ImageryNetworkPermissions::instance(), + window); + dialog.exec(); +} + + +void MapEditorController::setOfflineImagery(bool offline) +{ + imagery::TileNetworkManager::instance().setOfflineMode(offline); + window->showStatusBarMessage( + offline + ? tr("Imagery is offline; only cached tiles will be used.") + : tr("Imagery is online."), + 3000); +} + + void MapEditorController::reopenTemplateClicked() { hideAllTemplates(false); diff --git a/src/gui/map/map_editor.h b/src/gui/map/map_editor.h index 1e8bdee96..a9fe7bd61 100644 --- a/src/gui/map/map_editor.h +++ b/src/gui/map/map_editor.h @@ -356,6 +356,14 @@ public slots: void showTemplateWindow(bool show); /** Shows a file selector to open a template. */ void openTemplateClicked(); + /** Shows the online imagery source browser. */ + void addOnlineImageryClicked(); + /** Shows installed imagery catalogs. */ + void manageImageryCatalogsClicked(); + /** Reviews installation-local private-network imagery grants. */ + void manageImageryNetworkPermissionsClicked(); + /** Enables or disables application-wide offline imagery mode. */ + void setOfflineImagery(bool offline); /** Shows the ReopenTemplateDialog. */ void reopenTemplateClicked(); /** Adjusts action availability based on the presence of templates */ @@ -769,6 +777,10 @@ protected slots: QPointer template_dock_widget; TemplateListWidget* template_list_widget; QAction* open_template_act = {}; + QAction* online_imagery_act = {}; + QAction* manage_imagery_catalogs_act = {}; + QAction* manage_imagery_network_permissions_act = {}; + QAction* offline_imagery_act = {}; QAction* reopen_template_act = {}; QAction* tags_window_act = {}; diff --git a/src/gui/print_progress_dialog.cpp b/src/gui/print_progress_dialog.cpp index 0147d5ae9..b480df07c 100644 --- a/src/gui/print_progress_dialog.cpp +++ b/src/gui/print_progress_dialog.cpp @@ -38,6 +38,8 @@ PrintProgressDialog::PrintProgressDialog(MapPrinter* map_printer, QWidget* paren setWindowModality(Qt::ApplicationModal); // Required for OSX, cf. QTBUG-40112 setRange(0, 100); setMinimumDuration(0); + setAutoReset(false); + setAutoClose(false); setValue(0); Q_ASSERT(map_printer); @@ -52,11 +54,17 @@ PrintProgressDialog::~PrintProgressDialog() void PrintProgressDialog::paintRequested(QPrinter* printer) { + reset(); + setValue(0); if (!map_printer->printMap(printer)) { + if (wasCanceled() || map_printer->outputWasCanceled()) + return; QMessageBox::warning( parentWidget(), tr("Printing", "PrintWidget"), - tr("An error occurred during processing.", "PrintWidget"), + map_printer->outputError().isEmpty() + ? tr("An error occurred during processing.", "PrintWidget") + : map_printer->outputError(), QMessageBox::Ok, QMessageBox::Ok ); } } @@ -65,12 +73,18 @@ void PrintProgressDialog::setProgress(int value, const QString& status) { setLabelText(status); setValue(value); - if (!isVisible() && value < maximum()) + if (value >= maximum()) + { + hide(); + } + else if (!isVisible()) { show(); } - QApplication::processEvents(QEventLoop::ExcludeUserInputEvents, 100 /* ms */); // Drawing and Cancel events + // The dialog is application-modal, so accepting all events lets its Cancel + // button work without exposing the rest of the UI to re-entrant input. + QApplication::processEvents(QEventLoop::AllEvents, 100 /* ms */); } diff --git a/src/gui/print_widget.cpp b/src/gui/print_widget.cpp index da6080029..036b4cd1a 100644 --- a/src/gui/print_widget.cpp +++ b/src/gui/print_widget.cpp @@ -25,29 +25,38 @@ #include "gui/action_icon.h" #include +#include +#include // IWYU pragma: no_include #include #include #include // IWYU pragma: keep +#include #include #include #include #include +#include #include #include +#include #include +#include #include #include #include #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -65,6 +74,7 @@ #include #include #include +#include #include #include #include @@ -75,8 +85,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -107,7 +119,441 @@ namespace OpenOrienteering { namespace { - + + constexpr qint64 max_world_file_backup_size = 1024 * 1024; + constexpr qint64 max_image_transaction_size = 2 * 1024 * 1024; + constexpr auto image_transaction_format = + "org.openorienteering.image-export-transaction"; + + struct FileBackup + { + bool existed = false; + QByteArray contents; + QFileDevice::Permissions permissions; + }; + + struct ImageExportTransaction + { + bool old_image_existed = false; + QString image_backup_name; + FileBackup world_backup; + }; + + void setExportError(QString* error_message, QString message) + { + if (error_message) + *error_message = std::move(message); + } + + bool readFileBackup( + const QString& path, + FileBackup& backup, + QString* error_message) + { + const QFileInfo info(path); + backup.existed = info.exists() || info.isSymLink(); + if (!backup.existed) + return true; + + if (info.isSymLink() || !info.isFile()) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The existing world-file destination is not a regular file.")); + return false; + } + if (info.size() > max_world_file_backup_size) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The existing world file is unexpectedly large and was not replaced.")); + return false; + } + + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to preserve the existing world file:\n%1") + .arg(file.errorString())); + return false; + } + backup.contents = file.readAll(); + backup.permissions = file.permissions(); + if (file.error() != QFileDevice::NoError) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to preserve the existing world file:\n%1") + .arg(file.errorString())); + return false; + } + return true; + } + + bool writeWorldFile(QIODevice& device, const WorldFile& world_file) + { + QTextStream stream(&device); + stream.setRealNumberPrecision(10); + for (auto value : world_file.parameters) + stream << value << Qt::endl; + return stream.status() == QTextStream::Ok; + } + + bool restoreFile( + const QString& path, + const FileBackup& backup, + QString* error_message) + { + if (!backup.existed) + { + if (!QFileInfo::exists(path) || QFile::remove(path)) + return true; + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to remove the incomplete world file.")); + return false; + } + + QSaveFile output(path); + output.setDirectWriteFallback(false); + if (!output.open(QIODevice::WriteOnly) + || output.write(backup.contents) != qint64(backup.contents.size()) + || !output.commit()) + { + output.cancelWriting(); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to restore the previous world file:\n%1") + .arg(output.errorString())); + return false; + } + if (!QFile::setPermissions(path, backup.permissions)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The previous world-file contents were restored, but its permissions could not be restored.")); + return false; + } + return true; + } + + QString imageTransactionPath( + const QString& image_path) + { + return QFileInfo(image_path).absoluteFilePath() + + QStringLiteral(".mapper-export.json"); + } + + QString imageTransactionLockPath( + const QString& image_path) + { + return QFileInfo(image_path).absoluteFilePath() + + QStringLiteral(".mapper-export.lock"); + } + + QString imageBackupPrefix( + const QString& image_path) + { + return QLatin1Char('.') + + QFileInfo(image_path).fileName() + + QStringLiteral(".mapper-backup-"); + } + + bool writeImageTransaction( + const QString& image_path, + const ImageExportTransaction& transaction, + QString* error_message) + { + QJsonObject object { + { QStringLiteral("format"), + QString::fromLatin1( + image_transaction_format) }, + { QStringLiteral("version"), 1 }, + { QStringLiteral("oldImageExisted"), + transaction.old_image_existed }, + { QStringLiteral("imageBackupName"), + transaction.image_backup_name }, + { QStringLiteral("worldExisted"), + transaction.world_backup.existed }, + { QStringLiteral("worldContentsBase64"), + QString::fromLatin1( + transaction.world_backup.contents + .toBase64()) }, + { QStringLiteral("worldPermissions"), + int(transaction.world_backup.permissions) }, + }; + auto const bytes = + QJsonDocument(object).toJson( + QJsonDocument::Compact); + if (bytes.size() > max_image_transaction_size) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The image export recovery record is too large.")); + return false; + } + QSaveFile output( + imageTransactionPath(image_path)); + output.setDirectWriteFallback(false); + if (!output.open(QIODevice::WriteOnly) + || output.write(bytes) != bytes.size() + || !output.commit()) + { + output.cancelWriting(); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to create the image export recovery record:\n%1") + .arg(output.errorString())); + return false; + } + return true; + } + + std::optional + readImageTransaction( + const QString& image_path, + QString* error_message) + { + auto const transaction_path = + imageTransactionPath(image_path); + QFile input(transaction_path); + if (!input.exists()) + return ImageExportTransaction {}; + if (!input.open(QIODevice::ReadOnly) + || input.size() < 0 + || input.size() > max_image_transaction_size) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The image export recovery record cannot be read safely.")); + return std::nullopt; + } + QJsonParseError parse_error; + auto const document = + QJsonDocument::fromJson( + input.readAll(), + &parse_error); + if (parse_error.error + != QJsonParseError::NoError + || !document.isObject()) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The image export recovery record is invalid.")); + return std::nullopt; + } + auto const object = document.object(); + auto const prefix = + imageBackupPrefix(image_path); + auto const backup_name = + object.value( + QStringLiteral("imageBackupName")) + .toString(); + auto const backup_pattern = + QRegularExpression( + QStringLiteral("^%1[0-9a-f]{32}$") + .arg( + QRegularExpression::escape( + prefix))); + auto decoded_world = + QByteArray::fromBase64Encoding( + object.value( + QStringLiteral( + "worldContentsBase64")) + .toString() + .toLatin1(), + QByteArray:: + AbortOnBase64DecodingErrors); + if (object.value(QStringLiteral("format")) + .toString() + != QLatin1String( + image_transaction_format) + || object.value(QStringLiteral("version")) + .toInt(-1) + != 1 + || !backup_pattern.match(backup_name) + .hasMatch() + || !decoded_world + || decoded_world.decoded.size() + > max_world_file_backup_size) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The image export recovery record is invalid.")); + return std::nullopt; + } + + ImageExportTransaction transaction; + transaction.old_image_existed = + object.value( + QStringLiteral("oldImageExisted")) + .toBool(); + transaction.image_backup_name = + backup_name; + transaction.world_backup.existed = + object.value( + QStringLiteral("worldExisted")) + .toBool(); + transaction.world_backup.contents = + std::move(decoded_world.decoded); + transaction.world_backup.permissions = + QFileDevice::Permissions::fromInt( + object.value( + QStringLiteral( + "worldPermissions")) + .toInt()); + return transaction; + } + + bool recoverImageTransaction( + const QString& image_path, + const QString& world_path, + QString* error_message) + { + auto const transaction_path = + imageTransactionPath(image_path); + if (!QFileInfo::exists(transaction_path)) + return true; + auto transaction = + readImageTransaction( + image_path, + error_message); + if (!transaction) + return false; + + auto const directory = + QFileInfo(image_path).absoluteDir(); + auto const backup_path = + directory.filePath( + transaction->image_backup_name); + auto const image_info = + QFileInfo(image_path); + auto const backup_info = + QFileInfo(backup_path); + if ((image_info.exists() + && (image_info.isSymLink() + || !image_info.isFile())) + || (backup_info.exists() + && (backup_info.isSymLink() + || !backup_info.isFile()))) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The interrupted image export contains an unsafe destination.")); + return false; + } + + auto const image_exists = + image_info.exists(); + auto const backup_exists = + backup_info.exists(); + auto const published = + image_exists + && (!transaction->old_image_existed + || backup_exists); + auto const unchanged = + transaction->old_image_existed + && image_exists + && !backup_exists; + if (published || unchanged) + { + if (backup_exists + && !QFile::remove(backup_path)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to remove the completed image export backup.")); + return false; + } + } + else + { + // Keep the destination image absent until its matching world file is + // restored. If either operation fails, the journal and image backup + // remain sufficient for an idempotent retry. + if (!restoreFile( + world_path, + transaction->world_backup, + error_message)) + return false; + if (transaction->old_image_existed) + { + if (!backup_exists) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The interrupted image export lost both image copies.")); + return false; + } + if ((image_exists + && !QFile::remove(image_path)) + || !QFile::rename( + backup_path, + image_path)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to restore the previous image after an interrupted export.")); + return false; + } + } + else if (backup_exists + && !QFile::remove(backup_path)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to remove an incomplete image export backup.")); + return false; + } + } + if (!QFile::remove(transaction_path)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to finish recovering an interrupted image export.")); + return false; + } + return true; + } + QToolButton* createPrintModeButton(const QIcon& icon, const QString& label, QWidget* parent = nullptr) { static const QSize icon_size(48,48); @@ -125,6 +571,242 @@ namespace { } // namespace +bool PrintWidgetUtil::saveImageExport( + const QString& image_path, + const QImage& image, + const QByteArray& format, + const WorldFile* world_file, + QString* error_message) +{ + if (error_message) + error_message->clear(); + + auto const absolute_image_path = + QFileInfo(image_path).absoluteFilePath(); + auto const world_path = + WorldFile::pathForImage( + absolute_image_path); + QLockFile output_lock( + imageTransactionLockPath( + absolute_image_path)); + if (!output_lock.tryLock()) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Another process is already exporting this image.")); + return false; + } + if (!recoverImageTransaction( + absolute_image_path, + world_path, + error_message)) + return false; + + QSaveFile image_output(absolute_image_path); + image_output.setDirectWriteFallback(false); + if (!image_output.open(QIODevice::WriteOnly)) + { + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to open the image destination:\n%1") + .arg(image_output.errorString())); + return false; + } + if (!image.save( + &image_output, + format.isEmpty() ? nullptr : format.constData())) + { + image_output.cancelWriting(); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to encode the image.")); + return false; + } + + if (!world_file) + { + if (image_output.commit()) + return true; + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to finish saving the image:\n%1") + .arg(image_output.errorString())); + return false; + } + + FileBackup world_backup; + if (!readFileBackup(world_path, world_backup, error_message)) + { + image_output.cancelWriting(); + return false; + } + + QSaveFile world_output(world_path); + world_output.setDirectWriteFallback(false); + if (!world_output.open(QIODevice::WriteOnly | QIODevice::Text)) + { + image_output.cancelWriting(); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to open the world-file destination:\n%1") + .arg(world_output.errorString())); + return false; + } + if (!writeWorldFile(world_output, *world_file)) + { + world_output.cancelWriting(); + image_output.cancelWriting(); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to encode the world file.")); + return false; + } + + auto const image_info = + QFileInfo(absolute_image_path); + ImageExportTransaction transaction; + transaction.old_image_existed = + image_info.exists() + || image_info.isSymLink(); + if (transaction.old_image_existed + && (image_info.isSymLink() + || !image_info.isFile())) + { + world_output.cancelWriting(); + image_output.cancelWriting(); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The existing image destination is not a regular file.")); + return false; + } + transaction.image_backup_name = + imageBackupPrefix(absolute_image_path) + + QUuid::createUuid().toString( + QUuid::Id128); + transaction.world_backup = world_backup; + if (!writeImageTransaction( + absolute_image_path, + transaction, + error_message)) + { + world_output.cancelWriting(); + image_output.cancelWriting(); + return false; + } + auto const image_backup_path = + QFileInfo(absolute_image_path) + .absoluteDir() + .filePath( + transaction.image_backup_name); + if (transaction.old_image_existed + && !QFile::rename( + absolute_image_path, + image_backup_path)) + { + world_output.cancelWriting(); + image_output.cancelWriting(); + QFile::remove( + imageTransactionPath( + absolute_image_path)); + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "Failed to preserve the previous image before publishing the image and world file.")); + return false; + } + + // Publishing two conventional files cannot be one rename. Remove the old + // image into an atomic sibling backup before committing the sidecar, so a + // crash can expose either the old pair, no image, or the new pair—but never + // the old image with new georeferencing. The bounded journal makes the + // missing-image phase recoverable on the next export. + if (!world_output.commit()) + { + image_output.cancelWriting(); + auto const world_error = + world_output.errorString(); + QString recovery_error; + auto const recovered = + recoverImageTransaction( + absolute_image_path, + world_path, + &recovery_error); + setExportError( + error_message, + recovered + ? QCoreApplication::translate( + "PrintWidget", + "Failed to finish saving the world file:\n%1") + .arg(world_error) + : QCoreApplication::translate( + "PrintWidget", + "Failed to finish saving the world file:\n%1\n\n" + "Recovery also failed:\n%2") + .arg( + world_error, + recovery_error)); + return false; + } + if (image_output.commit()) + { + auto const backup_removed = + !QFileInfo::exists(image_backup_path) + || QFile::remove(image_backup_path); + auto const transaction_path = + imageTransactionPath( + absolute_image_path); + auto const journal_removed = + backup_removed + && (!QFileInfo::exists(transaction_path) + || QFile::remove(transaction_path)); + if (backup_removed && journal_removed) + return true; + setExportError( + error_message, + QCoreApplication::translate( + "PrintWidget", + "The image and world file were saved, but their recovery files could not be removed.")); + return false; + } + + const auto image_error = image_output.errorString(); + QString recovery_error; + const auto recovery_ok = + recoverImageTransaction( + absolute_image_path, + world_path, + &recovery_error); + setExportError( + error_message, + recovery_ok + ? QCoreApplication::translate( + "PrintWidget", + "Failed to finish saving the image:\n%1") + .arg(image_error) + : QCoreApplication::translate( + "PrintWidget", + "Failed to finish saving the image:\n%1\n\n" + "Recovery also failed:\n%2") + .arg(image_error, recovery_error)); + return false; +} + + //### PrintWidget ### PrintWidget::PrintWidget(Map* map, MainWindow* main_window, MapView* main_view, MapEditorController* editor, QWidget* parent) @@ -1232,18 +1914,44 @@ void PrintWidget::exportToKmz() progress.setWindowModality(Qt::ApplicationModal); // Required for OSX, cf. QTBUG-40112 progress.setWindowTitle(tr("Export map ...")); progress.setMinimumDuration(500); - progress.setAutoClose(true); + progress.setAutoReset(false); + progress.setAutoClose(false); + progress.setRange(0, 100); + connect( + map_printer, &MapPrinter::printProgress, + &progress, + [&progress](int value, const QString& status) { + progress.setLabelText(status); + progress.setValue(value); + QApplication::processEvents( + QEventLoop::AllEvents, 100); + }); + connect( + &progress, &QProgressDialog::canceled, + map_printer, &MapPrinter::cancelPrintMap); KmzGroundOverlayExport exporter(path, *map); exporter.setProgressObserver(&progress); if (!exporter.doExport(*map_printer, tile_size_combo->currentData().toInt())) { - progress.cancel(); - QMessageBox::warning(this, tr("Error"), tr("Failed to save the image:\n%1").arg(exporter.errorString())); - main_window->showStatusBarMessage(tr("Canceled."), 4000); + progress.hide(); + if (exporter.wasCanceled()) + { + main_window->showStatusBarMessage( + tr("Canceled."), 4000); + } + else + { + QMessageBox::warning( + this, + tr("Error"), + tr("Failed to export the map:\n%1") + .arg(exporter.errorString())); + } } else { + progress.hide(); main_window->showStatusBarMessage(tr("Exported successfully to %1").arg(path), 4000); emit finished(0); } @@ -1295,32 +2003,74 @@ void PrintWidget::exportToImage() image.fill(QColor(transparent_background ? Qt::transparent : Qt::white)); -#if 0 // Pointless unless drawPage drives the event loop and sends progress PrintProgressDialog progress(map_printer, main_window); progress.setWindowTitle(tr("Export map ...")); -#endif + if (!map_printer->prepareOutput()) + { + if (!progress.wasCanceled() + && !map_printer->outputWasCanceled()) + { + QMessageBox::warning( + this, tr("Error"), + map_printer->outputError().isEmpty() + ? tr("Failed to prepare the image.") + : map_printer->outputError()); + } + return; + } // Export the map QPainter p(&image); map_printer->drawPage(&p, map_printer->getPrintArea(), &image); - p.end(); - if (!image.save(path)) + auto const render_ok = p.isActive(); + if (render_ok) + p.end(); + map_printer->finishOutput(!render_ok); + if (!render_ok) { - QMessageBox::warning(this, tr("Error"), tr("Failed to save the image. Does the path exist? Do you have sufficient rights?")); + QMessageBox::warning( + this, + tr("Error"), + map_printer->outputError().isEmpty() + ? tr("Failed to render exact template imagery.") + : map_printer->outputError()); + return; + } + + auto const format = + QFileInfo(path).suffix().toLatin1(); + const auto save_world_file = world_file_check->isChecked(); + WorldFile world_file; + if (save_world_file) + world_file = worldFileForExport(); + QString save_error; + if (!PrintWidgetUtil::saveImageExport( + path, + image, + format, + save_world_file ? &world_file : nullptr, + &save_error)) + { + progress.hide(); + QMessageBox::warning( + this, + tr("Error"), + save_world_file + ? tr("Failed to save the image and world file:\n%1") + .arg(save_error) + : tr("Failed to save the image:\n%1") + .arg(save_error)); } else { + progress.setValue(100); + progress.hide(); main_window->showStatusBarMessage(tr("Exported successfully to %1").arg(path), 4000); - if (world_file_check->isChecked()) - { - if (!exportWorldFile(path)) - QMessageBox::warning(this, tr("Error"), tr("Failed to save the world file.")); - } emit finished(0); } } -bool PrintWidget::exportWorldFile(const QString& path) const +WorldFile PrintWidget::worldFileForExport() const { const auto& georef = map->getGeoreferencing(); const auto& mm_to_world = georef.mapToProjected(); @@ -1331,8 +2081,7 @@ bool PrintWidget::exportWorldFile(const QString& path) const const auto yskew = mm_to_world.m21() / pixel_per_mm; const auto top_left = georef.toProjectedCoords(MapCoord{map_printer->getPrintArea().topLeft()}); const QTransform pixel_to_world(xscale, yskew, xskew, yscale, top_left.x(), top_left.y()); - const WorldFile world_file(pixel_to_world); - return world_file.save(WorldFile::pathForImage(path)); + return WorldFile(pixel_to_world); } void PrintWidget::exportToPdf() @@ -1349,7 +2098,17 @@ void PrintWidget::exportToPdf() { path.append(QLatin1String(".pdf")); } - auto writer = map_printer->makePdfWriter(path); + QSaveFile output(path); + if (!output.open(QIODevice::WriteOnly)) + { + QMessageBox::warning( + this, + tr("Error"), + tr("Failed to open the PDF destination:\n%1") + .arg(output.errorString())); + return; + } + auto writer = map_printer->makePdfWriter(&output); writer->setCreator(main_window->appName()); writer->setTitle(QFileInfo(main_window->currentPath()).baseName()); @@ -1357,20 +2116,36 @@ void PrintWidget::exportToPdf() progress.setWindowTitle(tr("Export map ...")); // Export the map - if (!map_printer->printMap(writer.get(), copies_edit->value())) + auto const success = + map_printer->printMap(writer.get(), copies_edit->value()); + writer.reset(); + if (progress.wasCanceled() + || map_printer->outputWasCanceled()) { - QFile(path).remove(); - QMessageBox::warning(this, tr("Error"), tr("Failed to finish the PDF export.")); + output.cancelWriting(); + main_window->showStatusBarMessage(tr("Canceled."), 4000); } - else if (!progress.wasCanceled()) + else if (!success) { - main_window->showStatusBarMessage(tr("Exported successfully to %1").arg(path), 4000); - emit finished(0); + output.cancelWriting(); + QMessageBox::warning( + this, tr("Error"), + map_printer->outputError().isEmpty() + ? tr("Failed to finish the PDF export.") + : map_printer->outputError()); + } + else if (!output.commit()) + { + QMessageBox::warning( + this, + tr("Error"), + tr("Failed to finish the PDF export:\n%1") + .arg(output.errorString())); } else { - QFile(path).remove(); - main_window->showStatusBarMessage(tr("Canceled."), 4000); + main_window->showStatusBarMessage(tr("Exported successfully to %1").arg(path), 4000); + emit finished(0); } } @@ -1403,23 +2178,40 @@ void PrintWidget::print() progress.setWindowTitle(tr("Printing Progress")); // Print the map - if (!map_printer->printMap(printer.get())) + auto const success = map_printer->printMap(printer.get()); + const auto canceled = + progress.wasCanceled() || map_printer->outputWasCanceled(); + if (!success || canceled) { - QMessageBox::warning(main_window, tr("Error"), tr("An error occurred during printing.")); + const auto aborted = printer->abort(); + if (canceled) + { + if (aborted) + main_window->showStatusBarMessage(tr("Canceled."), 4000); + else + QMessageBox::warning( + main_window, tr("Error"), + tr("The print job could not be stopped.")); + } + else + { + auto error = + map_printer->outputError().isEmpty() + ? tr("An error occurred during printing.") + : map_printer->outputError(); + if (!aborted) + error.append( + tr("\n\nThe incomplete print job could not be stopped.")); + QMessageBox::warning( + main_window, tr("Error"), + error); + } } - else if (!progress.wasCanceled()) + else { main_window->showStatusBarMessage(tr("Successfully created print job"), 4000); emit finished(0); } - else if (printer->abort()) - { - main_window->showStatusBarMessage(tr("Canceled."), 4000); - } - else - { - QMessageBox::warning(main_window, tr("Error"), tr("The print job could not be stopped.")); - } } QList PrintWidget::defaultPageSizes() const diff --git a/src/gui/print_widget.h b/src/gui/print_widget.h index 0e743f3f0..1af168632 100644 --- a/src/gui/print_widget.h +++ b/src/gui/print_widget.h @@ -35,12 +35,14 @@ class QAbstractButton; +class QByteArray; class QButtonGroup; class QCheckBox; class QComboBox; class QDialogButtonBox; class QDoubleSpinBox; class QFormLayout; +class QImage; class QLabel; class QPageSize; class QPushButton; @@ -60,6 +62,28 @@ class MapPrinterOptions; class MapPrinterPageFormat; class MapView; class PrintTool; +struct WorldFile; + + +namespace PrintWidgetUtil { + +/** + * Transactionally saves an image, optionally with its world-file sidecar. + * + * Both files are staged and writers are serialized per destination. For a + * paired export, the old image is moved to a sibling backup before the world + * file is committed, so a crash never exposes old pixels with new + * georeferencing. A bounded journal rolls back the missing-image phase or + * completes cleanup on the next export. + */ +bool saveImageExport( + const QString& image_path, + const QImage& image, + const QByteArray& format, + const WorldFile* world_file, + QString* error_message = nullptr); + +} // namespace PrintWidgetUtil /** @@ -259,8 +283,8 @@ protected slots: /** Exports to an image file. */ void exportToImage(); - /** Export a world file */ - bool exportWorldFile(const QString& path) const; + /** Creates the world-file parameters for an image export. */ + WorldFile worldFileForExport() const; /** Exports to a PDF file. */ void exportToPdf(); diff --git a/src/gui/widgets/template_list_widget.cpp b/src/gui/widgets/template_list_widget.cpp index fd70f9c3d..ba8972e84 100644 --- a/src/gui/widgets/template_list_widget.cpp +++ b/src/gui/widgets/template_list_widget.cpp @@ -239,9 +239,17 @@ TemplateListWidget::TemplateListWidget(Map& map, MapView& main_view, MapEditorCo auto* new_button_menu = new QMenu(this); if (!mobile_mode) { + new_button_menu->addAction( + controller.getAction("openonlineimagery")); + new_button_menu->addSeparator(); new_button_menu->addAction(ActionIcon::fromName(u"open"), tr("Open..."), this, &TemplateListWidget::openTemplate); new_button_menu->addAction(controller.getAction("reopentemplate")); } + else + { + new_button_menu->addAction( + controller.getAction("openonlineimagery")); + } duplicate_action = new_button_menu->addAction(ActionIcon::fromName(u"tool-duplicate"), tr("Duplicate"), this, &TemplateListWidget::duplicateTemplate); #if 0 current_action = new_button_menu->addAction(tr("Sketch")); diff --git a/src/imagery/CMakeLists.txt b/src/imagery/CMakeLists.txt index d84ca2e70..bc9b7a357 100644 --- a/src/imagery/CMakeLists.txt +++ b/src/imagery/CMakeLists.txt @@ -5,10 +5,18 @@ # set(MAPPER_IMAGERY_CORE_SOURCES + arcgis_tile_service.cpp + arcgis_tile_service.h + imagery_catalog_store.cpp + imagery_catalog_store.h + manual_imagery_source.cpp + manual_imagery_source.h imagery_source.cpp imagery_source.h imagery_source_snapshot.cpp imagery_source_snapshot.h + oic_catalog.cpp + oic_catalog.h tile_matrix_set.cpp tile_matrix_set.h ) @@ -31,14 +39,24 @@ set_target_properties(mapper-imagery-core PROPERTIES mapper_translations_sources(${MAPPER_IMAGERY_CORE_SOURCES}) -add_library(mapper-imagery-network STATIC +set(MAPPER_IMAGERY_NETWORK_SOURCES + imagery_catalog_repository.cpp + imagery_catalog_repository.h + imagery_network_permissions.cpp + imagery_network_permissions.h tile_network_manager.cpp tile_network_manager.h ) + +add_library(mapper-imagery-network STATIC ${MAPPER_IMAGERY_NETWORK_SOURCES}) add_library(Mapper::ImageryNetwork ALIAS mapper-imagery-network) mapper_target_defaults(mapper-imagery-network) target_include_directories(mapper-imagery-network PUBLIC "${PROJECT_SOURCE_DIR}/src") -target_link_libraries(mapper-imagery-network PUBLIC Qt6::Core Qt6::Network) +target_link_libraries(mapper-imagery-network PUBLIC + Mapper::ImageryCore + Qt6::Core + Qt6::Network +) target_compile_definitions(mapper-imagery-network PRIVATE QT_NO_CAST_FROM_ASCII QT_NO_CAST_TO_ASCII @@ -46,4 +64,4 @@ target_compile_definitions(mapper-imagery-network PRIVATE ) set_target_properties(mapper-imagery-network PROPERTIES PREFIX "") -mapper_translations_sources(tile_network_manager.cpp tile_network_manager.h) +mapper_translations_sources(${MAPPER_IMAGERY_NETWORK_SOURCES}) diff --git a/src/imagery/arcgis_tile_service.cpp b/src/imagery/arcgis_tile_service.cpp new file mode 100644 index 000000000..4e039302a --- /dev/null +++ b/src/imagery/arcgis_tile_service.cpp @@ -0,0 +1,868 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "imagery/arcgis_tile_service.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/manual_imagery_source.h" + +namespace OpenOrienteering::imagery { + +namespace { + +constexpr auto maximum_published_tile_dimension = 4096; +constexpr qint64 maximum_published_tile_pixels = + qint64(8) * 1024 * 1024; + +ArcGisTileServiceResult invalid(QString detail) +{ + ArcGisTileServiceResult result; + result.outcome = ArcGisTileServiceOutcome::Invalid; + result.detail = std::move(detail); + return result; +} + +ArcGisTileServiceResult unsupported(QString detail) +{ + ArcGisTileServiceResult result; + result.outcome = ArcGisTileServiceOutcome::Unsupported; + result.detail = std::move(detail); + return result; +} + +bool finiteNumber(const QJsonValue& value) +{ + return value.isDouble() && std::isfinite(value.toDouble()); +} + +bool exactInteger( + const QJsonValue& value, + qint64 minimum, + qint64 maximum, + qint64* output) +{ + if (!finiteNumber(value)) + return false; + auto const number = value.toDouble(); + if (std::floor(number) != number + || number < double(minimum) || number > double(maximum) + || std::abs(number) > 9007199254740991.0) + { + return false; + } + *output = qint64(number); + return true; +} + +bool finiteMember( + const QJsonObject& object, + const QString& name, + double* output) +{ + auto const value = object.value(name); + if (!finiteNumber(value)) + return false; + *output = value.toDouble(); + return true; +} + +std::optional normalizedEpsg( + const QJsonObject& spatial_reference, + bool* malformed) +{ + *malformed = false; + for (auto const& name : { + QStringLiteral("latestWkid"), QStringLiteral("wkid") + }) + { + if (!spatial_reference.contains(name)) + continue; + qint64 wkid = 0; + if (!exactInteger( + spatial_reference.value(name), 1, 999999999, &wkid)) + { + *malformed = true; + return std::nullopt; + } + if (wkid == 102100 || wkid == 102113 || wkid == 900913) + return 3857; + if (wkid <= 99999) + return int(wkid); + } + return std::nullopt; +} + +bool validServiceUrl(const QUrl& url) +{ + auto const scheme = url.scheme().toLower(); + return url.isValid() && !url.isRelative() && !url.host().isEmpty() + && (scheme == QLatin1String("http") + || scheme == QLatin1String("https")) + && url.userName().isEmpty() && url.password().isEmpty() + && !url.hasFragment() + && url.toString(QUrl::FullyEncoded).size() <= 8192; +} + +QString filteredServiceQuery(const QUrl& url) +{ + QStringList retained; + for (auto const& item : + url.query(QUrl::FullyEncoded).split( + QLatin1Char('&'), Qt::SkipEmptyParts)) + { + auto const separator = item.indexOf(QLatin1Char('=')); + auto const encoded_name = + separator < 0 ? item : item.left(separator); + auto const name = + QUrl::fromPercentEncoding(encoded_name.toUtf8()); + if (name.compare( + QStringLiteral("f"), Qt::CaseInsensitive) != 0 + && name.compare( + QStringLiteral("callback"), Qt::CaseInsensitive) != 0) + { + retained.push_back(item); + } + } + return retained.join(QLatin1Char('&')); +} + +std::optional normalizedServiceUrl(const QUrl& input) +{ + if (!validServiceUrl(input)) + return std::nullopt; + static const QRegularExpression pattern( + QStringLiteral( + "^(.*/rest/services/.*/(?:MapServer|ImageServer))" + "(?:/tile/[^/]+/[^/]+/[^/]+)?/?$" + ), + QRegularExpression::CaseInsensitiveOption + ); + auto const match = pattern.match(input.path()); + if (!match.hasMatch()) + return std::nullopt; + + auto result = input; + result.setPath(match.captured(1)); + result.setFragment({}); + result.setQuery(filteredServiceQuery(input), QUrl::StrictMode); + return result; +} + +double snapNearInteger(double value) +{ + auto const nearest = std::round(value); + auto const tolerance = + std::max(1.0, std::abs(value)) * 1.0e-10; + return std::abs(value - nearest) <= tolerance + ? nearest + : value; +} + +bool extentIndex( + double value, + bool upper, + qint64 minimum, + qint64 maximum, + qint64* output) +{ + if (!std::isfinite(value)) + return false; + auto const snapped = snapNearInteger(value); + auto const indexed = upper + ? std::ceil(snapped) - 1.0 + : std::floor(snapped); + if (!std::isfinite(indexed) + || indexed < double(minimum) || indexed > double(maximum)) + { + return false; + } + *output = qint64(indexed); + return true; +} + +bool coveringDimension(double value, qint64* output) +{ + if (!std::isfinite(value) || value <= 0) + return false; + auto const dimension = std::ceil(snapNearInteger(value)); + if (!std::isfinite(dimension) || dimension < 1 + || dimension > double(std::numeric_limits::max())) + { + return false; + } + *output = qint64(dimension); + return true; +} + +QString generatedId(const QUrl& service_url) +{ + auto const digest = QCryptographicHash::hash( + service_url.toString(QUrl::FullyEncoded).toUtf8(), + QCryptographicHash::Sha256 + ).toHex(); + return QStringLiteral("arcgis-%1") + .arg(QString::fromLatin1(digest.first(16))); +} + +QString serviceTitle( + const QJsonObject& root, + const QUrl& service_url) +{ + for (auto const& value : { + root.value(QStringLiteral("name")), + root.value(QStringLiteral("mapName")), + }) + { + if (value.isString() && !value.toString().trimmed().isEmpty()) + return value.toString().trimmed(); + } + auto const document_info = + root.value(QStringLiteral("documentInfo")).toObject(); + auto const title = + document_info.value(QStringLiteral("Title")).toString().trimmed(); + if (!title.isEmpty()) + return title; + + auto path = service_url.path(); + static const QRegularExpression suffix( + QStringLiteral("/(?:MapServer|ImageServer)$"), + QRegularExpression::CaseInsensitiveOption + ); + path.remove(suffix); + auto result = path.section(QLatin1Char('/'), -1); + if (result.isEmpty()) + result = service_url.host().toLower(); + return result; +} + +QString mediaType(const QJsonObject& tile_info) +{ + if (!tile_info.value(QStringLiteral("format")).isString()) + return {}; + auto format = + tile_info.value(QStringLiteral("format")) + .toString().trimmed().toUpper(); + if (format == QLatin1String("JPG") + || format == QLatin1String("JPEG")) + { + return QStringLiteral("image/jpeg"); + } + if (format.startsWith(QLatin1String("PNG")) + || format == QLatin1String("MIXED")) + { + return QStringLiteral("image/png"); + } + return {}; +} + +QString tileTemplate(const QUrl& service_url) +{ + auto endpoint = service_url; + auto const query = QUrlQuery(endpoint); + endpoint.setQuery(QString {}); + auto base = endpoint.toString(QUrl::FullyEncoded); + while (base.endsWith(QLatin1Char('/'))) + base.chop(1); + QString result = + base + QStringLiteral("/tile/{z}/{y}/{x}"); + auto const encoded_query = + query.toString(QUrl::FullyEncoded); + if (!encoded_query.isEmpty()) + { + result += QLatin1Char('?'); + result += encoded_query; + } + return result; +} + +} // namespace + +bool ArcGisTileServiceResult::resolved() const noexcept +{ + return outcome == ArcGisTileServiceOutcome::Resolved + && source.has_value(); +} + +ArcGisTileServiceResult ArcGisTileService::parse( + const QByteArray& pjson, + const QUrl& service_url, + const ArcGisTileServiceSettings& settings) +{ + if (pjson.isEmpty()) + return invalid(tr("ArcGIS metadata is empty.")); + if (pjson.size() > maximum_metadata_size) + { + return invalid(tr( + "ArcGIS metadata exceeds the 1 MiB safety limit." + )); + } + auto const normalized_url = normalizedServiceUrl(service_url); + if (!normalized_url) + { + return invalid(tr( + "The ArcGIS service URL must identify an HTTP(S) MapServer or ImageServer endpoint." + )); + } + + QJsonParseError parse_error; + auto const document = + QJsonDocument::fromJson(pjson, &parse_error); + if (parse_error.error != QJsonParseError::NoError + || !document.isObject()) + { + return invalid(tr( + "ArcGIS metadata is not a valid JSON object." + )); + } + auto const root = document.object(); + if (root.value(QStringLiteral("error")).isObject()) + { + auto const error_object = + root.value(QStringLiteral("error")).toObject(); + qint64 code = 0; + if (exactInteger( + error_object.value(QStringLiteral("code")), + 0, std::numeric_limits::max(), &code)) + { + return invalid(tr( + "ArcGIS service returned error code %1." + ).arg(code)); + } + return invalid(tr( + "ArcGIS service returned an error response." + )); + } + if (root.contains(QStringLiteral("singleFusedMapCache")) + && !root.value(QStringLiteral("singleFusedMapCache")).isBool()) + { + return invalid(tr( + "ArcGIS singleFusedMapCache must be a boolean." + )); + } + if (root.value(QStringLiteral("singleFusedMapCache")).isBool() + && !root.value(QStringLiteral("singleFusedMapCache")).toBool()) + { + return unsupported(tr( + "The ArcGIS service is not a fused tile cache." + )); + } + if (!root.value(QStringLiteral("tileInfo")).isObject()) + { + return unsupported(tr( + "The ArcGIS service does not publish cached tile metadata." + )); + } + auto const tile_info = + root.value(QStringLiteral("tileInfo")).toObject(); + + qint64 rows = 0; + qint64 columns = 0; + if (!exactInteger( + tile_info.value(QStringLiteral("rows")), + 1, maximum_published_tile_dimension, &rows) + || !exactInteger( + tile_info.value(QStringLiteral("cols")), + 1, maximum_published_tile_dimension, &columns)) + { + return invalid(tr( + "ArcGIS tile dimensions are missing or invalid." + )); + } + if (rows * columns > maximum_published_tile_pixels) + { + return unsupported(tr( + "ArcGIS tiles exceed the bounded raster decode profile." + )); + } + if (!runtimeSupportsTileSize( + QSize(int(columns), int(rows)))) + { + return unsupported(tr( + "ArcGIS tiles exceed this build's raster execution profile." + )); + } + + if (!tile_info.value(QStringLiteral("origin")).isObject()) + { + return invalid(tr( + "ArcGIS tile origin is missing." + )); + } + auto const origin = + tile_info.value(QStringLiteral("origin")).toObject(); + double origin_x = 0; + double origin_y = 0; + if (!finiteMember(origin, QStringLiteral("x"), &origin_x) + || !finiteMember(origin, QStringLiteral("y"), &origin_y)) + { + return invalid(tr( + "ArcGIS tile origin must contain finite x and y coordinates." + )); + } + + if (!tile_info.value(QStringLiteral("spatialReference")).isObject()) + { + return unsupported(tr( + "ArcGIS tile metadata has no numeric spatial reference." + )); + } + bool malformed_crs = false; + auto const epsg = normalizedEpsg( + tile_info.value(QStringLiteral("spatialReference")).toObject(), + &malformed_crs + ); + if (malformed_crs) + { + return invalid(tr( + "ArcGIS spatial reference identifiers are malformed." + )); + } + if (!epsg) + { + return unsupported(tr( + "ArcGIS tile metadata uses a spatial reference that cannot be normalized to EPSG." + )); + } + if (root.contains(QStringLiteral("spatialReference"))) + { + if (!root.value(QStringLiteral("spatialReference")).isObject()) + { + return invalid(tr( + "ArcGIS service spatialReference is malformed." + )); + } + bool malformed_root_crs = false; + auto const root_epsg = normalizedEpsg( + root.value(QStringLiteral("spatialReference")).toObject(), + &malformed_root_crs + ); + if (malformed_root_crs) + { + return invalid(tr( + "ArcGIS service spatial reference is malformed." + )); + } + if (!root_epsg || *root_epsg != *epsg) + { + return unsupported(tr( + "ArcGIS service and tileInfo use different spatial references." + )); + } + } + auto const image_service = + normalized_url->path().endsWith( + QStringLiteral("/ImageServer"), Qt::CaseInsensitive + ); + if (image_service && root.contains(QStringLiteral("cacheType"))) + { + if (!root.value(QStringLiteral("cacheType")).isString()) + { + return invalid(tr( + "ArcGIS ImageServer cacheType must be a string." + )); + } + if (root.value(QStringLiteral("cacheType")).toString() + .compare(QStringLiteral("Map"), Qt::CaseInsensitive) != 0) + { + return unsupported(tr( + "ArcGIS ImageServer elevation and raster caches are not supported." + )); + } + } + + if (!root.value(QStringLiteral("fullExtent")).isObject()) + { + return unsupported(tr( + "ArcGIS fullExtent is required to derive finite tile matrix dimensions." + )); + } + auto const full_extent = + root.value(QStringLiteral("fullExtent")).toObject(); + double west = 0; + double south = 0; + double east = 0; + double north = 0; + if (!finiteMember(full_extent, QStringLiteral("xmin"), &west) + || !finiteMember(full_extent, QStringLiteral("ymin"), &south) + || !finiteMember(full_extent, QStringLiteral("xmax"), &east) + || !finiteMember(full_extent, QStringLiteral("ymax"), &north) + || !(west < east) || !(south < north)) + { + return invalid(tr( + "ArcGIS fullExtent must contain ordered finite bounds." + )); + } + if (full_extent.contains(QStringLiteral("spatialReference"))) + { + if (!full_extent.value( + QStringLiteral("spatialReference")).isObject()) + { + return invalid(tr( + "ArcGIS fullExtent spatialReference is malformed." + )); + } + bool malformed_extent_crs = false; + auto const extent_epsg = normalizedEpsg( + full_extent.value( + QStringLiteral("spatialReference")).toObject(), + &malformed_extent_crs + ); + if (malformed_extent_crs) + { + return invalid(tr( + "ArcGIS fullExtent spatial reference is malformed." + )); + } + if (!extent_epsg || *extent_epsg != *epsg) + { + return unsupported(tr( + "ArcGIS fullExtent and tileInfo use different spatial references." + )); + } + } + if (!tile_info.value(QStringLiteral("lods")).isArray()) + { + return invalid(tr( + "ArcGIS tile metadata has no LOD array." + )); + } + auto const lods = + tile_info.value(QStringLiteral("lods")).toArray(); + if (lods.isEmpty() || lods.size() > maximum_lods) + { + return unsupported(tr( + "ArcGIS LOD count is outside the supported range." + )); + } + + struct PublishedLod + { + int level = -1; + double resolution = 0; + }; + QVector published_lods; + published_lods.reserve(lods.size()); + for (qsizetype index = 0; index < lods.size(); ++index) + { + if (!lods.at(index).isObject()) + { + return invalid(tr( + "ArcGIS LOD entries must be objects." + )); + } + auto const lod = lods.at(index).toObject(); + qint64 level = -1; + double resolution = 0; + if (!exactInteger( + lod.value(QStringLiteral("level")), + 0, maximum_lods - 1, &level) + || !finiteMember( + lod, QStringLiteral("resolution"), &resolution) + || !(resolution > 0)) + { + return invalid(tr( + "ArcGIS LOD level or resolution is invalid." + )); + } + if (lod.contains(QStringLiteral("scale"))) + { + double scale = 0; + if (!finiteMember(lod, QStringLiteral("scale"), &scale) + || !(scale > 0)) + { + return invalid(tr( + "ArcGIS LOD scales must be finite and positive." + )); + } + } + published_lods.push_back({ int(level), resolution }); + } + std::sort( + published_lods.begin(), published_lods.end(), + [](auto const& first, auto const& second) { + return first.level < second.level; + } + ); + for (qsizetype index = 0; + index < published_lods.size(); ++index) + { + if (index > 0 + && published_lods.at(index - 1).level + == published_lods.at(index).level) + { + return invalid(tr( + "ArcGIS LOD levels must be unique." + )); + } + if (published_lods.at(index).level != index) + { + return unsupported(tr( + "ArcGIS LOD levels must begin at zero and be contiguous." + )); + } + } + + auto const base_resolution = published_lods.first().resolution; + for (qsizetype index = 0; + index < published_lods.size(); ++index) + { + auto const expected = + base_resolution / std::ldexp(1.0, int(index)); + auto const published = + published_lods.at(index).resolution; + auto const tolerance = + std::max({ 1.0e-15, expected, published }) * 1.0e-8; + if (std::abs(published - expected) > tolerance) + { + return unsupported(tr( + "ArcGIS LOD resolutions do not form a dyadic pyramid." + )); + } + } + + auto const highest_zoom = int(published_lods.size()) - 1; + auto minimum_zoom = 0; + auto maximum_zoom = highest_zoom; + for (auto const& item : { + std::pair { QStringLiteral("minLOD"), &minimum_zoom }, + std::pair { QStringLiteral("maxLOD"), &maximum_zoom }, + }) + { + if (!root.contains(item.first)) + continue; + qint64 value = 0; + if (!exactInteger( + root.value(item.first), 0, highest_zoom, &value)) + { + return invalid(tr( + "ArcGIS %1 is outside the published LOD range." + ).arg(item.first)); + } + *item.second = int(value); + } + if (minimum_zoom > maximum_zoom) + { + return invalid(tr( + "ArcGIS minLOD follows maxLOD." + )); + } + + auto const base_tile_width = + base_resolution * double(columns); + auto const base_tile_height = + base_resolution * double(rows); + qint64 base_matrix_width = 0; + qint64 base_matrix_height = 0; + constexpr auto web_mercator_half_world = 20037508.342789244; + auto const canonical_web_mercator = + *epsg == 3857 && rows == columns + && std::abs(origin_x + web_mercator_half_world) <= 0.02 + && std::abs(origin_y - web_mercator_half_world) <= 0.02 + && std::abs( + base_resolution + - (2 * web_mercator_half_world) / double(columns) + ) <= base_resolution * 1.0e-8; + auto runtime_base_resolution = base_resolution; + if (canonical_web_mercator) + { + origin_x = -web_mercator_half_world; + origin_y = web_mercator_half_world; + runtime_base_resolution = + (2 * web_mercator_half_world) / double(columns); + west = std::max(west, -web_mercator_half_world); + south = std::max(south, -web_mercator_half_world); + east = std::min(east, web_mercator_half_world); + north = std::min(north, web_mercator_half_world); + if (!(west < east) || !(south < north)) + { + return unsupported(tr( + "ArcGIS fullExtent does not intersect WebMercatorQuad." + )); + } + base_matrix_width = 1; + base_matrix_height = 1; + } + else + { + auto const extent_tolerance = + std::max({ + 1.0, std::abs(origin_x), std::abs(origin_y), + std::abs(west), std::abs(east), + std::abs(south), std::abs(north), + }) * 1.0e-12; + if (origin_x > west + extent_tolerance + || origin_y < north - extent_tolerance) + { + return unsupported(tr( + "ArcGIS fullExtent lies outside the top-left tile origin." + )); + } + if (!coveringDimension( + (east - origin_x) / base_tile_width, + &base_matrix_width) + || !coveringDimension( + (origin_y - south) / base_tile_height, + &base_matrix_height)) + { + return unsupported(tr( + "ArcGIS fullExtent cannot produce finite tile matrix dimensions." + )); + } + } + auto const factor = qint64(1) << highest_zoom; + if (base_matrix_width + > std::numeric_limits::max() / factor + || base_matrix_height + > std::numeric_limits::max() / factor) + { + return unsupported(tr( + "ArcGIS tile matrix dimensions overflow the runtime model." + )); + } + + TileMatrixSet matrix_set; + matrix_set.id = canonical_web_mercator && columns == 256 + ? QStringLiteral("WebMercatorQuad") + : canonical_web_mercator && columns == 512 + ? QStringLiteral("WebMercatorQuad512") + : QStringLiteral("ArcGISCacheEPSG%1").arg(*epsg); + matrix_set.crs = + QStringLiteral("EPSG:%1").arg(*epsg); + matrix_set.matrices.reserve(lods.size()); + QVector limits; + limits.reserve(lods.size()); + for (int zoom = 0; zoom < lods.size(); ++zoom) + { + auto const zoom_factor = qint64(1) << zoom; + auto const resolution = + runtime_base_resolution / double(zoom_factor); + auto const matrix_width = + base_matrix_width * zoom_factor; + auto const matrix_height = + base_matrix_height * zoom_factor; + matrix_set.matrices.push_back({ + QString::number(zoom), + zoom, + resolution, + QPointF(origin_x, origin_y), + QSize(int(columns), int(rows)), + matrix_width, + matrix_height, + }); + + auto const tile_width = resolution * double(columns); + auto const tile_height = resolution * double(rows); + qint64 min_column = 0; + qint64 max_column = -1; + qint64 min_row = 0; + qint64 max_row = -1; + if (!extentIndex( + (west - origin_x) / tile_width, + false, 0, matrix_width - 1, &min_column) + || !extentIndex( + (east - origin_x) / tile_width, + true, 0, matrix_width - 1, &max_column) + || !extentIndex( + (origin_y - north) / tile_height, + false, 0, matrix_height - 1, &min_row) + || !extentIndex( + (origin_y - south) / tile_height, + true, 0, matrix_height - 1, &max_row) + || min_column > max_column || min_row > max_row) + { + return unsupported(tr( + "ArcGIS fullExtent cannot be represented as tile limits." + )); + } + if (zoom >= minimum_zoom && zoom <= maximum_zoom + && (min_column != 0 || min_row != 0 + || max_column != matrix_width - 1 + || max_row != matrix_height - 1)) + { + limits.push_back({ + zoom, min_column, max_column, min_row, max_row + }); + } + } + + QString matrix_error; + if (!matrix_set.validateDyadicTopLeft(&matrix_error)) + { + return unsupported(tr( + "ArcGIS tile matrices are outside the runtime profile." + )); + } + + if (!tile_info.value(QStringLiteral("format")).isString() + || tile_info.value(QStringLiteral("format")) + .toString().trimmed().isEmpty()) + { + return invalid(tr( + "ArcGIS cache format is missing or malformed." + )); + } + auto const discovered_media_type = mediaType(tile_info); + if (discovered_media_type.isEmpty()) + { + return unsupported(tr( + "ArcGIS cache image format is not supported." + )); + } + + ArcGisTileServiceResult result; + result.outcome = ArcGisTileServiceOutcome::Resolved; + result.service_title = serviceTitle(root, *normalized_url); + result.likely_secret_parameters = + ManualImagerySource::likelySecretQueryParameters(*normalized_url); + + ResolvedImagerySource source; + source.metadata.id = settings.id.isEmpty() + ? generatedId(*normalized_url) + : settings.id; + source.metadata.name = settings.name.trimmed().isEmpty() + ? result.service_title + : settings.name.trimmed(); + source.notices.attribution_text = + settings.attribution_text.isEmpty() + ? root.value(QStringLiteral("copyrightText")).toString() + : settings.attribution_text; + source.notices.attribution_url = settings.attribution_url; + source.tile_urls = { { + tileTemplate(*normalized_url) + } }; + source.row_scheme = TileRowScheme::Xyz; + source.media_type = discovered_media_type; + source.tile_matrix_set = std::move(matrix_set); + source.min_zoom = minimum_zoom; + source.max_zoom = maximum_zoom; + source.tile_limits = std::move(limits); + source.request.referer = settings.referer; + source.request.empty_http_status_codes = + settings.empty_http_status_codes; + + QString source_error; + if (!source.validate(&source_error)) + { + return invalid(tr( + "ArcGIS source settings cannot satisfy the runtime requirements." + )); + } + result.source = std::move(source); + return result; +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/arcgis_tile_service.h b/src/imagery/arcgis_tile_service.h new file mode 100644 index 000000000..5f5414444 --- /dev/null +++ b/src/imagery/arcgis_tile_service.h @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_ARCGIS_TILE_SERVICE_H +#define OPENORIENTEERING_IMAGERY_ARCGIS_TILE_SERVICE_H + +#include + +#include +#include +#include +#include +#include +#include + +#include "imagery/imagery_source.h" + +namespace OpenOrienteering::imagery { + +enum class ArcGisTileServiceOutcome +{ + Resolved, + Unsupported, + Invalid, +}; + +struct ArcGisTileServiceSettings +{ + QString id; + QString name; + QUrl referer; + QVector empty_http_status_codes { 204, 404 }; + QString attribution_text; + QUrl attribution_url; + + bool operator==(const ArcGisTileServiceSettings&) const = default; +}; + +struct ArcGisTileServiceResult +{ + ArcGisTileServiceOutcome outcome = ArcGisTileServiceOutcome::Invalid; + QString detail; + QString service_title; + QStringList likely_secret_parameters; + std::optional source; + + bool resolved() const noexcept; +}; + +/** + * Pure parser for ArcGIS REST service metadata (`f=pjson`). + * + * This class performs no network access. The caller supplies the exact + * service URL and bounded response bytes obtained through its network policy. + */ +class ArcGisTileService +{ + Q_DECLARE_TR_FUNCTIONS( + OpenOrienteering::imagery::ArcGisTileService) + +public: + static constexpr qsizetype maximum_metadata_size = 1024 * 1024; + static constexpr int maximum_lods = 31; + + static ArcGisTileServiceResult parse( + const QByteArray& pjson, + const QUrl& service_url, + const ArcGisTileServiceSettings& settings = {} + ); +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/imagery_catalog_repository.cpp b/src/imagery/imagery_catalog_repository.cpp new file mode 100644 index 000000000..5c41fe48f --- /dev/null +++ b/src/imagery/imagery_catalog_repository.cpp @@ -0,0 +1,659 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "imagery/imagery_catalog_repository.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +struct ImageryCatalogRepository::Operation +{ + OperationId id = 0; + TileNetworkManager::Token network_token = 0; + ImageryCatalogFetchRequest fetch_request; + std::atomic_bool cancelled { false }; +}; + + +bool ImagerySourceHandle::isValid() const noexcept +{ + return !catalog_id.isEmpty() + && catalog_sha256.size() == 64 + && ((!source_id.isEmpty() && source_index < 0) + || (source_id.isEmpty() && source_index >= 0)); +} + + +const InstalledImageryCatalog* +ImageryCatalogRepositorySnapshot::catalog( + const QString& catalog_id, + const QByteArray& sha256) const noexcept +{ + for (auto const& installed : catalogs) + { + if (installed.read_result.catalog.id == catalog_id + && (sha256.isEmpty() + || installed.state.sha256 == sha256)) + return &installed; + } + return nullptr; +} + + +const OicSourceDefinition* +ImageryCatalogRepositorySnapshot::source( + const ImagerySourceHandle& handle, + const InstalledImageryCatalog** installed) const noexcept +{ + auto const* found_catalog = + catalog(handle.catalog_id, handle.catalog_sha256); + if (!found_catalog) + return nullptr; + if (handle.source_id.isEmpty()) + { + if (handle.source_index < 0 + || handle.source_index + >= found_catalog->read_result.catalog.sources.size()) + return nullptr; + if (installed) + *installed = found_catalog; + return &found_catalog->read_result.catalog.sources.at( + handle.source_index); + } + for (auto const& definition + : found_catalog->read_result.catalog.sources) + { + if (definition.metadata.id == handle.source_id) + { + if (installed) + *installed = found_catalog; + return &definition; + } + } + return nullptr; +} + + +std::optional +ImageryCatalogRepositorySnapshot::latestHandle( + const QString& catalog_id, + const QString& source_id) const +{ + if (source_id.isEmpty()) + return std::nullopt; + auto const* found_catalog = catalog(catalog_id); + if (!found_catalog) + return std::nullopt; + for (auto const& definition + : found_catalog->read_result.catalog.sources) + { + if (definition.metadata.id == source_id) + { + return ImagerySourceHandle { + catalog_id, + source_id, + found_catalog->state.sha256, + -1, + }; + } + } + return std::nullopt; +} + + +ImageryCatalogRepository::ImageryCatalogRepository( + QString store_root, + TileNetworkManager* network, + QObject* parent) + : QObject(parent) + , store_root_( + store_root.isEmpty() + ? ImageryCatalogStore {}.rootPath() + : std::move(store_root)) + , network_( + network ? network : &TileNetworkManager::instance()) + , snapshot_( + QSharedPointer::create()) +{ + Q_ASSERT(network_); + Q_ASSERT(thread() == network_->thread()); + worker_pool_.setMaxThreadCount(1); + worker_pool_.setExpiryTimeout(30'000); + qRegisterMetaType(); + connect( + network_, + &TileNetworkManager::finished, + this, + &ImageryCatalogRepository::onNetworkFinished); + reload(); +} + + +ImageryCatalogRepository::~ImageryCatalogRepository() +{ + for (auto const& operation : std::as_const(operations_)) + { + operation->cancelled.store(true); + if (operation->network_token) + network_->cancel(operation->network_token); + } + operations_.clear(); + network_operations_.clear(); + worker_pool_.clear(); + worker_pool_.waitForDone(); +} + + +Q_APPLICATION_STATIC( + ImageryCatalogRepository, + application_catalog_repository) + +ImageryCatalogRepository& +ImageryCatalogRepository::instance() +{ + auto* application = QCoreApplication::instance(); + Q_ASSERT(application); + Q_ASSERT(QThread::currentThread() == application->thread()); + return *application_catalog_repository; +} + + +QString ImageryCatalogRepository::storeRoot() const +{ + return store_root_; +} + + +TileNetworkManager& +ImageryCatalogRepository::networkManager() const noexcept +{ + return *network_; +} + + +ImageryCatalogRepositorySnapshotPtr +ImageryCatalogRepository::snapshot() const +{ + return snapshot_; +} + + +void ImageryCatalogRepository::reload() +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto const generation = ++reload_generation_; + auto const root = store_root_; + QPointer receiver(this); + worker_pool_.start([receiver, root, generation] { + auto next = + QSharedPointer::create(); + next->generation = generation; + ImageryCatalogStore store(root); + next->catalogs = store.catalogs(&next->issues); + QMetaObject::invokeMethod( + receiver, + [receiver, generation, next = std::move(next)] { + if (!receiver + || generation != receiver->reload_generation_) + return; + receiver->snapshot_ = next; + emit receiver->snapshotChanged(generation); + }, + Qt::QueuedConnection); + }); +} + + +ImageryCatalogRepository::OperationId +ImageryCatalogRepository::nextOperationId() +{ + auto const id = next_operation_id_++; + if (id == 0) + qFatal("Imagery catalog operation identity space exhausted"); + return id; +} + + +ImageryCatalogRepository::OperationId +ImageryCatalogRepository::readCatalogFile( + const QString& path) +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto operation = std::make_shared(); + operation->id = nextOperationId(); + operations_.insert(operation->id, operation); + + auto const absolute_path = QFileInfo(path).absoluteFilePath(); + auto const origin = + QUrl::fromLocalFile(absolute_path).toString(); + QPointer receiver(this); + auto const root = store_root_; + worker_pool_.start( + [receiver, operation, absolute_path, origin, root] { + ImageryCatalogOperationResult result; + if (operation->cancelled.load()) + return; + QFile file(absolute_path); + if (!file.open(QIODevice::ReadOnly)) + { + result.kind = ImageryCatalogOperationKind::Failed; + result.error = file.errorString(); + } + else if (file.size() < 0 + || file.size() + > OicCatalogReader::maximum_document_size) + { + result.kind = ImageryCatalogOperationKind::Failed; + result.error = ImageryCatalogRepository::tr( + "The catalog exceeds the %1 MiB safety limit.") + .arg( + OicCatalogReader::maximum_document_size + / (1024 * 1024)); + } + else + { + auto candidate = + QSharedPointer::create(); + candidate->metadata.origin = origin; + candidate->metadata.final_url = origin; + candidate->read_result = + OicCatalogReader::read(file.readAll()); + ImageryCatalogStore store(root); + candidate->analysis = + store.analyze(candidate->read_result); + result.kind = + ImageryCatalogOperationKind::CandidateReady; + result.candidate = std::move(candidate); + } + QMetaObject::invokeMethod( + receiver, + [receiver, id = operation->id, + result = std::move(result)]() mutable { + if (receiver) + receiver->complete(id, std::move(result)); + }, + Qt::QueuedConnection); + }); + return operation->id; +} + + +ImageryCatalogRepository::OperationId +ImageryCatalogRepository::fetchCatalog( + const ImageryCatalogFetchRequest& request) +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto operation = std::make_shared(); + operation->id = nextOperationId(); + operation->fetch_request = request; + operations_.insert(operation->id, operation); + + auto const scheme = request.url.scheme().toLower(); + if (scheme == QLatin1String("http") + && !request.allow_insecure_http) + { + ImageryCatalogOperationResult result; + result.kind = ImageryCatalogOperationKind::Failed; + result.error = tr( + "Downloading a catalog over plain HTTP requires explicit approval."); + QMetaObject::invokeMethod( + this, + [this, id = operation->id, + result = std::move(result)]() mutable { + complete(id, std::move(result)); + }, + Qt::QueuedConnection); + return operation->id; + } + + TileNetworkRequest network_request; + network_request.url = request.url; + network_request.client_id = + TileNetworkManager::nextClientId(); + network_request.generation = operation->id; + network_request.priority = TileRequestPriority::Visible; + network_request.payload_kind = + NetworkPayloadKind::JsonDocument; + network_request.empty_http_status_codes.clear(); + network_request.if_none_match = request.etag; + network_request.if_modified_since = + request.last_modified; + network_request.max_response_bytes = + OicCatalogReader::maximum_document_size; + operation->network_token = + network_->submit(std::move(network_request)); + network_operations_.insert( + operation->network_token, + operation->id); + return operation->id; +} + + +void ImageryCatalogRepository::parseCandidate( + const std::shared_ptr& operation, + QByteArray bytes, + ImageryCatalogInstallMetadata metadata) +{ + QPointer receiver(this); + auto const root = store_root_; + worker_pool_.start( + [receiver, operation, root, + bytes = std::move(bytes), + metadata = std::move(metadata)]() mutable { + if (operation->cancelled.load()) + return; + auto candidate = + QSharedPointer::create(); + candidate->metadata = std::move(metadata); + candidate->read_result = + OicCatalogReader::read(bytes); + auto const expected_catalog_id = + operation->fetch_request.installed_catalog_id; + if (!expected_catalog_id.isEmpty() + && candidate->read_result.catalog.id + != expected_catalog_id) + { + ImageryCatalogOperationResult result; + result.kind = + ImageryCatalogOperationKind::Failed; + result.error = ImageryCatalogRepository::tr( + "The downloaded catalog ID does not match the " + "installed catalog being updated."); + QMetaObject::invokeMethod( + receiver, + [receiver, id = operation->id, + result = std::move(result)]() mutable { + if (receiver) + receiver->complete( + id, + std::move(result)); + }, + Qt::QueuedConnection); + return; + } + ImageryCatalogStore store(root); + candidate->analysis = + store.analyze(candidate->read_result); + ImageryCatalogOperationResult result; + result.kind = + ImageryCatalogOperationKind::CandidateReady; + result.candidate = std::move(candidate); + QMetaObject::invokeMethod( + receiver, + [receiver, id = operation->id, + result = std::move(result)]() mutable { + if (receiver) + receiver->complete(id, std::move(result)); + }, + Qt::QueuedConnection); + }); +} + + +ImageryCatalogRepository::OperationId +ImageryCatalogRepository::installCandidate( + ImageryCatalogCandidatePtr candidate, + ImageryCatalogInstallOptions options) +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto operation = std::make_shared(); + operation->id = nextOperationId(); + operations_.insert(operation->id, operation); + + QPointer receiver(this); + auto const root = store_root_; + worker_pool_.start( + [receiver, operation, root, + candidate = std::move(candidate), + options]() mutable { + ImageryCatalogOperationResult result; + if (operation->cancelled.load()) + return; + if (!candidate) + { + result.kind = ImageryCatalogOperationKind::Failed; + result.error = ImageryCatalogRepository::tr( + "No imagery catalog candidate was provided."); + } + else + { + QString error; + ImageryCatalogStore store(root); + auto const success = store.install( + candidate->read_result, + candidate->metadata, + options, + &error); + result.kind = success + ? ImageryCatalogOperationKind::Installed + : ImageryCatalogOperationKind::Failed; + result.catalog_id = + candidate->read_result.catalog.id; + result.error = std::move(error); + } + QMetaObject::invokeMethod( + receiver, + [receiver, id = operation->id, + result = std::move(result)]() mutable { + if (receiver) + receiver->complete(id, std::move(result)); + }, + Qt::QueuedConnection); + }); + return operation->id; +} + + +ImageryCatalogRepository::OperationId +ImageryCatalogRepository::removeCatalog( + const QString& catalog_id) +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto operation = std::make_shared(); + operation->id = nextOperationId(); + operations_.insert(operation->id, operation); + + QPointer receiver(this); + auto const root = store_root_; + worker_pool_.start( + [receiver, operation, root, catalog_id] { + ImageryCatalogOperationResult result; + if (operation->cancelled.load()) + return; + QString error; + ImageryCatalogStore store(root); + auto const success = + store.remove(catalog_id, &error); + result.kind = success + ? ImageryCatalogOperationKind::Removed + : ImageryCatalogOperationKind::Failed; + result.catalog_id = catalog_id; + result.error = std::move(error); + QMetaObject::invokeMethod( + receiver, + [receiver, id = operation->id, + result = std::move(result)]() mutable { + if (receiver) + receiver->complete(id, std::move(result)); + }, + Qt::QueuedConnection); + }); + return operation->id; +} + + +void ImageryCatalogRepository::cancel( + OperationId operation_id) +{ + Q_ASSERT(QThread::currentThread() == thread()); + auto operation = operations_.take(operation_id); + if (!operation) + return; + operation->cancelled.store(true); + if (operation->network_token) + { + network_operations_.remove( + operation->network_token); + network_->cancel(operation->network_token); + } + ImageryCatalogOperationResult result; + result.kind = ImageryCatalogOperationKind::Cancelled; + emit operationFinished(operation_id, result); +} + + +void ImageryCatalogRepository::onNetworkFinished( + TileNetworkManager::Token token, + const TileNetworkResult& network_result) +{ + auto const operation_id = + network_operations_.take(token); + if (!operation_id) + return; + auto operation = operations_.value(operation_id); + if (!operation || operation->cancelled.load()) + return; + operation->network_token = 0; + + ImageryCatalogInstallMetadata metadata; + metadata.origin = + operation->fetch_request.url.toString(); + metadata.final_url = + network_result.final_url.isEmpty() + ? metadata.origin + : network_result.final_url.toString(); + metadata.etag = network_result.etag; + metadata.last_modified = + network_result.last_modified; + + if (network_result.outcome + == TileNetworkResult::Outcome::Success) + { + parseCandidate( + operation, + network_result.body, + std::move(metadata)); + return; + } + if (network_result.outcome + == TileNetworkResult::Outcome::NotModified) + { + if (metadata.etag.isEmpty()) + metadata.etag = + operation->fetch_request.etag; + if (metadata.last_modified.isEmpty()) + metadata.last_modified = + operation->fetch_request.last_modified; + auto const catalog_id = + operation->fetch_request.installed_catalog_id; + if (catalog_id.isEmpty()) + { + ImageryCatalogOperationResult result; + result.kind = + ImageryCatalogOperationKind::NotModified; + result.metadata = std::move(metadata); + complete(operation_id, std::move(result)); + return; + } + + QPointer receiver(this); + auto const root = store_root_; + worker_pool_.start( + [receiver, operation, root, catalog_id, + metadata = std::move(metadata)]() mutable { + if (operation->cancelled.load()) + return; + QString error; + ImageryCatalogStore store(root); + auto const success = store.markChecked( + catalog_id, + metadata.final_url, + metadata.etag, + metadata.last_modified, + &error); + ImageryCatalogOperationResult result; + result.kind = success + ? ImageryCatalogOperationKind::NotModified + : ImageryCatalogOperationKind::Failed; + result.catalog_id = catalog_id; + result.metadata = std::move(metadata); + result.error = std::move(error); + QMetaObject::invokeMethod( + receiver, + [receiver, id = operation->id, + result = std::move(result)]() mutable { + if (receiver) + { + receiver->complete( + id, + std::move(result)); + } + }, + Qt::QueuedConnection); + }); + return; + } + + ImageryCatalogOperationResult result; + result.kind = network_result.outcome + == TileNetworkResult::Outcome::Cancelled + ? ImageryCatalogOperationKind::Cancelled + : ImageryCatalogOperationKind::Failed; + result.error = network_result.error_string; + result.metadata = std::move(metadata); + auto const approval_url = + network_result.private_network_rejected_url; + auto const approval_scheme = + approval_url.scheme().toLower(); + if (network_result.private_network_rejected + && approval_url.isValid() + && !approval_url.isRelative() + && !approval_url.host().isEmpty() + && approval_url.userInfo().isEmpty() + && !approval_url.hasFragment() + && (approval_scheme == QLatin1String("http") + || approval_scheme == QLatin1String("https")) + && !network_->isPrivateOriginApproved( + approval_url)) + { + result.private_network_approval_url = + approval_url; + } + complete(operation_id, std::move(result)); +} + + +void ImageryCatalogRepository::complete( + OperationId operation_id, + ImageryCatalogOperationResult result) +{ + auto operation = operations_.take(operation_id); + if (!operation || operation->cancelled.load()) + return; + if (result.kind == ImageryCatalogOperationKind::Installed + || result.kind == ImageryCatalogOperationKind::Removed + || result.kind == ImageryCatalogOperationKind::NotModified) + reload(); + emit operationFinished(operation_id, result); +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/imagery_catalog_repository.h b/src/imagery/imagery_catalog_repository.h new file mode 100644 index 000000000..12d99b9ee --- /dev/null +++ b/src/imagery/imagery_catalog_repository.h @@ -0,0 +1,191 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_CATALOG_REPOSITORY_H +#define OPENORIENTEERING_IMAGERY_CATALOG_REPOSITORY_H + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/imagery_catalog_store.h" +#include "imagery/tile_network_manager.h" + +namespace OpenOrienteering::imagery { + +struct ImagerySourceHandle +{ + QString catalog_id; + QString source_id; + QByteArray catalog_sha256; + int source_index = -1; + + bool isValid() const noexcept; + bool operator==(const ImagerySourceHandle&) const = default; +}; + +inline size_t qHash( + const ImagerySourceHandle& handle, + size_t seed = 0) noexcept +{ + seed = qHash(handle.catalog_id, seed); + seed = qHash(handle.source_id, seed); + seed = qHash(handle.catalog_sha256, seed); + return ::qHash(handle.source_index, seed); +} + +struct ImageryCatalogRepositorySnapshot +{ + quint64 generation = 0; + QVector catalogs; + QVector issues; + + const InstalledImageryCatalog* catalog( + const QString& catalog_id, + const QByteArray& sha256 = {}) const noexcept; + const OicSourceDefinition* source( + const ImagerySourceHandle& handle, + const InstalledImageryCatalog** installed = nullptr) const noexcept; + std::optional latestHandle( + const QString& catalog_id, + const QString& source_id) const; +}; + +using ImageryCatalogRepositorySnapshotPtr = + QSharedPointer; + +struct ImageryCatalogCandidate +{ + OicCatalogReadResult read_result; + ImageryCatalogAnalysis analysis; + ImageryCatalogInstallMetadata metadata; +}; + +using ImageryCatalogCandidatePtr = + QSharedPointer; + +struct ImageryCatalogFetchRequest +{ + QUrl url; + QByteArray etag; + QByteArray last_modified; + QString installed_catalog_id; + bool allow_insecure_http = false; +}; + +enum class ImageryCatalogOperationKind +{ + CandidateReady, + NotModified, + Installed, + Removed, + Cancelled, + Failed, +}; + +struct ImageryCatalogOperationResult +{ + ImageryCatalogOperationKind kind = + ImageryCatalogOperationKind::Failed; + ImageryCatalogCandidatePtr candidate; + ImageryCatalogInstallMetadata metadata; + QString catalog_id; + QString error; + QUrl private_network_approval_url; +}; + +/** + * Application-facing asynchronous catalog index and operation coordinator. + * + * Store scans, JSON parsing, update analysis, and filesystem mutations run on + * one bounded worker. Network transfers share TileNetworkManager's connection + * pool and security policy. The published snapshot is immutable and selections + * use catalog/source/document identities. Invalid rows without a trustworthy + * source ID use their index only within an immutable catalog snapshot. + */ +class ImageryCatalogRepository final : public QObject +{ +Q_OBJECT + +public: + using OperationId = quint64; + + explicit ImageryCatalogRepository( + QString store_root = {}, + TileNetworkManager* network = nullptr, + QObject* parent = nullptr); + ~ImageryCatalogRepository() override; + + ImageryCatalogRepository( + const ImageryCatalogRepository&) = delete; + ImageryCatalogRepository& operator=( + const ImageryCatalogRepository&) = delete; + + static ImageryCatalogRepository& instance(); + + QString storeRoot() const; + TileNetworkManager& networkManager() const noexcept; + ImageryCatalogRepositorySnapshotPtr snapshot() const; + + void reload(); + OperationId readCatalogFile(const QString& path); + OperationId fetchCatalog( + const ImageryCatalogFetchRequest& request); + OperationId installCandidate( + ImageryCatalogCandidatePtr candidate, + ImageryCatalogInstallOptions options = {}); + OperationId removeCatalog(const QString& catalog_id); + void cancel(OperationId operation_id); + +signals: + void snapshotChanged(quint64 generation); + void operationFinished( + OpenOrienteering::imagery::ImageryCatalogRepository::OperationId id, + const OpenOrienteering::imagery::ImageryCatalogOperationResult& result); + +private: + struct Operation; + + OperationId nextOperationId(); + void parseCandidate( + const std::shared_ptr& operation, + QByteArray bytes, + ImageryCatalogInstallMetadata metadata); + void complete( + OperationId operation_id, + ImageryCatalogOperationResult result); + void onNetworkFinished( + TileNetworkManager::Token token, + const TileNetworkResult& result); + + QString store_root_; + TileNetworkManager* network_ = nullptr; + quint64 next_operation_id_ = 1; + quint64 reload_generation_ = 0; + ImageryCatalogRepositorySnapshotPtr snapshot_; + QThreadPool worker_pool_; + QHash> operations_; + QHash network_operations_; +}; + +} // namespace OpenOrienteering::imagery + +Q_DECLARE_METATYPE( + OpenOrienteering::imagery::ImageryCatalogOperationResult) +Q_DECLARE_METATYPE(OpenOrienteering::imagery::ImagerySourceHandle) + +#endif diff --git a/src/imagery/imagery_catalog_store.cpp b/src/imagery/imagery_catalog_store.cpp new file mode 100644 index 000000000..8a1d4e4bf --- /dev/null +++ b/src/imagery/imagery_catalog_store.cpp @@ -0,0 +1,1257 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "imagery/imagery_catalog_store.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +constexpr auto current_format = + "org.openorienteering.imagery-catalog-installation"; +constexpr auto snapshot_format = + "org.openorienteering.imagery-catalog-snapshot"; +constexpr int store_version = 1; +constexpr qint64 maximum_state_size = 64 * 1024; +constexpr qsizetype maximum_validator_size = 4096; + +QString tr(const char* text) +{ + return QCoreApplication::translate( + "OpenOrienteering::imagery::ImageryCatalogStore", + text); +} + +QByteArray sha256(const QByteArray& bytes) +{ + return QCryptographicHash::hash(bytes, QCryptographicHash::Sha256).toHex(); +} + +bool isSha256(const QByteArray& value) +{ + if (value.size() != 64) + return false; + for (auto const character : value) + { + if (!((character >= '0' && character <= '9') + || (character >= 'a' && character <= 'f'))) + return false; + } + return true; +} + +QByteArray sanitizedValidator(QByteArray value) +{ + value = value.trimmed(); + if (value.size() > maximum_validator_size) + return {}; + for (auto const character : value) + { + auto const byte = static_cast(character); + if (byte < 0x20 || byte == 0x7f) + return {}; + } + return value; +} + +bool isRemoteOrigin(const QString& value) +{ + auto const scheme = QUrl(value).scheme().toLower(); + return scheme == QLatin1String("http") + || scheme == QLatin1String("https"); +} + +bool fail(QString* error, QString message) +{ + if (error) + *error = std::move(message); + return false; +} + +bool saveFile( + const QString& path, + const QByteArray& bytes, + QString* error) +{ + QSaveFile file(path); + if (!file.open(QIODevice::WriteOnly)) + return fail(error, file.errorString()); + if (file.write(bytes) != bytes.size()) + return fail(error, file.errorString()); + if (!file.commit()) + return fail(error, file.errorString()); + return true; +} + +std::optional readSmallFile( + const QString& path, + qint64 maximum_size, + QString* error) +{ + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) + { + fail(error, file.errorString()); + return std::nullopt; + } + if (file.size() < 0 || file.size() > maximum_size) + { + fail( + error, + tr("The catalog store metadata exceeds its safety limit.")); + return std::nullopt; + } + return file.readAll(); +} + +std::optional readObject( + const QString& path, + QString* error) +{ + auto const bytes = readSmallFile(path, maximum_state_size, error); + if (!bytes) + return std::nullopt; + QJsonParseError parse_error; + auto const document = QJsonDocument::fromJson(*bytes, &parse_error); + if (parse_error.error != QJsonParseError::NoError || !document.isObject()) + { + fail( + error, + tr("Invalid catalog store metadata: %1") + .arg(parse_error.errorString())); + return std::nullopt; + } + return document.object(); +} + +QDateTime dateTime( + const QJsonObject& object, + const QString& name) +{ + return QDateTime::fromString( + object.value(name).toString(), + Qt::ISODateWithMs); +} + +QByteArray decodedHeader( + const QJsonObject& object, + const QString& name) +{ + auto const encoded = object.value(name).toString().toLatin1(); + return sanitizedValidator( + QByteArray::fromBase64( + encoded, + QByteArray::Base64UrlEncoding)); +} + +struct SnapshotMetadata +{ + QDateTime stored_at; + QString origin; + QString final_url; + QDateTime updated_at; + QByteArray etag; + QByteArray last_modified; +}; + +QJsonObject currentObject( + const OicCatalogReadResult& catalog, + const ImageryCatalogState& state) +{ + return { + { QStringLiteral("format"), QString::fromLatin1(current_format) }, + { QStringLiteral("version"), store_version }, + { QStringLiteral("catalogId"), catalog.catalog.id }, + { QStringLiteral("catalogRevision"), catalog.catalog.revision }, + { QStringLiteral("sha256"), QString::fromLatin1(state.sha256) }, + { QStringLiteral("previousSha256"), + QString::fromLatin1(state.previous_sha256) }, + { QStringLiteral("origin"), state.origin }, + { QStringLiteral("finalUrl"), state.final_url }, + { QStringLiteral("installedAt"), + state.installed_at.toUTC().toString(Qt::ISODateWithMs) }, + { QStringLiteral("updatedAt"), + state.updated_at.toUTC().toString(Qt::ISODateWithMs) }, + { QStringLiteral("checkedAt"), + state.checked_at.toUTC().toString(Qt::ISODateWithMs) }, + { QStringLiteral("etagBase64"), + QString::fromLatin1( + state.etag.toBase64(QByteArray::Base64UrlEncoding + | QByteArray::OmitTrailingEquals)) }, + { QStringLiteral("lastModifiedBase64"), + QString::fromLatin1( + state.last_modified.toBase64( + QByteArray::Base64UrlEncoding + | QByteArray::OmitTrailingEquals)) }, + }; +} + +QJsonObject snapshotObject( + const OicCatalogReadResult& catalog, + const QDateTime& stored_at, + const ImageryCatalogState& state) +{ + return { + { QStringLiteral("format"), QString::fromLatin1(snapshot_format) }, + { QStringLiteral("version"), store_version }, + { QStringLiteral("catalogId"), catalog.catalog.id }, + { QStringLiteral("catalogRevision"), catalog.catalog.revision }, + { QStringLiteral("sha256"), + QString::fromLatin1(catalog.catalog.document_sha256) }, + { QStringLiteral("storedAt"), + stored_at.toUTC().toString(Qt::ISODateWithMs) }, + { QStringLiteral("origin"), state.origin }, + { QStringLiteral("finalUrl"), state.final_url }, + { QStringLiteral("updatedAt"), + state.updated_at.toUTC().toString(Qt::ISODateWithMs) }, + { QStringLiteral("etagBase64"), + QString::fromLatin1( + state.etag.toBase64( + QByteArray::Base64UrlEncoding + | QByteArray::OmitTrailingEquals)) }, + { QStringLiteral("lastModifiedBase64"), + QString::fromLatin1( + state.last_modified.toBase64( + QByteArray::Base64UrlEncoding + | QByteArray::OmitTrailingEquals)) }, + }; +} + +std::optional readSnapshotMetadata( + const QString& snapshot_directory, + const QByteArray& expected_sha) +{ + auto const object = readObject( + QDir(snapshot_directory).filePath( + QStringLiteral("snapshot.json")), + nullptr); + if (!object + || object->value(QStringLiteral("format")).toString() + != QLatin1String(snapshot_format) + || object->value(QStringLiteral("version")).toInt(-1) + != store_version + || object->value(QStringLiteral("sha256")) + .toString() + .toLatin1() + != expected_sha) + { + return std::nullopt; + } + auto const stored_at = + dateTime(*object, QStringLiteral("storedAt")); + if (!stored_at.isValid()) + return std::nullopt; + + SnapshotMetadata metadata; + metadata.stored_at = stored_at; + metadata.origin = + object->value(QStringLiteral("origin")).toString(); + metadata.final_url = + object->value(QStringLiteral("finalUrl")).toString(); + if (metadata.final_url.isEmpty()) + metadata.final_url = metadata.origin; + metadata.updated_at = + dateTime(*object, QStringLiteral("updatedAt")); + if (!metadata.updated_at.isValid()) + metadata.updated_at = stored_at; + metadata.etag = + decodedHeader(*object, QStringLiteral("etagBase64")); + metadata.last_modified = + decodedHeader( + *object, + QStringLiteral("lastModifiedBase64")); + return metadata; +} + +QString catalogFilename() +{ + return QStringLiteral("catalog.") + OicCatalogReader::fileExtension(); +} + +bool safeDirectory(const QString& path, QString* error) +{ + QFileInfo info(path); + if (info.exists() && (info.isSymLink() || !info.isDir())) + { + return fail( + error, + tr( + "The catalog store contains an unsafe directory entry: %1") + .arg(path)); + } + return true; +} + +bool parseCurrentState( + const QJsonObject& object, + ImageryCatalogState* state, + QString* catalog_id, + int* revision, + QString* error) +{ + if (object.value(QStringLiteral("format")).toString() + != QLatin1String(current_format) + || object.value(QStringLiteral("version")).toInt(-1) + != store_version) + { + return fail( + error, + tr("Unsupported catalog store metadata format.")); + } + + *catalog_id = object.value(QStringLiteral("catalogId")).toString(); + *revision = object.value(QStringLiteral("catalogRevision")).toInt(); + state->sha256 = + object.value(QStringLiteral("sha256")).toString().toLatin1(); + state->previous_sha256 = + object.value(QStringLiteral("previousSha256")).toString().toLatin1(); + state->origin = object.value(QStringLiteral("origin")).toString(); + state->final_url = + object.value(QStringLiteral("finalUrl")).toString(); + if (state->final_url.isEmpty()) + state->final_url = state->origin; + state->installed_at = + dateTime(object, QStringLiteral("installedAt")); + state->updated_at = + dateTime(object, QStringLiteral("updatedAt")); + state->checked_at = + dateTime(object, QStringLiteral("checkedAt")); + state->etag = decodedHeader(object, QStringLiteral("etagBase64")); + state->last_modified = + decodedHeader(object, QStringLiteral("lastModifiedBase64")); + + if (catalog_id->isEmpty() || *revision <= 0 + || !isSha256(state->sha256) + || (!state->previous_sha256.isEmpty() + && !isSha256(state->previous_sha256)) + || !state->installed_at.isValid() + || !state->updated_at.isValid() + || !state->checked_at.isValid()) + { + return fail( + error, + tr("Catalog store metadata is incomplete or invalid.")); + } + return true; +} + +bool parseLegacyState( + const QJsonObject& object, + ImageryCatalogState* state) +{ + state->origin = object.value(QStringLiteral("origin")).toString(); + state->final_url = state->origin; + state->installed_at = + dateTime(object, QStringLiteral("installedAt")); + state->updated_at = state->installed_at; + state->checked_at = state->installed_at; + state->sha256 = + object.value(QStringLiteral("sha256")).toString().toLatin1(); + state->etag = + sanitizedValidator( + object.value(QStringLiteral("etag")).toString().toLatin1()); + state->last_modified = + sanitizedValidator( + object.value(QStringLiteral("lastModified")).toString().toLatin1()); + state->legacy_layout = true; + return state->installed_at.isValid() && isSha256(state->sha256); +} + +bool validateSnapshot( + const QString& snapshot_directory, + const QByteArray& expected_sha, + QString* error) +{ + if (!safeDirectory(snapshot_directory, error)) + return false; + auto const path = + QDir(snapshot_directory).filePath(catalogFilename()); + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) + return fail(error, file.errorString()); + if (file.size() < 0 + || file.size() > OicCatalogReader::maximum_document_size) + { + return fail( + error, + tr("Stored catalog exceeds the document safety limit.")); + } + auto const bytes = file.readAll(); + if (sha256(bytes) != expected_sha) + { + return fail( + error, + tr( + "Stored catalog bytes do not match their snapshot identity.")); + } + return true; +} + +bool createSnapshot( + const QString& catalog_directory, + const OicCatalogReadResult& catalog, + const QDateTime& stored_at, + const ImageryCatalogState& state, + QString* error) +{ + auto const snapshots_directory = + QDir(catalog_directory).filePath(QStringLiteral("snapshots")); + if (!safeDirectory(snapshots_directory, error) + || !QDir().mkpath(snapshots_directory)) + { + return fail( + error, + error && !error->isEmpty() + ? *error + : tr( + "Could not create the catalog snapshot directory.")); + } + + auto const sha = catalog.catalog.document_sha256; + auto const final_path = + QDir(snapshots_directory).filePath(QString::fromLatin1(sha)); + if (QFileInfo::exists(final_path)) + { + if (!validateSnapshot(final_path, sha, error)) + return false; + + // Catalog bytes are immutable, but transport provenance may advance + // when identical bytes are fetched from a new origin or revalidated. + // Refresh that bounded recovery metadata atomically while retaining the + // snapshot's original creation time. + auto snapshot_stored_at = stored_at; + if (auto const metadata = + readSnapshotMetadata(final_path, sha)) + snapshot_stored_at = metadata->stored_at; + return saveFile( + QDir(final_path).filePath( + QStringLiteral("snapshot.json")), + QJsonDocument( + snapshotObject( + catalog, + snapshot_stored_at, + state)) + .toJson(QJsonDocument::Compact), + error); + } + + auto const staging_path = QDir(snapshots_directory).filePath( + QStringLiteral(".staging-") + + QUuid::createUuid().toString(QUuid::Id128)); + if (!QDir().mkpath(staging_path) + || !saveFile( + QDir(staging_path).filePath(catalogFilename()), + catalog.catalog.original_bytes, + error) + || !saveFile( + QDir(staging_path).filePath(QStringLiteral("snapshot.json")), + QJsonDocument( + snapshotObject( + catalog, + stored_at, + state)) + .toJson(QJsonDocument::Compact), + error)) + { + QDir(staging_path).removeRecursively(); + if (error && error->isEmpty()) + *error = tr( + "Could not write the catalog snapshot."); + return false; + } + + if (!QDir().rename(staging_path, final_path)) + { + QDir(staging_path).removeRecursively(); + if (QFileInfo::exists(final_path)) + return validateSnapshot(final_path, sha, error); + return fail( + error, + tr("Could not activate the catalog snapshot.")); + } + return true; +} + +void pruneSnapshots( + const QString& catalog_directory, + const QSet& retained) +{ + QDir snapshots( + QDir(catalog_directory).filePath(QStringLiteral("snapshots"))); + if (!snapshots.exists()) + return; + for (auto const& info : snapshots.entryInfoList( + QDir::Dirs | QDir::NoDotAndDotDot | QDir::Hidden, + QDir::Name)) + { + if (info.isSymLink()) + continue; + auto const name = info.fileName().toLatin1(); + if ((name.startsWith(".staging-") + || (isSha256(name) && !retained.contains(name)))) + QDir(info.absoluteFilePath()).removeRecursively(); + } +} + +bool writeCurrent( + const QString& catalog_directory, + const OicCatalogReadResult& catalog, + const ImageryCatalogState& state, + QString* error) +{ + return saveFile( + QDir(catalog_directory).filePath(QStringLiteral("current.json")), + QJsonDocument(currentObject(catalog, state)) + .toJson(QJsonDocument::Compact), + error); +} + +bool lockStore( + const QString& root, + std::unique_ptr* lock, + QString* error) +{ + if (!QDir().mkpath(root)) + { + return fail( + error, + tr("Could not create the imagery catalog store.")); + } + QFileInfo root_info(root); + if (root_info.isSymLink() || !root_info.isDir()) + { + return fail( + error, + tr("The imagery catalog store path is not a safe directory.")); + } + auto acquired = std::make_unique( + QDir(root).filePath(QStringLiteral(".store.lock"))); + acquired->setStaleLockTime(30'000); + if (!acquired->tryLock(5'000)) + { + return fail( + error, + tr( + "Another process is updating the imagery catalog store.")); + } + *lock = std::move(acquired); + return true; +} + +} // namespace + + +ImageryCatalogStore::ImageryCatalogStore(QString root) +{ + this->root = root.isEmpty() + ? QDir( + QStandardPaths::writableLocation( + QStandardPaths::AppDataLocation)) + .filePath(QStringLiteral("imagery-catalogs")) + : std::move(root); +} + + +QString ImageryCatalogStore::rootPath() const +{ + return root; +} + + +QString ImageryCatalogStore::directoryKey( + const QString& catalog_id) const +{ + return QString::fromLatin1( + sha256(catalog_id.toUtf8()).left(32)); +} + + +InstalledImageryCatalog ImageryCatalogStore::loadDirectory( + const QString& directory, + QString* error) const +{ + InstalledImageryCatalog installed; + installed.directory = directory; + if (!safeDirectory(directory, error)) + return installed; + + auto const current_path = + QDir(directory).filePath(QStringLiteral("current.json")); + if (QFileInfo::exists(current_path)) + { + auto const object = readObject(current_path, error); + if (!object) + return installed; + QString catalog_id; + int revision = 0; + if (!parseCurrentState( + *object, + &installed.state, + &catalog_id, + &revision, + error)) + return installed; + + auto const expected_key = directoryKey(catalog_id); + if (QFileInfo(directory).fileName() != expected_key) + { + fail( + error, + tr("Catalog store directory identity does not match its catalog.")); + return installed; + } + auto const snapshot_directory = + QDir(directory).filePath( + QStringLiteral("snapshots/") + + QString::fromLatin1(installed.state.sha256)); + QString active_error; + if (validateSnapshot( + snapshot_directory, + installed.state.sha256, + &active_error)) + { + installed.catalog_path = + QDir(snapshot_directory).filePath(catalogFilename()); + } + else if (!installed.state.previous_sha256.isEmpty()) + { + auto const previous_directory = + QDir(directory).filePath( + QStringLiteral("snapshots/") + + QString::fromLatin1( + installed.state.previous_sha256)); + QString previous_error; + if (!validateSnapshot( + previous_directory, + installed.state.previous_sha256, + &previous_error)) + { + fail( + error, + tr("The active and previous catalog snapshots " + "are both unavailable: %1; %2") + .arg(active_error, previous_error)); + return installed; + } + installed.state.sha256 = + installed.state.previous_sha256; + installed.state.previous_sha256.clear(); + if (auto const metadata = + readSnapshotMetadata( + previous_directory, + installed.state.sha256); + metadata && !metadata->origin.isEmpty()) + { + installed.state.origin = + metadata->origin; + installed.state.final_url = + metadata->final_url.isEmpty() + ? metadata->origin + : metadata->final_url; + installed.state.updated_at = + metadata->updated_at; + installed.state.etag = + metadata->etag; + installed.state.last_modified = + metadata->last_modified; + } + else + { + // Older snapshots did not retain transport provenance. Never + // apply the damaged active snapshot's validators to different + // bytes; force an unconditional refresh from the best origin + // still available in current.json. + installed.state.final_url = + installed.state.origin; + installed.state.etag.clear(); + installed.state.last_modified.clear(); + if (metadata) + installed.state.updated_at = + metadata->updated_at; + } + installed.state.recovered_previous = true; + installed.catalog_path = + QDir(previous_directory).filePath( + catalogFilename()); + if (error) + { + *error = tr( + "The active catalog snapshot was damaged; " + "the previous snapshot is being used."); + } + } + else + { + fail(error, active_error); + return installed; + } + } + else + { + installed.catalog_path = + QDir(directory).filePath(catalogFilename()); + auto const state_path = + QDir(directory).filePath(QStringLiteral("state.json")); + if (!QFileInfo::exists(installed.catalog_path) + || !QFileInfo::exists(state_path)) + { + fail( + error, + tr("Catalog store entry has no active snapshot.")); + return installed; + } + auto const object = readObject(state_path, error); + if (!object + || !parseLegacyState(*object, &installed.state)) + { + fail( + error, + tr("Legacy catalog store metadata is invalid.")); + return installed; + } + } + + QFile catalog_file(installed.catalog_path); + if (!catalog_file.open(QIODevice::ReadOnly)) + { + fail(error, catalog_file.errorString()); + return installed; + } + if (catalog_file.size() < 0 + || catalog_file.size() + > OicCatalogReader::maximum_document_size) + { + fail( + error, + tr("Stored catalog exceeds the document safety limit.")); + return installed; + } + auto const bytes = catalog_file.readAll(); + if (sha256(bytes) != installed.state.sha256) + { + fail( + error, + tr("Stored catalog checksum does not match its metadata.")); + return installed; + } + installed.read_result = OicCatalogReader::read(bytes); + if (!installed.read_result.accepted()) + { + fail(error, tr("Installed catalog is invalid.")); + return installed; + } + if (installed.read_result.catalog.document_sha256 + != installed.state.sha256 + || QFileInfo(directory).fileName() + != directoryKey(installed.read_result.catalog.id)) + { + fail( + error, + tr("Installed catalog identity does not match its store entry.")); + installed.read_result = {}; + return installed; + } + return installed; +} + + +QVector ImageryCatalogStore::catalogs( + QVector* issues) const +{ + QVector installed; + QDir directory(root); + if (!directory.exists()) + return installed; + if (!QFileInfo(root).isDir() || QFileInfo(root).isSymLink()) + { + if (issues) + { + issues->push_back({ + root, + tr("The imagery catalog store path is not a safe directory."), + }); + } + return installed; + } + + for (auto const& info : directory.entryInfoList( + QDir::Dirs | QDir::NoDotAndDotDot, + QDir::Name)) + { + if (info.fileName().startsWith(QLatin1Char('.'))) + continue; + QString load_error; + auto catalog = + loadDirectory(info.absoluteFilePath(), &load_error); + if (!catalog.read_result.accepted()) + { + if (issues) + { + issues->push_back({ + info.absoluteFilePath(), + load_error.isEmpty() + ? tr("Could not load the installed imagery catalog.") + : load_error, + }); + } + continue; + } + if (issues && !load_error.isEmpty()) + { + issues->push_back({ + info.absoluteFilePath(), + load_error, + }); + } + installed.push_back(std::move(catalog)); + } + + std::stable_sort( + installed.begin(), + installed.end(), + [](auto const& left, auto const& right) { + auto const by_name = QString::localeAwareCompare( + left.read_result.catalog.name, + right.read_result.catalog.name); + if (by_name != 0) + return by_name < 0; + return left.read_result.catalog.id + < right.read_result.catalog.id; + }); + return installed; +} + + +ImageryCatalogAnalysis ImageryCatalogStore::analyze( + const OicCatalogReadResult& candidate, + QVector* issues) const +{ + ImageryCatalogAnalysis analysis; + QSet invalid_sources; + QSet unsupported_sources; + for (auto const& diagnostic : candidate.diagnostics) + { + if (diagnostic.source_index < 0) + continue; + if (diagnostic.kind == OicDiagnosticKind::SourceError) + invalid_sources.insert(diagnostic.source_index); + else if (diagnostic.kind + == OicDiagnosticKind::UnsupportedSource) + unsupported_sources.insert(diagnostic.source_index); + } + analysis.invalid = invalid_sources.size(); + analysis.unsupported = unsupported_sources.size(); + + auto const installed = catalogs(issues); + const InstalledImageryCatalog* previous = nullptr; + for (auto const& catalog : installed) + { + if (catalog.read_result.catalog.id == candidate.catalog.id) + { + previous = &catalog; + break; + } + } + + if (previous) + { + if (previous->read_result.catalog.revision + < candidate.catalog.revision) + analysis.update_kind = + ImageryCatalogAnalysis::UpdateKind::HigherRevision; + else if (previous->read_result.catalog.revision + > candidate.catalog.revision) + analysis.update_kind = + ImageryCatalogAnalysis::UpdateKind::LowerRevision; + else if (previous->state.sha256 + == candidate.catalog.document_sha256) + analysis.update_kind = + ImageryCatalogAnalysis::UpdateKind::ExactReimport; + else + analysis.update_kind = + ImageryCatalogAnalysis::UpdateKind::SameRevisionConflict; + + QMap old_sources; + QMap old_operational_sources; + QMap old_source_names; + for (auto const& source + : previous->read_result.catalog.sources) + { + if (!source.metadata.id.isEmpty()) + { + old_sources.insert( + source.metadata.id, + source.full_fingerprint); + old_operational_sources.insert( + source.metadata.id, + source.operational_fingerprint); + old_source_names.insert( + source.metadata.id, + source.metadata.name); + } + } + for (auto const& source : candidate.catalog.sources) + { + if (source.metadata.id.isEmpty()) + continue; + auto const found = + old_sources.find(source.metadata.id); + if (found == old_sources.end()) + { + ++analysis.added; + analysis.source_changes.push_back({ + ImageryCatalogSourceChangeKind::Added, + source.metadata.id, + source.metadata.name, + }); + } + else + { + if (found.value() != source.full_fingerprint) + { + ++analysis.changed; + auto const operational = + old_operational_sources.value( + source.metadata.id) + != source.operational_fingerprint; + if (operational) + ++analysis.operational_changed; + else + ++analysis.metadata_only_changed; + analysis.source_changes.push_back({ + operational + ? ImageryCatalogSourceChangeKind::Operational + : ImageryCatalogSourceChangeKind::MetadataOnly, + source.metadata.id, + source.metadata.name, + }); + } + old_sources.erase(found); + old_operational_sources.remove(source.metadata.id); + old_source_names.remove(source.metadata.id); + } + } + analysis.removed = old_sources.size(); + for (auto it = old_sources.cbegin(); + it != old_sources.cend(); + ++it) + { + analysis.source_changes.push_back({ + ImageryCatalogSourceChangeKind::Removed, + it.key(), + old_source_names.value(it.key()), + }); + } + } + else + { + for (auto const& source : candidate.catalog.sources) + { + if (!source.metadata.id.isEmpty()) + { + ++analysis.added; + analysis.source_changes.push_back({ + ImageryCatalogSourceChangeKind::Added, + source.metadata.id, + source.metadata.name, + }); + } + } + } + + QSet existing_full; + QSet existing_operational; + for (auto const& catalog : installed) + { + if (catalog.read_result.catalog.id == candidate.catalog.id) + continue; + for (auto const& existing + : catalog.read_result.catalog.sources) + { + if (!existing.full_fingerprint.isEmpty()) + existing_full.insert(existing.full_fingerprint); + if (!existing.operational_fingerprint.isEmpty()) + { + existing_operational.insert( + existing.operational_fingerprint); + } + } + } + for (auto const& source : candidate.catalog.sources) + { + if (source.full_fingerprint.isEmpty() + || source.operational_fingerprint.isEmpty()) + continue; + if (existing_full.contains(source.full_fingerprint)) + ++analysis.exact_duplicates; + else if (existing_operational.contains( + source.operational_fingerprint)) + ++analysis.potential_duplicates; + } + return analysis; +} + + +bool ImageryCatalogStore::install( + const OicCatalogReadResult& catalog, + const QString& origin, + const QByteArray& etag, + const QByteArray& last_modified, + QString* error) const +{ + return install( + catalog, + ImageryCatalogInstallMetadata { + origin, + origin, + etag, + last_modified, + }, + ImageryCatalogInstallOptions {}, + error); +} + + +bool ImageryCatalogStore::install( + const OicCatalogReadResult& catalog, + const ImageryCatalogInstallMetadata& metadata, + const ImageryCatalogInstallOptions& options, + QString* error) const +{ + if (!catalog.accepted()) + return fail(error, tr("Catalog is not installable.")); + if (!isSha256(catalog.catalog.document_sha256) + || sha256(catalog.catalog.original_bytes) + != catalog.catalog.document_sha256) + { + return fail( + error, + tr("Catalog document identity is invalid.")); + } + + std::unique_ptr lock; + if (!lockStore(root, &lock, error)) + return false; + + auto const catalog_directory = QDir(root).filePath( + directoryKey(catalog.catalog.id)); + if (!safeDirectory(catalog_directory, error) + || !QDir().mkpath(catalog_directory)) + { + return fail( + error, + error && !error->isEmpty() + ? *error + : tr("Could not create the catalog installation directory.")); + } + + QString previous_error; + auto previous = + loadDirectory(catalog_directory, &previous_error); + auto const had_previous = previous.read_result.accepted(); + auto const now = QDateTime::currentDateTimeUtc(); + auto const exact_reimport = + had_previous + && previous.state.sha256 + == catalog.catalog.document_sha256; + if (had_previous + && previous.read_result.catalog.revision + > catalog.catalog.revision + && !options.allow_lower_revision) + { + return fail( + error, + tr("Installing an older catalog revision requires explicit approval.")); + } + if (had_previous + && previous.read_result.catalog.revision + == catalog.catalog.revision + && previous.state.sha256 + != catalog.catalog.document_sha256 + && !options.allow_same_revision_conflict) + { + return fail( + error, + tr("This catalog revision was republished with different contents.")); + } + + ImageryCatalogState state; + auto const incoming_final_url = metadata.final_url.isEmpty() + ? metadata.origin + : metadata.final_url; + auto const preserve_remote_provenance = + exact_reimport + && isRemoteOrigin( + previous.state.final_url.isEmpty() + ? previous.state.origin + : previous.state.final_url) + && !isRemoteOrigin(incoming_final_url); + state.origin = preserve_remote_provenance + ? previous.state.origin + : metadata.origin; + state.final_url = preserve_remote_provenance + ? previous.state.final_url + : incoming_final_url; + state.installed_at = + had_previous ? previous.state.installed_at : now; + state.updated_at = + exact_reimport ? previous.state.updated_at : now; + state.checked_at = now; + state.sha256 = catalog.catalog.document_sha256; + state.etag = sanitizedValidator( + preserve_remote_provenance + ? previous.state.etag + : metadata.etag); + state.last_modified = sanitizedValidator( + preserve_remote_provenance + ? previous.state.last_modified + : metadata.last_modified); + if (had_previous + && previous.state.sha256 != state.sha256) + state.previous_sha256 = previous.state.sha256; + else if (had_previous) + state.previous_sha256 = previous.state.previous_sha256; + + if (had_previous && previous.state.legacy_layout + && !createSnapshot( + catalog_directory, + previous.read_result, + previous.state.installed_at, + previous.state, + error)) + return false; + if (!createSnapshot( + catalog_directory, + catalog, + now, + state, + error)) + return false; + + if (!writeCurrent( + catalog_directory, + catalog, + state, + error)) + return false; + + QSet retained { state.sha256 }; + if (!state.previous_sha256.isEmpty()) + retained.insert(state.previous_sha256); + pruneSnapshots(catalog_directory, retained); + return true; +} + + +bool ImageryCatalogStore::markChecked( + const QString& catalog_id, + const QString& final_url, + const QByteArray& etag, + const QByteArray& last_modified, + QString* error) const +{ + std::unique_ptr lock; + if (!lockStore(root, &lock, error)) + return false; + auto const catalog_directory = + QDir(root).filePath(directoryKey(catalog_id)); + QString load_error; + auto installed = + loadDirectory(catalog_directory, &load_error); + if (!installed.read_result.accepted()) + { + return fail( + error, + load_error.isEmpty() + ? tr("The imagery catalog is not installed.") + : load_error); + } + if (installed.read_result.catalog.id != catalog_id) + return fail(error, tr("Catalog store identity mismatch.")); + + if (installed.state.legacy_layout + && !createSnapshot( + catalog_directory, + installed.read_result, + installed.state.installed_at, + installed.state, + error)) + return false; + installed.state.legacy_layout = false; + installed.state.checked_at = QDateTime::currentDateTimeUtc(); + if (!final_url.isEmpty()) + installed.state.final_url = final_url; + installed.state.etag = sanitizedValidator(etag); + installed.state.last_modified = + sanitizedValidator(last_modified); + if (!createSnapshot( + catalog_directory, + installed.read_result, + installed.state.installed_at, + installed.state, + error)) + return false; + return writeCurrent( + catalog_directory, + installed.read_result, + installed.state, + error); +} + + +bool ImageryCatalogStore::remove( + const QString& catalog_id, + QString* error) const +{ + std::unique_ptr lock; + if (!lockStore(root, &lock, error)) + return false; + auto const path = + QDir(root).filePath(directoryKey(catalog_id)); + QFileInfo info(path); + if (!info.exists()) + return true; + if (info.isSymLink() || !info.isDir()) + { + return fail( + error, + tr("Refusing to remove an unsafe catalog store entry.")); + } + if (!QDir(path).removeRecursively()) + { + return fail( + error, + tr("Could not remove the imagery catalog.")); + } + return true; +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/imagery_catalog_store.h b/src/imagery/imagery_catalog_store.h new file mode 100644 index 000000000..30a3ee386 --- /dev/null +++ b/src/imagery/imagery_catalog_store.h @@ -0,0 +1,174 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_CATALOG_STORE_H +#define OPENORIENTEERING_IMAGERY_CATALOG_STORE_H + +#include +#include +#include +#include + +#include "imagery/oic_catalog.h" + +namespace OpenOrienteering::imagery { + +struct ImageryCatalogState +{ + QString origin; + QString final_url; + QDateTime installed_at; + QDateTime updated_at; + QDateTime checked_at; + QByteArray sha256; + QByteArray previous_sha256; + QByteArray etag; + QByteArray last_modified; + bool legacy_layout = false; + bool recovered_previous = false; + + bool operator==(const ImageryCatalogState&) const = default; +}; + +struct ImageryCatalogInstallMetadata +{ + QString origin; + QString final_url; + QByteArray etag; + QByteArray last_modified; + + bool operator==(const ImageryCatalogInstallMetadata&) const = default; +}; + +struct ImageryCatalogInstallOptions +{ + bool allow_lower_revision = false; + bool allow_same_revision_conflict = false; + + bool operator==(const ImageryCatalogInstallOptions&) const = default; +}; + +enum class ImageryCatalogSourceChangeKind +{ + Added, + Removed, + Operational, + MetadataOnly, +}; + +struct ImageryCatalogSourceChange +{ + ImageryCatalogSourceChangeKind kind = + ImageryCatalogSourceChangeKind::Added; + QString source_id; + QString name; + + bool operator==(const ImageryCatalogSourceChange&) const = default; +}; + +struct InstalledImageryCatalog +{ + OicCatalogReadResult read_result; + ImageryCatalogState state; + QString directory; + QString catalog_path; +}; + +struct ImageryCatalogStoreIssue +{ + QString path; + QString message; + + bool operator==(const ImageryCatalogStoreIssue&) const = default; +}; + +struct ImageryCatalogAnalysis +{ + enum class UpdateKind + { + NewCatalog, + ExactReimport, + HigherRevision, + LowerRevision, + SameRevisionConflict, + }; + + UpdateKind update_kind = UpdateKind::NewCatalog; + int added = 0; + int changed = 0; + int operational_changed = 0; + int metadata_only_changed = 0; + int removed = 0; + int invalid = 0; + int unsupported = 0; + int exact_duplicates = 0; + int potential_duplicates = 0; + QVector source_changes; + + bool operator==(const ImageryCatalogAnalysis&) const = default; +}; + +/** + * Crash-safe local store for imported OIC catalog snapshots. + * + * Catalog bytes are immutable and addressed by their document SHA-256. An + * atomically replaced current.json selects the active snapshot, so a process + * interruption cannot combine catalog bytes and metadata from two revisions. + * The immediately previous snapshot is retained for diagnostics and recovery. + * + * The reader also accepts the one-directory catalog.oic/state.json layout used + * by the earlier mapper-coc implementation. The next successful write migrates + * that installation to the snapshot layout without changing its catalog bytes. + */ +class ImageryCatalogStore +{ +public: + explicit ImageryCatalogStore(QString root = {}); + + QString rootPath() const; + QString directoryKey(const QString& catalog_id) const; + + QVector catalogs( + QVector* issues = nullptr) const; + ImageryCatalogAnalysis analyze( + const OicCatalogReadResult& candidate, + QVector* issues = nullptr) const; + + bool install( + const OicCatalogReadResult& catalog, + const QString& origin, + const QByteArray& etag = {}, + const QByteArray& last_modified = {}, + QString* error = nullptr) const; + bool install( + const OicCatalogReadResult& catalog, + const ImageryCatalogInstallMetadata& metadata, + const ImageryCatalogInstallOptions& options, + QString* error = nullptr) const; + bool markChecked( + const QString& catalog_id, + const QString& final_url, + const QByteArray& etag, + const QByteArray& last_modified, + QString* error = nullptr) const; + bool remove(const QString& catalog_id, QString* error = nullptr) const; + +private: + InstalledImageryCatalog loadDirectory( + const QString& directory, + QString* error) const; + + QString root; +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/imagery_network_permissions.cpp b/src/imagery/imagery_network_permissions.cpp new file mode 100644 index 000000000..bfc67a05e --- /dev/null +++ b/src/imagery/imagery_network_permissions.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "imagery/imagery_network_permissions.h" + +#include + +#include +#include +#include +#include + +#include "imagery/tile_network_manager.h" + +namespace OpenOrienteering::imagery { + +namespace { + +constexpr auto settings_key = + "onlineImagery/approvedPrivateOrigins"; +// Blocked requests are untrusted input. Keep review candidates memory-only and +// bounded; only an explicit approval is persisted. +constexpr qsizetype max_pending_origins = 128; + +} // namespace + + +ImageryNetworkPermissions::ImageryNetworkPermissions( + TileNetworkManager& network, + QObject* parent) + : QObject(parent) + , network_(network) +{ + auto const stored_origins = QSettings {} + .value(QString::fromLatin1(settings_key)) + .toStringList(); + approved_origins_ = stored_origins; + approved_origins_.removeDuplicates(); + std::sort( + approved_origins_.begin(), + approved_origins_.end()); + connect( + &network_, + &TileNetworkManager::privateOriginApprovalChanged, + this, + [this](const QString& origin, bool approved) { + auto changed = false; + auto pending_changed = false; + if (approved) + { + if (!approved_origins_.contains(origin)) + { + approved_origins_.push_back(origin); + std::sort( + approved_origins_.begin(), + approved_origins_.end()); + changed = true; + } + pending_changed = + pending_origins_.removeAll(origin) > 0; + } + else + { + changed = + approved_origins_.removeAll(origin) > 0; + } + if (changed) + { + save(); + emit approvalsChanged(); + } + if (pending_changed) + emit pendingOriginsChanged(); + }); + connect( + &network_, + &TileNetworkManager::finished, + this, + [this]( + TileNetworkManager::Token, + const TileNetworkResult& result) { + if (!result.private_network_rejected + || result.private_network_permission_revoked + || result.private_network_rejected_url.isEmpty()) + return; + auto const origin = + TileNetworkManager::canonicalOrigin( + result.private_network_rejected_url); + auto const url = QUrl(origin); + auto const scheme = url.scheme().toLower(); + if (!url.isValid() || url.host().isEmpty() + || !url.userInfo().isEmpty() + || (scheme != QLatin1String("http") + && scheme != QLatin1String("https")) + || network_.isPrivateOriginApproved(url) + || pending_origins_.contains(origin)) + return; + if (pending_origins_.size() >= max_pending_origins) + pending_origins_.removeFirst(); + pending_origins_.push_back(origin); + emit pendingOriginsChanged(); + }); + QStringList valid; + for (auto const& origin : std::as_const(approved_origins_)) + { + auto const url = QUrl(origin); + if (network_.approvePrivateOrigin(url)) + valid.push_back( + TileNetworkManager::canonicalOrigin(url)); + } + valid.removeDuplicates(); + std::sort(valid.begin(), valid.end()); + approved_origins_ = std::move(valid); + if (approved_origins_ != stored_origins) + save(); +} + + +Q_APPLICATION_STATIC( + ImageryNetworkPermissions, + application_imagery_network_permissions, + TileNetworkManager::instance()) + +ImageryNetworkPermissions& +ImageryNetworkPermissions::instance() +{ + auto* application = QCoreApplication::instance(); + Q_ASSERT(application); + Q_ASSERT(QThread::currentThread() == application->thread()); + return *application_imagery_network_permissions; +} + + +QStringList ImageryNetworkPermissions::approvedOrigins() const +{ + return approved_origins_; +} + + +QStringList ImageryNetworkPermissions::pendingOrigins() const +{ + return pending_origins_; +} + + +bool ImageryNetworkPermissions::isApproved( + const QUrl& url) const +{ + return network_.isPrivateOriginApproved(url); +} + + +bool ImageryNetworkPermissions::approve( + const QUrl& url) +{ + if (!network_.approvePrivateOrigin(url)) + return false; + auto const origin = + TileNetworkManager::canonicalOrigin(url); + if (!approved_origins_.contains(origin)) + { + approved_origins_.push_back(origin); + std::sort( + approved_origins_.begin(), + approved_origins_.end()); + save(); + emit approvalsChanged(); + } + if (pending_origins_.removeAll(origin) > 0) + emit pendingOriginsChanged(); + return true; +} + + +bool ImageryNetworkPermissions::revoke( + const QUrl& url) +{ + auto const origin = + TileNetworkManager::canonicalOrigin(url); + auto const removed = + approved_origins_.removeAll(origin) > 0; + network_.revokePrivateOrigin(url); + if (removed) + { + save(); + emit approvalsChanged(); + } + return removed; +} + + +bool ImageryNetworkPermissions::dismissPending( + const QUrl& url) +{ + auto const origin = + TileNetworkManager::canonicalOrigin(url); + if (pending_origins_.removeAll(origin) == 0) + return false; + emit pendingOriginsChanged(); + return true; +} + + +void ImageryNetworkPermissions::save() +{ + QSettings {}.setValue( + QString::fromLatin1(settings_key), + approved_origins_); +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/imagery_network_permissions.h b/src/imagery/imagery_network_permissions.h new file mode 100644 index 000000000..49be588a4 --- /dev/null +++ b/src/imagery/imagery_network_permissions.h @@ -0,0 +1,62 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_NETWORK_PERMISSIONS_H +#define OPENORIENTEERING_IMAGERY_NETWORK_PERMISSIONS_H + +#include +#include +#include + +namespace OpenOrienteering::imagery { + +class TileNetworkManager; + +/** + * Installation-local approvals for private-network imagery origins. + * + * Catalogs and map documents cannot mutate this policy. Only an explicit UI + * action calls approve(), after which the canonical origin is persisted in + * local application settings and applied to the shared network manager. + */ +class ImageryNetworkPermissions final : public QObject +{ +Q_OBJECT + +public: + explicit ImageryNetworkPermissions( + TileNetworkManager& network, + QObject* parent = nullptr); + + static ImageryNetworkPermissions& instance(); + + QStringList approvedOrigins() const; + QStringList pendingOrigins() const; + bool isApproved(const QUrl& url) const; + bool approve(const QUrl& url); + bool revoke(const QUrl& url); + bool dismissPending(const QUrl& url); + +signals: + void approvalsChanged(); + void pendingOriginsChanged(); + +private: + void save(); + + TileNetworkManager& network_; + QStringList approved_origins_; + QStringList pending_origins_; +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/imagery_source.cpp b/src/imagery/imagery_source.cpp index 2141de050..810ff4d3b 100644 --- a/src/imagery/imagery_source.cpp +++ b/src/imagery/imagery_source.cpp @@ -291,6 +291,17 @@ bool ResolvedImagerySource::validate(QString* error) const { return fail(error, QStringLiteral("Imagery source zoom range is invalid")); } + for (auto const& matrix : tile_matrix_set.matrices) + { + if (matrix.zoom >= min_zoom && matrix.zoom <= max_zoom + && !runtimeSupportsTileSize(matrix.tile_size)) + { + return fail( + error, + QStringLiteral( + "Imagery tile dimensions exceed the runtime decode profile")); + } + } if (!validateTileMatrixLimits(tile_limits, tile_matrix_set, error)) return false; for (auto const& limit : tile_limits) diff --git a/src/imagery/imagery_source.h b/src/imagery/imagery_source.h index d296d33e7..6bafabb29 100644 --- a/src/imagery/imagery_source.h +++ b/src/imagery/imagery_source.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,25 @@ namespace OpenOrienteering::imagery { +#ifdef Q_OS_ANDROID +inline constexpr qint64 maximum_runtime_tile_pixels = + qint64(1) * 1024 * 1024; +inline constexpr int maximum_runtime_tile_dimension = 2048; +#else +inline constexpr qint64 maximum_runtime_tile_pixels = + qint64(4) * 1024 * 1024; +inline constexpr int maximum_runtime_tile_dimension = 4096; +#endif + +inline bool runtimeSupportsTileSize(const QSize& size) noexcept +{ + return size.width() > 0 && size.height() > 0 + && size.width() <= maximum_runtime_tile_dimension + && size.height() <= maximum_runtime_tile_dimension + && qint64(size.width()) * size.height() + <= maximum_runtime_tile_pixels; +} + enum class ImageryCategory { Aerial, diff --git a/src/imagery/imagery_source_snapshot.h b/src/imagery/imagery_source_snapshot.h index 367f99ae8..eab8e49b2 100644 --- a/src/imagery/imagery_source_snapshot.h +++ b/src/imagery/imagery_source_snapshot.h @@ -39,6 +39,8 @@ class ImagerySourceSnapshotCodec public: static constexpr int version = 1; static constexpr qsizetype maximum_size = 1024 * 1024; + static constexpr qsizetype maximum_base64_size = + ((maximum_size + 2) / 3) * 4; static QString formatIdentifier(); static std::optional encode( diff --git a/src/imagery/manual_imagery_source.cpp b/src/imagery/manual_imagery_source.cpp new file mode 100644 index 000000000..db26796ba --- /dev/null +++ b/src/imagery/manual_imagery_source.cpp @@ -0,0 +1,470 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + */ + +#include "imagery/manual_imagery_source.h" + +#include +#include + +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +constexpr auto arcgis_path_pattern = + "^(.*/rest/services/.*/(MapServer|ImageServer))" + "(?:/tile/[^/]+/[^/]+/[^/]+)?/?$"; + +bool containsWhitespaceOrControl(const QString& value) +{ + for (auto const character : value) + { + auto const code = character.unicode(); + if (code < 0x20 || code == 0x7f || character.isSpace()) + return true; + } + return false; +} + +bool authorityContainsUserInfo(const QString& value) +{ + auto const scheme_end = value.indexOf(QStringLiteral("://")); + if (scheme_end < 0) + return false; + auto authority_end = value.size(); + for (auto const separator : { + QLatin1Char('/'), QLatin1Char('?'), QLatin1Char('#') + }) + { + auto const position = + value.indexOf(separator, scheme_end + 3); + if (position >= 0) + authority_end = std::min(authority_end, position); + } + return value.mid( + scheme_end + 3, + authority_end - (scheme_end + 3) + ).contains(QLatin1Char('@')); +} + +bool validHttpUrl(const QUrl& url) +{ + auto const scheme = url.scheme().toLower(); + return url.isValid() && !url.isRelative() && !url.host().isEmpty() + && (scheme == QLatin1String("http") + || scheme == QLatin1String("https")) + && url.userName().isEmpty() && url.password().isEmpty() + && !url.hasFragment(); +} + +QString compactQueryKey(QString value) +{ + value = value.toLower(); + value.remove(QLatin1Char('-')); + value.remove(QLatin1Char('_')); + value.remove(QLatin1Char('.')); + return value; +} + +bool isLikelySecretKey(const QString& key) +{ + static const QSet names { + QStringLiteral("token"), + QStringLiteral("accesstoken"), + QStringLiteral("apikey"), + QStringLiteral("key"), + QStringLiteral("auth"), + QStringLiteral("authorization"), + QStringLiteral("signature"), + QStringLiteral("sig"), + QStringLiteral("secret"), + QStringLiteral("clientsecret"), + QStringLiteral("credential"), + QStringLiteral("credentials"), + QStringLiteral("session"), + QStringLiteral("sessionid"), + QStringLiteral("jwt"), + QStringLiteral("password"), + QStringLiteral("passwd"), + QStringLiteral("pass"), + QStringLiteral("subscriptionkey"), + QStringLiteral("xamzsignature"), + QStringLiteral("xamzcredential"), + QStringLiteral("xamzsecuritytoken"), + QStringLiteral("googsignature"), + QStringLiteral("googleaccessid"), + }; + return names.contains(compactQueryKey(key)); +} + +QString queryService(const QUrl& url) +{ + for (auto const& item : + QUrlQuery(url).queryItems(QUrl::FullyDecoded)) + { + if (item.first.compare( + QStringLiteral("service"), Qt::CaseInsensitive) == 0) + { + return item.second.trimmed().toLower(); + } + } + return {}; +} + +bool pathHasServiceSegment( + const QString& path, + const QString& service) +{ + auto const pattern = QStringLiteral("(?:^|/)%1(?:/|$)") + .arg(QRegularExpression::escape(service)); + return QRegularExpression( + pattern, QRegularExpression::CaseInsensitiveOption + ).match(path).hasMatch(); +} + +QString filteredArcGisQuery(const QUrl& url) +{ + QStringList retained; + for (auto const& item : + url.query(QUrl::FullyEncoded).split( + QLatin1Char('&'), Qt::SkipEmptyParts)) + { + auto const separator = item.indexOf(QLatin1Char('=')); + auto const encoded_name = + separator < 0 ? item : item.left(separator); + auto const name = + QUrl::fromPercentEncoding(encoded_name.toUtf8()); + if (name.compare( + QStringLiteral("f"), Qt::CaseInsensitive) != 0 + && name.compare( + QStringLiteral("callback"), Qt::CaseInsensitive) != 0) + { + retained.push_back(item); + } + } + return retained.join(QLatin1Char('&')); +} + +QUrl urlProbe(const QString& normalized_template) +{ + auto probe = normalized_template; + probe.replace(QStringLiteral("{z}"), QStringLiteral("0")); + probe.replace(QStringLiteral("{x}"), QStringLiteral("0")); + probe.replace(QStringLiteral("{y}"), QStringLiteral("0")); + return QUrl(probe, QUrl::StrictMode); +} + +QString generatedId(const QString& normalized_template) +{ + auto const digest = QCryptographicHash::hash( + normalized_template.toUtf8(), QCryptographicHash::Sha256 + ).toHex(); + return QStringLiteral("manual-%1") + .arg(QString::fromLatin1(digest.first(16))); +} + +QString suggestedHostName(const QUrl& url) +{ + auto name = url.host().toLower(); + if (name.isEmpty()) + { + name = QCoreApplication::translate( + "OpenOrienteering::imagery::ManualImagerySource", + "Online imagery"); + } + return name; +} + +TileMatrixSet webMercatorMatrixSet(int tile_size, int maximum_zoom) +{ + constexpr auto half_world = 20037508.342789244; + auto const base_cell_size = + (2 * half_world) / double(tile_size); + + TileMatrixSet result; + result.id = tile_size == 256 + ? QStringLiteral("WebMercatorQuad") + : QStringLiteral("WebMercatorQuad512"); + result.crs = QStringLiteral("EPSG:3857"); + result.matrices.reserve(maximum_zoom + 1); + for (int zoom = 0; zoom <= maximum_zoom; ++zoom) + { + auto const dimension = qint64(1) << zoom; + result.matrices.push_back({ + QString::number(zoom), + zoom, + base_cell_size / double(dimension), + QPointF(-half_world, half_world), + QSize(tile_size, tile_size), + dimension, + dimension, + }); + } + return result; +} + +void addSecretWarning( + ManualImageryDiscoveryResult& result, + const QUrl& url) +{ + result.likely_secret_parameters = + ManualImagerySource::likelySecretQueryParameters(url); + if (!result.likely_secret_parameters.isEmpty()) + { + result.warnings.push_back( + ManualImageryWarning::LikelySecretQueryParameters + ); + } +} + +bool classifyArcGis( + const QUrl& probe, + ManualImageryDiscoveryResult& result) +{ + static const QRegularExpression pattern( + QString::fromLatin1(arcgis_path_pattern), + QRegularExpression::CaseInsensitiveOption + ); + auto const match = pattern.match(probe.path()); + if (!match.hasMatch()) + return false; + + result.outcome = ManualImageryOutcome::NeedsDiscovery; + result.input_kind = + match.captured(2).compare( + QStringLiteral("MapServer"), Qt::CaseInsensitive) == 0 + ? ManualImageryInputKind::ArcGisMapServer + : ManualImageryInputKind::ArcGisImageServer; + + result.service_url = probe; + result.service_url.setPath(match.captured(1)); + result.service_url.setFragment({}); + auto const service_query = filteredArcGisQuery(probe); + result.service_url.setQuery(service_query, QUrl::StrictMode); + auto discovery_text = + result.service_url.toString(QUrl::FullyEncoded); + discovery_text += service_query.isEmpty() + ? QStringLiteral("?f=pjson") + : QStringLiteral("&f=pjson"); + result.discovery_url = + QUrl(discovery_text, QUrl::StrictMode); + + auto service_path = match.captured(1); + service_path.chop(match.captured(2).size()); + while (service_path.endsWith(QLatin1Char('/'))) + service_path.chop(1); + result.suggested_name = + service_path.section(QLatin1Char('/'), -1); + if (result.suggested_name.isEmpty()) + result.suggested_name = suggestedHostName(probe); + result.detail = QCoreApplication::translate( + "OpenOrienteering::imagery::ManualImagerySource", + "ArcGIS service metadata is required before this source can be used."); + return true; +} + +} // namespace + +bool ManualImageryDiscoveryResult::isDirect() const noexcept +{ + return outcome == ManualImageryOutcome::Direct + && source.has_value(); +} + +ManualImageryDiscoveryResult ManualImagerySource::classify( + const QString& input, + const ManualTiledSourceSettings& settings) +{ + ManualImageryDiscoveryResult result; + if (input.isEmpty()) + { + result.detail = tr("Enter an imagery URL."); + return result; + } + if (input != input.trimmed() + || input.size() > 8192 + || containsWhitespaceOrControl(input) + || authorityContainsUserInfo(input)) + { + result.detail = tr( + "The URL is too long or contains whitespace, controls, or user information." + ); + return result; + } + + result.normalized_template = normalizeTemplateAliases(input); + auto const probe = urlProbe(result.normalized_template); + if (!validHttpUrl(probe)) + { + result.detail = tr( + "Imagery URLs must use HTTP or HTTPS with a host and no fragment." + ); + return result; + } + result.suggested_name = suggestedHostName(probe); + addSecretWarning(result, probe); + + auto const service = queryService(probe); + auto const path = probe.path(); + auto const has_xyz_placeholders = + result.normalized_template.contains(QStringLiteral("{z}")) + && result.normalized_template.contains(QStringLiteral("{x}")) + && result.normalized_template.contains(QStringLiteral("{y}")); + if (service == QLatin1String("wms") + || (service.isEmpty() && !has_xyz_placeholders + && pathHasServiceSegment(path, QStringLiteral("wms")))) + { + result.outcome = ManualImageryOutcome::Unsupported; + result.input_kind = ManualImageryInputKind::Wms; + result.detail = tr( + "WMS sources are recognized but are not supported by the tiled raster runtime." + ); + return result; + } + if (service == QLatin1String("wmts") + || (service.isEmpty() && !has_xyz_placeholders + && pathHasServiceSegment(path, QStringLiteral("wmts")))) + { + result.outcome = ManualImageryOutcome::Unsupported; + result.input_kind = ManualImageryInputKind::Wmts; + result.detail = tr( + "WMTS sources are recognized but are not supported by the tiled raster runtime." + ); + return result; + } + + if (classifyArcGis(probe, result)) + return result; + if (pathHasServiceSegment(path, QStringLiteral("MapServer")) + || pathHasServiceSegment(path, QStringLiteral("ImageServer"))) + { + result.input_kind = + pathHasServiceSegment(path, QStringLiteral("MapServer")) + ? ManualImageryInputKind::ArcGisMapServer + : ManualImageryInputKind::ArcGisImageServer; + result.detail = tr( + "ArcGIS URLs must identify the service root or an exact three-coordinate tile endpoint." + ); + return result; + } + + if (!has_xyz_placeholders + && QRegularExpression( + QStringLiteral("\\.tiff?$"), + QRegularExpression::CaseInsensitiveOption + ).match(path).hasMatch()) + { + result.outcome = ManualImageryOutcome::Unsupported; + result.input_kind = + ManualImageryInputKind::CloudOptimizedGeoTiff; + result.detail = tr( + "Cloud Optimized GeoTIFF sources are recognized but are not supported by the tiled raster runtime." + ); + return result; + } + + if (!has_xyz_placeholders) + { + result.detail = tr( + "A direct tiled source must contain {z}, {x}, and {y} placeholders." + ); + return result; + } + + result.input_kind = ManualImageryInputKind::TiledUrlTemplate; + TileUrlTemplate tile_url { result.normalized_template }; + QString error; + if (!tile_url.validate(&error)) + { + result.detail = tr( + "The URL template does not satisfy the tiled source requirements."); + return result; + } + if (settings.min_zoom < 0 + || settings.max_zoom < settings.min_zoom + || settings.max_zoom > maximum_zoom) + { + result.detail = tr( + "The zoom range must be ordered and fall between 0 and %1." + ).arg(maximum_zoom); + return result; + } + if (settings.tile_size != 256 && settings.tile_size != 512) + { + result.detail = tr( + "Direct tiled sources must use 256 or 512 pixel square tiles." + ); + return result; + } + + ResolvedImagerySource source; + source.metadata.id = settings.id.isEmpty() + ? generatedId(result.normalized_template) + : settings.id; + source.metadata.name = settings.name.trimmed().isEmpty() + ? result.suggested_name + : settings.name.trimmed(); + source.notices.attribution_text = settings.attribution_text; + source.notices.attribution_url = settings.attribution_url; + source.tile_urls = { std::move(tile_url) }; + source.row_scheme = settings.scheme; + source.media_type = settings.media_type; + source.tile_matrix_set = + webMercatorMatrixSet(settings.tile_size, settings.max_zoom); + source.min_zoom = settings.min_zoom; + source.max_zoom = settings.max_zoom; + source.request.referer = settings.referer; + source.request.empty_http_status_codes = + settings.empty_http_status_codes; + + if (!source.validate(&error)) + { + result.detail = tr( + "The direct tiled source settings do not satisfy the runtime requirements." + ); + return result; + } + result.outcome = ManualImageryOutcome::Direct; + result.source = std::move(source); + result.detail.clear(); + return result; +} + +QString ManualImagerySource::normalizeTemplateAliases(QString value) +{ + value.replace(QStringLiteral("${z}"), QStringLiteral("{z}")); + value.replace(QStringLiteral("${x}"), QStringLiteral("{x}")); + value.replace(QStringLiteral("${y}"), QStringLiteral("{y}")); + return value; +} + +QStringList ManualImagerySource::likelySecretQueryParameters( + const QUrl& url) +{ + QStringList result; + for (auto const& item : + QUrlQuery(url).queryItems(QUrl::FullyDecoded)) + { + if (isLikelySecretKey(item.first) + && !result.contains(item.first, Qt::CaseInsensitive)) + { + result.push_back(item.first); + } + } + std::sort( + result.begin(), result.end(), + [](const QString& first, const QString& second) { + return first.compare(second, Qt::CaseInsensitive) < 0; + } + ); + return result; +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/manual_imagery_source.h b/src/imagery/manual_imagery_source.h new file mode 100644 index 000000000..74372da7a --- /dev/null +++ b/src/imagery/manual_imagery_source.h @@ -0,0 +1,120 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_MANUAL_IMAGERY_SOURCE_H +#define OPENORIENTEERING_IMAGERY_MANUAL_IMAGERY_SOURCE_H + +#include + +#include +#include +#include +#include +#include + +#include "imagery/imagery_source.h" + +namespace OpenOrienteering::imagery { + +enum class ManualImageryOutcome +{ + Direct, + NeedsDiscovery, + Unsupported, + Invalid, +}; + +enum class ManualImageryInputKind +{ + TiledUrlTemplate, + ArcGisMapServer, + ArcGisImageServer, + Wms, + Wmts, + CloudOptimizedGeoTiff, + Unknown, +}; + +enum class ManualImageryWarning +{ + /** + * Query names look credential-bearing. The complete endpoint will be + * embedded in any map snapshot that uses the source. + */ + LikelySecretQueryParameters, +}; + +/** + * Explicit advanced settings for a direct Web Mercator XYZ/TMS source. + * + * The defaults are intentionally represented in the model so the UI can show + * them rather than applying hidden behavior. + */ +struct ManualTiledSourceSettings +{ + QString id; + QString name; + TileRowScheme scheme = TileRowScheme::Xyz; + int min_zoom = 0; + int max_zoom = 19; + int tile_size = 256; + QString media_type = QStringLiteral("image/png"); + QUrl referer; + QVector empty_http_status_codes { 204, 404 }; + QString attribution_text; + QUrl attribution_url; + + bool operator==(const ManualTiledSourceSettings&) const = default; +}; + +struct ManualImageryDiscoveryResult +{ + ManualImageryOutcome outcome = ManualImageryOutcome::Invalid; + ManualImageryInputKind input_kind = ManualImageryInputKind::Unknown; + QString detail; + QString normalized_template; + QString suggested_name; + QUrl service_url; + QUrl discovery_url; + QVector warnings; + QStringList likely_secret_parameters; + std::optional source; + + bool isDirect() const noexcept; + + /** + * Manual endpoints may contain credentials and are deliberately never + * eligible for recent-source persistence. This does not prevent the + * complete endpoint from being embedded in a saved map snapshot. + */ + static constexpr bool permitsRecentPersistence() noexcept { return false; } +}; + +class ManualImagerySource +{ + Q_DECLARE_TR_FUNCTIONS( + OpenOrienteering::imagery::ManualImagerySource) + +public: + static constexpr int maximum_zoom = 30; + + static ManualImageryDiscoveryResult classify( + const QString& input, + const ManualTiledSourceSettings& settings = {} + ); + + static QString normalizeTemplateAliases(QString value); + static QStringList likelySecretQueryParameters(const QUrl& url); +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/oic_catalog.cpp b/src/imagery/oic_catalog.cpp new file mode 100644 index 000000000..86328adb6 --- /dev/null +++ b/src/imagery/oic_catalog.cpp @@ -0,0 +1,4153 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#include "imagery/oic_catalog.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace OpenOrienteering::imagery { + +namespace { + +constexpr auto catalog_format = "org.openorienteering.imagery-catalog"; +constexpr auto web_mercator_quad_host = "www.opengis.net"; +constexpr auto web_mercator_quad_path = + "/def/tilematrixset/OGC/1.0/WebMercatorQuad"; +constexpr double maximum_exact_integer = 9007199254740991.0; + +bool isValidUtf8(const QByteArray& bytes) +{ + auto const* data = reinterpret_cast(bytes.constData()); + for (qsizetype index = 0; index < bytes.size(); ++index) + { + auto const first = data[index]; + if (first <= 0x7f) + continue; + + int continuation_count = 0; + uint codepoint = 0; + if (first >= 0xc2 && first <= 0xdf) + { + continuation_count = 1; + codepoint = first & 0x1f; + } + else if (first >= 0xe0 && first <= 0xef) + { + continuation_count = 2; + codepoint = first & 0x0f; + } + else if (first >= 0xf0 && first <= 0xf4) + { + continuation_count = 3; + codepoint = first & 0x07; + } + else + { + return false; + } + + if (index + continuation_count >= bytes.size()) + return false; + for (int offset = 0; offset < continuation_count; ++offset) + { + auto const next = data[++index]; + if ((next & 0xc0) != 0x80) + return false; + codepoint = (codepoint << 6) | (next & 0x3f); + } + if ((continuation_count == 1 && codepoint < 0x80) + || (continuation_count == 2 && codepoint < 0x800) + || (continuation_count == 3 && codepoint < 0x10000) + || codepoint > 0x10ffff + || (codepoint >= 0xd800 && codepoint <= 0xdfff)) + { + return false; + } + } + return true; +} + +class JsonPreflight +{ +public: + explicit JsonPreflight(const QByteArray& input) + : input(input) + {} + + bool validate() + { + if (!isValidUtf8(input)) + return fail(QStringLiteral("Catalog is not valid UTF-8")); + skipSpace(); + if (!parseValue(1)) + return false; + skipSpace(); + if (position != input.size()) + return fail(QStringLiteral("Unexpected data after the JSON document")); + return true; + } + + QString errorString() const { return error; } + +private: + bool parseValue(int depth) + { + if (depth > OicCatalogReader::maximum_nesting_depth) + { + return fail(QStringLiteral("JSON nesting exceeds %1 levels") + .arg(OicCatalogReader::maximum_nesting_depth)); + } + if (position >= input.size()) + return fail(QStringLiteral("Unexpected end of JSON input")); + + switch (input.at(position)) + { + case '{': return parseObject(depth); + case '[': return parseArray(depth); + case '"': return parseString(nullptr); + case 't': return parseLiteral("true"); + case 'f': return parseLiteral("false"); + case 'n': return parseLiteral("null"); + default: return parseNumber(); + } + } + + bool parseObject(int depth) + { + ++position; + skipSpace(); + QSet keys; + if (consume('}')) + return true; + while (position < input.size()) + { + QString key; + if (!parseString(&key)) + return false; + if (keys.contains(key)) + return fail(QStringLiteral("Duplicate JSON object member: %1").arg(key)); + keys.insert(key); + skipSpace(); + if (!consume(':')) + return fail(QStringLiteral("Expected ':' after an object member name")); + skipSpace(); + if (!parseValue(depth + 1)) + return false; + skipSpace(); + if (consume('}')) + return true; + if (!consume(',')) + return fail(QStringLiteral("Expected ',' or '}' in an object")); + skipSpace(); + } + return fail(QStringLiteral("Unterminated JSON object")); + } + + bool parseArray(int depth) + { + ++position; + skipSpace(); + if (consume(']')) + return true; + while (position < input.size()) + { + if (!parseValue(depth + 1)) + return false; + skipSpace(); + if (consume(']')) + return true; + if (!consume(',')) + return fail(QStringLiteral("Expected ',' or ']' in an array")); + skipSpace(); + } + return fail(QStringLiteral("Unterminated JSON array")); + } + + bool parseString(QString* output) + { + if (!consume('"')) + return fail(QStringLiteral("Expected a JSON string")); + QString decoded; + while (position < input.size()) + { + auto const byte = static_cast(input.at(position++)); + if (byte == '"') + { + if (decoded.size() > OicCatalogReader::maximum_string_length) + { + return fail(QStringLiteral("JSON string exceeds %1 characters") + .arg(OicCatalogReader::maximum_string_length)); + } + if (output) + *output = decoded; + return true; + } + if (byte < 0x20) + return fail(QStringLiteral("Unescaped control character in JSON string")); + if (byte == '\\') + { + if (position >= input.size()) + return fail(QStringLiteral("Unterminated JSON escape")); + auto const escaped = input.at(position++); + switch (escaped) + { + case '"': decoded.append(QLatin1Char('"')); break; + case '\\': decoded.append(QLatin1Char('\\')); break; + case '/': decoded.append(QLatin1Char('/')); break; + case 'b': decoded.append(QLatin1Char('\b')); break; + case 'f': decoded.append(QLatin1Char('\f')); break; + case 'n': decoded.append(QLatin1Char('\n')); break; + case 'r': decoded.append(QLatin1Char('\r')); break; + case 't': decoded.append(QLatin1Char('\t')); break; + case 'u': + { + uint codepoint = 0; + if (!parseHexQuad(&codepoint)) + return false; + if (codepoint >= 0xd800 && codepoint <= 0xdbff) + { + if (position + 2 > input.size() + || input.at(position) != '\\' + || input.at(position + 1) != 'u') + { + return fail(QStringLiteral("Unpaired high surrogate in JSON string")); + } + position += 2; + uint low = 0; + if (!parseHexQuad(&low)) + return false; + if (low < 0xdc00 || low > 0xdfff) + return fail(QStringLiteral("Invalid low surrogate in JSON string")); + codepoint = 0x10000 + + ((codepoint - 0xd800) << 10) + + (low - 0xdc00); + } + else if (codepoint >= 0xdc00 && codepoint <= 0xdfff) + { + return fail(QStringLiteral("Unpaired low surrogate in JSON string")); + } + auto const scalar = char32_t(codepoint); + decoded.append(QString::fromUcs4(&scalar, 1)); + break; + } + default: + return fail(QStringLiteral("Invalid JSON escape")); + } + } + else if (byte < 0x80) + { + decoded.append(QChar(ushort(byte))); + } + else + { + auto const start = position - 1; + auto const length = byte < 0xe0 ? 2 : (byte < 0xf0 ? 3 : 4); + position = start + length; + decoded.append(QString::fromUtf8(input.constData() + start, length)); + } + } + return fail(QStringLiteral("Unterminated JSON string")); + } + + bool parseHexQuad(uint* value) + { + if (position + 4 > input.size()) + return fail(QStringLiteral("Incomplete Unicode escape")); + uint result = 0; + for (int index = 0; index < 4; ++index) + { + auto const character = input.at(position++); + result <<= 4; + if (character >= '0' && character <= '9') + result += uint(character - '0'); + else if (character >= 'a' && character <= 'f') + result += uint(character - 'a' + 10); + else if (character >= 'A' && character <= 'F') + result += uint(character - 'A' + 10); + else + return fail(QStringLiteral("Invalid Unicode escape")); + } + *value = result; + return true; + } + + bool parseNumber() + { + auto const start = position; + auto const negative = consume('-'); + if (negative && position >= input.size()) + return fail(QStringLiteral("Incomplete JSON number")); + if (consume('0')) + { + if (position < input.size() + && input.at(position) >= '0' && input.at(position) <= '9') + { + return fail(QStringLiteral("Leading zero in JSON number")); + } + } + else + { + if (position >= input.size() + || input.at(position) < '1' || input.at(position) > '9') + { + return fail(QStringLiteral("Invalid JSON value")); + } + while (position < input.size() + && input.at(position) >= '0' && input.at(position) <= '9') + { + ++position; + } + } + if (consume('.')) + { + if (position >= input.size() + || input.at(position) < '0' || input.at(position) > '9') + { + return fail(QStringLiteral("Missing fraction digits in JSON number")); + } + while (position < input.size() + && input.at(position) >= '0' && input.at(position) <= '9') + { + ++position; + } + } + if (position < input.size() + && (input.at(position) == 'e' || input.at(position) == 'E')) + { + ++position; + if (position < input.size() + && (input.at(position) == '+' || input.at(position) == '-')) + { + ++position; + } + if (position >= input.size() + || input.at(position) < '0' || input.at(position) > '9') + { + return fail(QStringLiteral("Missing exponent digits in JSON number")); + } + while (position < input.size() + && input.at(position) >= '0' && input.at(position) <= '9') + { + ++position; + } + } + + bool ok = false; + auto const token = QString::fromLatin1(input.mid(start, position - start)); + auto const value = QLocale::c().toDouble(token, &ok); + if (!ok || !std::isfinite(value)) + return fail(QStringLiteral("JSON number is outside the finite IEEE 754 range")); + if (negative && value == 0) + return fail(QStringLiteral("Negative zero is not permitted in OIC JSON")); + if (std::floor(value) == value && std::abs(value) > maximum_exact_integer) + { + return fail(QStringLiteral("JSON integer exceeds the exact IEEE 754 range")); + } + return true; + } + + bool parseLiteral(const char* literal) + { + auto const length = qsizetype(qstrlen(literal)); + if (input.mid(position, length) != literal) + return fail(QStringLiteral("Invalid JSON literal")); + position += length; + return true; + } + + void skipSpace() + { + while (position < input.size()) + { + auto const character = input.at(position); + if (character != ' ' && character != '\t' + && character != '\r' && character != '\n') + { + break; + } + ++position; + } + } + + bool consume(char character) + { + if (position < input.size() && input.at(position) == character) + { + ++position; + return true; + } + return false; + } + + bool fail(const QString& message) + { + error = QStringLiteral("%1 at byte %2").arg(message).arg(position); + return false; + } + + const QByteArray& input; + qsizetype position = 0; + QString error; +}; + +bool utf16Less(const QString& first, const QString& second) +{ + auto const common = std::min(first.size(), second.size()); + for (qsizetype index = 0; index < common; ++index) + { + auto const left = first.at(index).unicode(); + auto const right = second.at(index).unicode(); + if (left != right) + return left < right; + } + return first.size() < second.size(); +} + +class CanonicalJsonEncoder +{ +public: + bool encode(const QJsonValue& value) + { + return encode(value, 0); + } + + QByteArray result() const { return output; } + QString errorString() const { return error; } + +private: + bool encode(const QJsonValue& value, int depth) + { + if (depth > 256) + return fail(QStringLiteral("Canonical JSON nesting limit exceeded")); + switch (value.type()) + { + case QJsonValue::Null: + output += "null"; + return true; + case QJsonValue::Bool: + output += value.toBool() ? "true" : "false"; + return true; + case QJsonValue::Double: + return encodeNumber(value.toDouble()); + case QJsonValue::String: + return encodeString(value.toString()); + case QJsonValue::Array: + { + output += '['; + auto const array = value.toArray(); + for (qsizetype index = 0; index < array.size(); ++index) + { + if (index) + output += ','; + if (!encode(array.at(index), depth + 1)) + return false; + } + output += ']'; + return true; + } + case QJsonValue::Object: + { + auto const object = value.toObject(); + auto keys = object.keys(); + std::sort(keys.begin(), keys.end(), utf16Less); + output += '{'; + for (qsizetype index = 0; index < keys.size(); ++index) + { + if (index) + output += ','; + if (!encodeString(keys.at(index))) + return false; + output += ':'; + if (!encode(object.value(keys.at(index)), depth + 1)) + return false; + } + output += '}'; + return true; + } + case QJsonValue::Undefined: + return fail(QStringLiteral("Undefined is not a JSON value")); + } + return fail(QStringLiteral("Unknown JSON value type")); + } + + bool encodeString(const QString& string) + { + output += '"'; + for (qsizetype index = 0; index < string.size(); ++index) + { + auto const code_unit = string.at(index).unicode(); + switch (code_unit) + { + case 0x08: output += "\\b"; continue; + case 0x09: output += "\\t"; continue; + case 0x0a: output += "\\n"; continue; + case 0x0c: output += "\\f"; continue; + case 0x0d: output += "\\r"; continue; + case '"': output += "\\\""; continue; + case '\\': output += "\\\\"; continue; + default: break; + } + if (code_unit < 0x20) + { + static const char hex[] = "0123456789abcdef"; + output += "\\u00"; + output += hex[(code_unit >> 4) & 0x0f]; + output += hex[code_unit & 0x0f]; + continue; + } + if (QChar::isHighSurrogate(code_unit)) + { + if (index + 1 >= string.size() + || !QChar::isLowSurrogate(string.at(index + 1).unicode())) + { + return fail(QStringLiteral("String contains an unpaired high surrogate")); + } + QString pair; + pair.append(string.at(index)); + pair.append(string.at(++index)); + output += pair.toUtf8(); + continue; + } + if (QChar::isLowSurrogate(code_unit)) + return fail(QStringLiteral("String contains an unpaired low surrogate")); + output += QString(string.at(index)).toUtf8(); + } + output += '"'; + return true; + } + + bool encodeNumber(double number) + { + if (!std::isfinite(number)) + return fail(QStringLiteral("Canonical JSON number is nonfinite")); + if (number == 0) + { + output += '0'; + return true; + } + + auto shortest = + QJsonDocument(QJsonArray { number }).toJson(QJsonDocument::Compact); + shortest = shortest.mid(1, shortest.size() - 2); + auto const negative = shortest.startsWith('-'); + if (negative) + shortest.remove(0, 1); + + int exponent = 0; + auto exponent_position = shortest.indexOf('e'); + if (exponent_position < 0) + exponent_position = shortest.indexOf('E'); + auto mantissa = shortest; + if (exponent_position >= 0) + { + bool ok = false; + exponent = shortest.mid(exponent_position + 1).toInt(&ok); + if (!ok) + return fail(QStringLiteral("Unable to canonicalize a number exponent")); + mantissa = shortest.left(exponent_position); + } + + auto decimal_position = mantissa.indexOf('.'); + if (decimal_position < 0) + decimal_position = mantissa.size(); + else + mantissa.remove(decimal_position, 1); + auto leading_zeroes = 0; + while (leading_zeroes < mantissa.size() + && mantissa.at(leading_zeroes) == '0') + { + ++leading_zeroes; + } + mantissa.remove(0, leading_zeroes); + decimal_position -= leading_zeroes; + while (mantissa.size() > 1 && mantissa.endsWith('0')) + mantissa.chop(1); + if (mantissa.isEmpty()) + return fail(QStringLiteral("Unable to canonicalize number digits")); + + auto const decimal_point = decimal_position + exponent; + if (negative) + output += '-'; + if (decimal_point > 0 && decimal_point <= 21) + { + if (decimal_point >= mantissa.size()) + { + output += mantissa; + output += QByteArray(decimal_point - mantissa.size(), '0'); + } + else + { + output += mantissa.left(decimal_point); + output += '.'; + output += mantissa.mid(decimal_point); + } + } + else if (decimal_point <= 0 && decimal_point > -6) + { + output += "0."; + output += QByteArray(-decimal_point, '0'); + output += mantissa; + } + else + { + output += mantissa.at(0); + if (mantissa.size() > 1) + { + output += '.'; + output += mantissa.mid(1); + } + output += 'e'; + auto const scientific_exponent = decimal_point - 1; + if (scientific_exponent >= 0) + output += '+'; + output += QByteArray::number(scientific_exponent); + } + return true; + } + + bool fail(const QString& message) + { + error = message; + return false; + } + + QByteArray output; + QString error; +}; + +QByteArray canonicalJson(const QJsonValue& value, QString* error = nullptr) +{ + CanonicalJsonEncoder encoder; + if (!encoder.encode(value)) + { + if (error) + *error = encoder.errorString(); + return {}; + } + if (error) + error->clear(); + return encoder.result(); +} + +QByteArray sha256(const QByteArray& value) +{ + return QCryptographicHash::hash(value, QCryptographicHash::Sha256).toHex(); +} + +QString normalizeUrl(QString value) +{ + value.replace(QStringLiteral("${z}"), QStringLiteral("{z}")); + value.replace(QStringLiteral("${x}"), QStringLiteral("{x}")); + value.replace(QStringLiteral("${y}"), QStringLiteral("{y}")); + + auto const scheme_end = value.indexOf(QStringLiteral("://")); + if (scheme_end < 0) + return value; + auto const authority_start = scheme_end + 3; + auto authority_end = value.size(); + for (auto const separator : { + QLatin1Char('/'), QLatin1Char('?'), QLatin1Char('#') + }) + { + auto const position = value.indexOf(separator, authority_start); + if (position >= 0) + authority_end = std::min(authority_end, position); + } + + auto probe = value; + probe.replace(QStringLiteral("{z}"), QStringLiteral("0")); + probe.replace(QStringLiteral("{x}"), QStringLiteral("0")); + probe.replace(QStringLiteral("{y}"), QStringLiteral("0")); + auto const parsed = QUrl(probe, QUrl::StrictMode); + auto host = QString::fromLatin1(QUrl::toAce(parsed.host())).toLower(); + if (host.contains(QLatin1Char(':'))) + host = QLatin1Char('[') + host + QLatin1Char(']'); + auto const scheme = value.left(scheme_end).toLower(); + auto const port = parsed.port(-1); + auto const default_port = + (scheme == QLatin1String("http") && port == 80) + || (scheme == QLatin1String("https") && port == 443); + QString authority = host; + if (port >= 0 && !default_port) + authority += QLatin1Char(':') + QString::number(port); + return scheme + QStringLiteral("://") + authority + value.mid(authority_end); +} + +QJsonArray sortedStrings(QStringList values) +{ + values.removeDuplicates(); + std::sort(values.begin(), values.end()); + return QJsonArray::fromStringList(values); +} + +QJsonArray sortedStatusCodes(QVector values) +{ + std::sort(values.begin(), values.end()); + values.erase(std::unique(values.begin(), values.end()), values.end()); + QJsonArray result; + for (auto const value : values) + result.push_back(value); + return result; +} + +QJsonObject normalizedRequest(const OicSourceDefinition& source) +{ + QJsonObject request; + if (!source.request.referer.isEmpty()) + { + request.insert( + QStringLiteral("referer"), + normalizeUrl(source.request.referer.toString(QUrl::FullyEncoded)) + ); + } + if (!source.request.empty_http_status_codes.isEmpty()) + { + request.insert( + QStringLiteral("emptyHttpStatusCodes"), + sortedStatusCodes(source.request.empty_http_status_codes) + ); + } + return request; +} + +QJsonObject normalizedMatrixSet(const OicSourceDefinition& source) +{ + QJsonArray matrices; + for (auto const& definition : source.tile_matrix_set.matrices) + { + auto const& matrix = definition.matrix; + QJsonObject object { + { QStringLiteral("id"), matrix.id }, + { QStringLiteral("scaleDenominator"), definition.scale_denominator }, + { QStringLiteral("cellSize"), matrix.cell_size }, + { QStringLiteral("pointOfOrigin"), QJsonArray { + matrix.point_of_origin.x(), matrix.point_of_origin.y() + } }, + { QStringLiteral("cornerOfOrigin"), definition.corner_of_origin }, + { QStringLiteral("tileWidth"), matrix.tile_size.width() }, + { QStringLiteral("tileHeight"), matrix.tile_size.height() }, + { QStringLiteral("matrixWidth"), double(matrix.matrix_width) }, + { QStringLiteral("matrixHeight"), double(matrix.matrix_height) }, + }; + if (definition.has_variable_matrix_widths) + { + object.insert( + QStringLiteral("variableMatrixWidths"), + definition.original_object.value(QStringLiteral("variableMatrixWidths")) + ); + } + matrices.push_back(object); + } + QJsonObject result { + { QStringLiteral("id"), source.tile_matrix_set.matrix_set.id }, + { QStringLiteral("crs"), source.tile_matrix_set.matrix_set.crs }, + { QStringLiteral("tileMatrices"), matrices }, + }; + if (!source.tile_matrix_set.ordered_axes.isEmpty()) + { + result.insert( + QStringLiteral("orderedAxes"), + QJsonArray::fromStringList(source.tile_matrix_set.ordered_axes) + ); + } + return result; +} + +QJsonObject normalizedFullMatrixSet(const OicSourceDefinition& source) +{ + auto result = source.tile_matrix_set.original_object; + result.insert( + QStringLiteral("crs"), + source.tile_matrix_set.matrix_set.crs + ); + return result; +} + +QJsonArray normalizedLimits(const OicSourceDefinition& source) +{ + auto limits = source.tile_limit_definitions; + auto matrix_index = [&source](const QString& id) { + auto const& matrices = source.tile_matrix_set.matrix_set.matrices; + for (qsizetype index = 0; index < matrices.size(); ++index) + { + if (matrices.at(index).id == id) + return int(index); + } + return std::numeric_limits::max(); + }; + std::sort( + limits.begin(), limits.end(), + [&matrix_index](auto const& first, auto const& second) { + auto const first_index = matrix_index(first.tile_matrix); + auto const second_index = matrix_index(second.tile_matrix); + if (first_index != second_index) + return first_index < second_index; + if (first.tile_matrix != second.tile_matrix) + return first.tile_matrix < second.tile_matrix; + if (first.min_row != second.min_row) + return first.min_row < second.min_row; + if (first.max_row != second.max_row) + return first.max_row < second.max_row; + if (first.min_column != second.min_column) + return first.min_column < second.min_column; + return first.max_column < second.max_column; + } + ); + QJsonArray result; + for (auto const& limit : limits) + { + result.push_back(QJsonObject { + { QStringLiteral("tileMatrix"), limit.tile_matrix }, + { QStringLiteral("minTileRow"), double(limit.min_row) }, + { QStringLiteral("maxTileRow"), double(limit.max_row) }, + { QStringLiteral("minTileCol"), double(limit.min_column) }, + { QStringLiteral("maxTileCol"), double(limit.max_column) }, + }); + } + return result; +} + +QStringList operationalCapabilities(const OicSourceDefinition& source) +{ + auto result = source.required_capabilities; + result.push_back(QStringLiteral("tile-matrix-set.ogc-2.0")); + if (source.tile_matrix_set.dyadic_top_left) + result.push_back(QStringLiteral("tile-matrix-set.dyadic.v1")); + else if (!source.tile_matrix_set_uri.isEmpty() + && source.tile_matrix_set.matrix_set.matrices.isEmpty()) + result.push_back(QStringLiteral("tile-matrix-set.external.v1")); + else + result.push_back(QStringLiteral("tile-matrix-set.nondyadic.v1")); + switch (source.registration.kind) + { + case OicRegistrationKind::None: + break; + case OicRegistrationKind::Translation2d: + result.push_back(QStringLiteral("registration.translation2d.v1")); + break; + case OicRegistrationKind::Affine2d: + result.push_back(QStringLiteral("registration.affine2d.v1")); + break; + case OicRegistrationKind::GridShift: + result.push_back(QStringLiteral("registration.grid-shift.v1")); + break; + } + return result; +} + +QJsonObject normalizedRegistration(const OicSourceDefinition& source) +{ + auto registration = source.registration.original_object; + if (registration.isEmpty()) + return {}; + auto source_frame = + registration.value(QStringLiteral("sourceFrame")).toObject(); + source_frame.insert(QStringLiteral("crs"), source.registration.source_crs); + registration.insert(QStringLiteral("sourceFrame"), source_frame); + auto target_frame = + registration.value(QStringLiteral("targetFrame")).toObject(); + target_frame.insert(QStringLiteral("crs"), source.registration.target_crs); + registration.insert(QStringLiteral("targetFrame"), target_frame); + return registration; +} + +QJsonObject normalizedOperationalSource(const OicSourceDefinition& source) +{ + QJsonArray tiles; + for (auto const& tile : source.tile_urls) + tiles.push_back(normalizeUrl(tile.value)); + QJsonObject result { + { QStringLiteral("fingerprintVersion"), 1 }, + { QStringLiteral("type"), source.type }, + { QStringLiteral("tiles"), tiles }, + { QStringLiteral("scheme"), tileRowSchemeName(source.row_scheme) }, + { QStringLiteral("format"), source.media_type }, + { QStringLiteral("minTileMatrix"), source.min_tile_matrix }, + { QStringLiteral("maxTileMatrix"), source.max_tile_matrix }, + { QStringLiteral("tileMatrixLimits"), normalizedLimits(source) }, + { QStringLiteral("request"), normalizedRequest(source) }, + { QStringLiteral("registration"), + source.registration.kind == OicRegistrationKind::None + ? QJsonValue(QJsonValue::Null) + : QJsonValue(normalizedRegistration(source)) }, + { QStringLiteral("requires"), + sortedStrings(operationalCapabilities(source)) }, + }; + if (!source.tile_matrix_set.matrix_set.matrices.isEmpty()) + result.insert(QStringLiteral("tileMatrixSet"), normalizedMatrixSet(source)); + else + { + result.insert( + QStringLiteral("tileMatrixSetURI"), + normalizeUrl(source.tile_matrix_set_uri) + ); + } + return result; +} + +QJsonObject normalizedFullSource(const OicSourceDefinition& source) +{ + auto result = source.original_object; + QJsonArray tiles; + for (auto const& tile : source.tile_urls) + tiles.push_back(normalizeUrl(tile.value)); + result.insert(QStringLiteral("tiles"), tiles); + result.insert(QStringLiteral("format"), source.media_type); + result.insert(QStringLiteral("minTileMatrix"), source.min_tile_matrix); + result.insert(QStringLiteral("maxTileMatrix"), source.max_tile_matrix); + if (!source.tile_matrix_set.matrix_set.matrices.isEmpty()) + { + result.remove(QStringLiteral("tileMatrixSetURI")); + result.insert( + QStringLiteral("tileMatrixSet"), + normalizedFullMatrixSet(source) + ); + } + else + { + result.insert( + QStringLiteral("tileMatrixSetURI"), + normalizeUrl(source.tile_matrix_set_uri) + ); + } + if (result.contains(QStringLiteral("requires"))) + { + result.insert( + QStringLiteral("requires"), + sortedStrings(source.required_capabilities) + ); + } + if (result.contains(QStringLiteral("request"))) + result.insert(QStringLiteral("request"), normalizedRequest(source)); + if (result.contains(QStringLiteral("tileMatrixLimits"))) + result.insert(QStringLiteral("tileMatrixLimits"), normalizedLimits(source)); + if (result.contains(QStringLiteral("registration"))) + result.insert(QStringLiteral("registration"), normalizedRegistration(source)); + if (result.value(QStringLiteral("notices")).isObject()) + { + auto notices = result.value(QStringLiteral("notices")).toObject(); + for (auto const& name : { + QStringLiteral("attributionUrl"), QStringLiteral("sourceUrl"), + QStringLiteral("termsUrl"), QStringLiteral("privacyUrl") + }) + { + if (notices.value(name).isString()) + notices.insert(name, normalizeUrl(notices.value(name).toString())); + } + result.insert(QStringLiteral("notices"), notices); + } + return result; +} + +bool calculateFingerprints( + OicSourceDefinition& source, + QString* error) +{ + auto const full_object = QJsonObject { + { QStringLiteral("fingerprintVersion"), 1 }, + { QStringLiteral("source"), normalizedFullSource(source) }, + }; + auto const full = canonicalJson(full_object, error); + if (full.isEmpty()) + return false; + auto const operational = + canonicalJson(normalizedOperationalSource(source), error); + if (operational.isEmpty()) + return false; + source.full_fingerprint = sha256(full); + source.operational_fingerprint = sha256(operational); + return true; +} + +bool containsControl(const QString& value) +{ + for (auto const character : value) + { + auto const code = character.unicode(); + if (code < 0x20 || code == 0x7f) + return true; + } + return false; +} + +bool containsUrlWhitespaceOrControl(const QString& value) +{ + for (auto const character : value) + { + auto const code = character.unicode(); + if (code < 0x20 || code == 0x7f || character.isSpace()) + return true; + } + return false; +} + +bool urlAuthorityContainsUserInfo(const QString& value) +{ + auto const scheme_end = value.indexOf(QStringLiteral("://")); + if (scheme_end < 0) + return false; + auto authority_end = value.size(); + for (auto const separator : { + QLatin1Char('/'), QLatin1Char('?'), QLatin1Char('#') + }) + { + auto const position = + value.indexOf(separator, scheme_end + 3); + if (position >= 0) + authority_end = std::min(authority_end, position); + } + return value.mid( + scheme_end + 3, + authority_end - (scheme_end + 3) + ).contains(QLatin1Char('@')); +} + +class CatalogValidator +{ +public: + explicit CatalogValidator(OicCatalogReadResult& result) + : result(result) + {} + + void validate(const QJsonObject& root, const QByteArray& bytes) + { + static const QSet fields { + QStringLiteral("$schema"), QStringLiteral("format"), + QStringLiteral("version"), QStringLiteral("id"), + QStringLiteral("revision"), QStringLiteral("name"), + QStringLiteral("description"), QStringLiteral("publisher"), + QStringLiteral("created"), QStringLiteral("updated"), + QStringLiteral("catalogLicense"), QStringLiteral("requires"), + QStringLiteral("resources"), QStringLiteral("extensions"), + QStringLiteral("sources"), + }; + checkUnknownFields(root, fields, QStringLiteral("$"), true); + + auto& catalog = result.catalog; + catalog.original_bytes = bytes; + catalog.original_object = root; + catalog.format = + requiredString(root, QStringLiteral("format"), + QStringLiteral("$.format"), true); + if (catalog.format != QLatin1String(catalog_format)) + { + catalogError( + QStringLiteral("unsupported-format"), + QStringLiteral("$.format"), + QStringLiteral("Unsupported catalog format") + ); + } + catalog.version = + int(requiredInteger(root, QStringLiteral("version"), + QStringLiteral("$.version"), true, + 1, std::numeric_limits::max())); + if (catalog.version != 1) + { + catalogError( + QStringLiteral("unsupported-version"), + QStringLiteral("$.version"), + QStringLiteral("Unsupported catalog version") + ); + } + catalog.id = + requiredId(root, QStringLiteral("id"), + QStringLiteral("$.id"), true); + catalog.revision = + int(requiredInteger(root, QStringLiteral("revision"), + QStringLiteral("$.revision"), true, + 1, std::numeric_limits::max())); + catalog.name = + requiredText(root, QStringLiteral("name"), + QStringLiteral("$.name"), true, + OicCatalogReader::maximum_string_length); + catalog.description = + optionalText(root, QStringLiteral("description"), + QStringLiteral("$.description"), true, + OicCatalogReader::maximum_string_length); + catalog.created = + optionalDate(root, QStringLiteral("created"), + QStringLiteral("$.created"), true); + catalog.updated = + optionalDate(root, QStringLiteral("updated"), + QStringLiteral("$.updated"), true); + if (catalog.created.isValid() && catalog.updated.isValid() + && catalog.created > catalog.updated) + { + catalogError( + QStringLiteral("date-order"), + QStringLiteral("$.updated"), + QStringLiteral("Catalog update date precedes its creation date") + ); + } + catalog.catalog_license = + optionalText(root, QStringLiteral("catalogLicense"), + QStringLiteral("$.catalogLicense"), true, + 4096); + if (root.contains(QStringLiteral("$schema"))) + { + absoluteUrl( + requiredString(root, QStringLiteral("$schema"), + QStringLiteral("$.$schema"), true), + QStringLiteral("$.$schema"), true + ); + } + if (root.contains(QStringLiteral("publisher"))) + catalog.publisher = validatePublisher( + root.value(QStringLiteral("publisher")), + QStringLiteral("$.publisher") + ); + if (root.contains(QStringLiteral("requires"))) + { + catalog.required_capabilities = + validateCapabilities( + root.value(QStringLiteral("requires")), + QStringLiteral("$.requires"), true + ); + for (auto const& capability : catalog.required_capabilities) + { + if (!runtimeCapabilities().contains(capability)) + { + catalogError( + QStringLiteral("unsupported-capability"), + QStringLiteral("$.requires"), + QStringLiteral( + "Required catalog capability is unsupported: %1" + ).arg(capability) + ); + } + } + } + if (root.contains(QStringLiteral("resources"))) + { + validateResources( + root.value(QStringLiteral("resources")), + QStringLiteral("$.resources") + ); + } + if (root.contains(QStringLiteral("extensions"))) + { + catalog.extensions = + objectValue( + root.value(QStringLiteral("extensions")), + QStringLiteral("$.extensions"), true + ); + validateExtensions( + catalog.extensions, QStringLiteral("$.extensions"), true + ); + } + + if (!root.value(QStringLiteral("sources")).isArray()) + { + catalogError( + QStringLiteral("missing-sources"), + QStringLiteral("$.sources"), + QStringLiteral("Required member must be an array") + ); + return; + } + auto const sources = root.value(QStringLiteral("sources")).toArray(); + if (sources.isEmpty()) + { + catalogError( + QStringLiteral("empty-sources"), + QStringLiteral("$.sources"), + QStringLiteral("Catalog must contain at least one source") + ); + } + if (sources.size() > OicCatalogReader::maximum_sources) + { + catalogError( + QStringLiteral("source-limit"), + QStringLiteral("$.sources"), + QStringLiteral("Catalog exceeds the %1 source limit") + .arg(OicCatalogReader::maximum_sources) + ); + } + + QSet source_ids; + auto const count = + std::min(sources.size(), qsizetype(OicCatalogReader::maximum_sources)); + for (qsizetype index = 0; index < count; ++index) + { + current_source = int(index); + auto const path = QStringLiteral("$.sources[%1]").arg(index); + if (!sources.at(index).isObject()) + { + sourceError( + QStringLiteral("source-type"), path, + QStringLiteral("Source must be an object") + ); + result.catalog.sources.push_back({}); + continue; + } + + auto source = validateSource(sources.at(index).toObject(), path); + if (!source.metadata.id.isEmpty()) + { + if (source_ids.contains(source.metadata.id)) + { + catalogError( + QStringLiteral("duplicate-source-id"), + path + QStringLiteral(".id"), + QStringLiteral("Duplicate source ID: %1") + .arg(source.metadata.id) + ); + } + source_ids.insert(source.metadata.id); + } + result.catalog.sources.push_back(std::move(source)); + } + current_source = -1; + } + +private: + OicSourceDefinition validateSource( + const QJsonObject& object, + const QString& path) + { + static const QSet fields { + QStringLiteral("id"), QStringLiteral("name"), + QStringLiteral("description"), QStringLiteral("type"), + QStringLiteral("tiles"), QStringLiteral("scheme"), + QStringLiteral("format"), QStringLiteral("minTileMatrix"), + QStringLiteral("maxTileMatrix"), + QStringLiteral("tileMatrixSetURI"), + QStringLiteral("tileMatrixSet"), + QStringLiteral("tileMatrixLimits"), + QStringLiteral("request"), QStringLiteral("requires"), + QStringLiteral("category"), QStringLiteral("startDate"), + QStringLiteral("endDate"), QStringLiteral("coverage"), + QStringLiteral("notices"), QStringLiteral("registration"), + QStringLiteral("extensions"), + }; + auto const diagnostic_start = result.diagnostics.size(); + checkUnknownFields(object, fields, path, false); + + OicSourceDefinition source; + source.original_object = object; + source.metadata.id = + requiredId(object, QStringLiteral("id"), + path + QStringLiteral(".id"), false); + if (!source.metadata.id.isEmpty() + && !validRuntimeId(source.metadata.id)) + { + addUnsupported( + source, QStringLiteral("source-id.runtime.v1"), + path + QStringLiteral(".id"), + QStringLiteral( + "Source ID exceeds the resolved runtime identifier profile" + ) + ); + } + if (!result.catalog.id.isEmpty() + && !validRuntimeId(result.catalog.id)) + { + addUnsupported( + source, QStringLiteral("catalog-id.runtime.v1"), + QStringLiteral("$.id"), + QStringLiteral( + "Catalog ID exceeds the resolved runtime identifier profile" + ) + ); + } + source.metadata.name = + requiredText(object, QStringLiteral("name"), + path + QStringLiteral(".name"), false, 512); + source.metadata.description = + optionalText(object, QStringLiteral("description"), + path + QStringLiteral(".description"), false, 4096); + source.type = + requiredString(object, QStringLiteral("type"), + path + QStringLiteral(".type"), false); + if (source.type != QLatin1String("raster-tiles")) + { + sourceError( + QStringLiteral("unsupported-source-type"), + path + QStringLiteral(".type"), + QStringLiteral("Source type must be raster-tiles") + ); + } + + validateTiles( + object.value(QStringLiteral("tiles")), + path + QStringLiteral(".tiles"), source + ); + auto const scheme = + requiredString(object, QStringLiteral("scheme"), + path + QStringLiteral(".scheme"), false); + auto const parsed_scheme = tileRowSchemeFromName(scheme); + if (!parsed_scheme) + { + sourceError( + QStringLiteral("invalid-scheme"), + path + QStringLiteral(".scheme"), + QStringLiteral("Scheme must be xyz or tms") + ); + } + else + { + source.row_scheme = *parsed_scheme; + } + + if (object.contains(QStringLiteral("format"))) + { + source.media_type = + requiredString(object, QStringLiteral("format"), + path + QStringLiteral(".format"), false); + } + static const QRegularExpression media_type_pattern( + QStringLiteral("^image/[A-Za-z0-9!#$&^_.+-]{1,96}$") + ); + if (!media_type_pattern.match(source.media_type).hasMatch()) + { + sourceError( + QStringLiteral("invalid-media-type"), + path + QStringLiteral(".format"), + QStringLiteral("Format must be a supported image media type") + ); + } + + source.min_tile_matrix = + optionalString(object, QStringLiteral("minTileMatrix"), + path + QStringLiteral(".minTileMatrix"), false); + source.max_tile_matrix = + optionalString(object, QStringLiteral("maxTileMatrix"), + path + QStringLiteral(".maxTileMatrix"), false); + if ((!source.min_tile_matrix.isEmpty() + && !validMatrixId(source.min_tile_matrix)) + || (!source.max_tile_matrix.isEmpty() + && !validMatrixId(source.max_tile_matrix))) + { + sourceError( + QStringLiteral("invalid-matrix-id"), path, + QStringLiteral("Tile matrix identifiers are invalid") + ); + } + + auto const has_uri = object.contains(QStringLiteral("tileMatrixSetURI")); + auto const has_inline = object.contains(QStringLiteral("tileMatrixSet")); + if (has_uri == has_inline) + { + sourceError( + QStringLiteral("matrix-set-choice"), path, + QStringLiteral( + "Source must contain exactly one of tileMatrixSetURI " + "or tileMatrixSet" + ) + ); + } + else if (has_uri) + { + validateMatrixSetUri( + object.value(QStringLiteral("tileMatrixSetURI")), + path + QStringLiteral(".tileMatrixSetURI"), source + ); + } + else + { + validateTileMatrixSet( + object.value(QStringLiteral("tileMatrixSet")), + path + QStringLiteral(".tileMatrixSet"), source + ); + } + + if (object.contains(QStringLiteral("tileMatrixLimits"))) + { + validateTileMatrixLimits( + object.value(QStringLiteral("tileMatrixLimits")), + path + QStringLiteral(".tileMatrixLimits"), source + ); + } + validateMatrixRange(source, path); + if (object.contains(QStringLiteral("request"))) + { + validateRequest( + object.value(QStringLiteral("request")), + path + QStringLiteral(".request"), source + ); + } + if (object.contains(QStringLiteral("requires"))) + { + source.required_capabilities = + validateCapabilities( + object.value(QStringLiteral("requires")), + path + QStringLiteral(".requires"), false + ); + for (auto const& capability : source.required_capabilities) + { + if (!runtimeCapabilities().contains(capability)) + { + addUnsupported( + source, capability, + path + QStringLiteral(".requires"), + QStringLiteral("Required source capability is unsupported") + ); + } + } + } + validatePresentation(object, path, source); + if (object.contains(QStringLiteral("registration"))) + { + validateRegistration( + object.value(QStringLiteral("registration")), + path + QStringLiteral(".registration"), source + ); + } + if (object.contains(QStringLiteral("extensions"))) + { + source.extensions = + objectValue( + object.value(QStringLiteral("extensions")), + path + QStringLiteral(".extensions"), false + ); + validateExtensions( + source.extensions, + path + QStringLiteral(".extensions"), false + ); + } + + source.valid = !hasSourceErrors(diagnostic_start); + source.supported = + source.valid && source.unsupported_capabilities.isEmpty(); + if (source.valid) + { + QString fingerprint_error; + if (!calculateFingerprints(source, &fingerprint_error)) + { + sourceError( + QStringLiteral("fingerprint"), + path, + QStringLiteral("Unable to fingerprint source: %1") + .arg(fingerprint_error) + ); + source.valid = false; + source.supported = false; + } + } + if (source.supported) + resolveSource(source, path); + return source; + } + + void validateTiles( + const QJsonValue& value, + const QString& path, + OicSourceDefinition& source) + { + if (!value.isArray()) + { + sourceError( + QStringLiteral("tiles-type"), path, + QStringLiteral("Tiles must be a nonempty array") + ); + return; + } + auto const array = value.toArray(); + if (array.isEmpty() + || array.size() > OicCatalogReader::maximum_tiles_per_source) + { + sourceError( + QStringLiteral("tiles-count"), path, + QStringLiteral("Tiles must contain between 1 and %1 templates") + .arg(OicCatalogReader::maximum_tiles_per_source) + ); + } + QSet unique; + auto const count = std::min( + array.size(), + qsizetype(OicCatalogReader::maximum_tiles_per_source) + ); + for (qsizetype index = 0; index < count; ++index) + { + auto const item_path = path + QStringLiteral("[%1]").arg(index); + if (!array.at(index).isString()) + { + sourceError( + QStringLiteral("template-type"), item_path, + QStringLiteral("Tile template must be a string") + ); + continue; + } + auto const published_template = array.at(index).toString(); + auto runtime_template = published_template; + runtime_template.replace( + QStringLiteral("${z}"), QStringLiteral("{z}") + ); + runtime_template.replace( + QStringLiteral("${x}"), QStringLiteral("{x}") + ); + runtime_template.replace( + QStringLiteral("${y}"), QStringLiteral("{y}") + ); + TileUrlTemplate tile { std::move(runtime_template) }; + QString error; + if (containsUrlWhitespaceOrControl(published_template) + || urlAuthorityContainsUserInfo(published_template)) + { + sourceError( + QStringLiteral("invalid-template"), item_path, + QStringLiteral( + "Tile URL template contains raw whitespace, " + "a control character, or user information" + ) + ); + continue; + } + if (!tile.validate(&error)) + { + sourceError( + QStringLiteral("invalid-template"), item_path, error + ); + continue; + } + if (unique.contains(tile.value)) + { + sourceError( + QStringLiteral("duplicate-template"), item_path, + QStringLiteral("Duplicate tile URL template") + ); + continue; + } + unique.insert(tile.value); + source.tile_urls.push_back(std::move(tile)); + } + } + + void validateMatrixSetUri( + const QJsonValue& value, + const QString& path, + OicSourceDefinition& source) + { + if (!value.isString()) + { + sourceError( + QStringLiteral("matrix-uri-type"), path, + QStringLiteral("Tile matrix set URI must be a string") + ); + return; + } + source.tile_matrix_set_uri = value.toString(); + auto const uri = + httpUrl(source.tile_matrix_set_uri, path, false); + if (!uri.isValid()) + return; + + auto const scheme = uri.scheme().toLower(); + auto const port = uri.port(-1); + auto const default_port = + port < 0 + || (scheme == QLatin1String("http") && port == 80) + || (scheme == QLatin1String("https") && port == 443); + auto const known = + (scheme == QLatin1String("http") + || scheme == QLatin1String("https")) + && uri.host().compare( + QLatin1String(web_mercator_quad_host), + Qt::CaseInsensitive + ) == 0 + && uri.path(QUrl::FullyEncoded) + == QLatin1String(web_mercator_quad_path) + && default_port + && !uri.hasQuery(); + if (!known) + { + addUnsupported( + source, QStringLiteral("tile-matrix-set.external.v1"), + path, QStringLiteral("Unknown tile matrix set URI") + ); + return; + } + + source.tile_matrix_set.matrix_set = + TileMatrixSet::webMercatorQuad(); + source.tile_matrix_set.dyadic_top_left = true; + source.tile_matrix_set.ordered_axes = { + QStringLiteral("X"), QStringLiteral("Y") + }; + constexpr auto base_scale_denominator = 559082264.0287178; + for (auto const& matrix : source.tile_matrix_set.matrix_set.matrices) + { + source.tile_matrix_set.matrices.push_back({ + matrix, + base_scale_denominator / std::ldexp(1.0, matrix.zoom), + QStringLiteral("topLeft"), + false, + {}, + }); + } + source.tile_matrix_set.original_object = + normalizedMatrixSet(source); + } + + void validateTileMatrixSet( + const QJsonValue& value, + const QString& path, + OicSourceDefinition& source) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("matrix-set-type"), path, + QStringLiteral("Tile matrix set must be an object") + ); + return; + } + auto const object = value.toObject(); + auto& definition = source.tile_matrix_set; + definition.original_object = object; + static const QSet fields { + QStringLiteral("id"), QStringLiteral("title"), + QStringLiteral("description"), QStringLiteral("keywords"), + QStringLiteral("uri"), QStringLiteral("orderedAxes"), + QStringLiteral("crs"), QStringLiteral("wellKnownScaleSet"), + QStringLiteral("boundingBox"), QStringLiteral("tileMatrices"), + }; + checkUnknownFields(object, fields, path, false); + definition.matrix_set.id = + requiredId(object, QStringLiteral("id"), + path + QStringLiteral(".id"), false); + optionalText( + object, QStringLiteral("title"), + path + QStringLiteral(".title"), false, 4096 + ); + optionalText( + object, QStringLiteral("description"), + path + QStringLiteral(".description"), false, 4096 + ); + if (object.contains(QStringLiteral("keywords"))) + { + validateStringArray( + object.value(QStringLiteral("keywords")), + path + QStringLiteral(".keywords"), false, false, 64 + ); + } + if (object.contains(QStringLiteral("uri"))) + { + absoluteUrl( + requiredString( + object, QStringLiteral("uri"), + path + QStringLiteral(".uri"), false + ), + path + QStringLiteral(".uri"), false + ); + } + if (object.contains(QStringLiteral("wellKnownScaleSet"))) + { + absoluteUrl( + requiredString( + object, QStringLiteral("wellKnownScaleSet"), + path + QStringLiteral(".wellKnownScaleSet"), false + ), + path + QStringLiteral(".wellKnownScaleSet"), false + ); + } + definition.matrix_set.crs = + normalizeCrs( + requiredString( + object, QStringLiteral("crs"), + path + QStringLiteral(".crs"), false + ), + path + QStringLiteral(".crs") + ); + if (object.contains(QStringLiteral("orderedAxes"))) + { + definition.ordered_axes = + validateStringArray( + object.value(QStringLiteral("orderedAxes")), + path + QStringLiteral(".orderedAxes"), false, false, 2 + ); + if (definition.ordered_axes.size() != 2) + { + sourceError( + QStringLiteral("axis-count"), + path + QStringLiteral(".orderedAxes"), + QStringLiteral("orderedAxes must contain exactly two axes") + ); + } + else + { + auto const first = + definition.ordered_axes.at(0).trimmed().toLower(); + auto const second = + definition.ordered_axes.at(1).trimmed().toLower(); + auto const supported_axes = + (first == QLatin1String("x") + && second == QLatin1String("y")) + || (first == QLatin1String("e") + && second == QLatin1String("n")) + || (first == QLatin1String("easting") + && second == QLatin1String("northing")) + || (first == QLatin1String("longitude") + && second == QLatin1String("latitude")); + if (!supported_axes) + { + addUnsupported( + source, + QStringLiteral("tile-matrix-set.axis-order.v1"), + path + QStringLiteral(".orderedAxes"), + QStringLiteral( + "Tile matrix axes must put the east-west " + "coordinate before the north-south coordinate" + ) + ); + } + } + } + if (object.contains(QStringLiteral("boundingBox"))) + { + validateBoundingBox( + object.value(QStringLiteral("boundingBox")), + path + QStringLiteral(".boundingBox") + ); + } + + if (!object.value(QStringLiteral("tileMatrices")).isArray()) + { + sourceError( + QStringLiteral("matrices-type"), + path + QStringLiteral(".tileMatrices"), + QStringLiteral("Required member must be an array") + ); + return; + } + auto const matrices = + object.value(QStringLiteral("tileMatrices")).toArray(); + if (matrices.isEmpty() + || matrices.size() > OicCatalogReader::maximum_tile_matrices) + { + sourceError( + QStringLiteral("matrix-count"), + path + QStringLiteral(".tileMatrices"), + QStringLiteral("Tile matrix count must be between 1 and %1") + .arg(OicCatalogReader::maximum_tile_matrices) + ); + } + + QSet ids; + auto const count = std::min( + matrices.size(), + qsizetype(OicCatalogReader::maximum_tile_matrices) + ); + auto bottom_left = false; + auto variable_width = false; + for (qsizetype index = 0; index < count; ++index) + { + auto matrix_definition = + validateTileMatrix( + matrices.at(index), + path + QStringLiteral(".tileMatrices[%1]").arg(index), + int(index) + ); + if (!matrix_definition.matrix.id.isEmpty() + && ids.contains(matrix_definition.matrix.id)) + { + sourceError( + QStringLiteral("duplicate-matrix-id"), + path + QStringLiteral(".tileMatrices[%1].id").arg(index), + QStringLiteral("Duplicate tile matrix ID") + ); + } + ids.insert(matrix_definition.matrix.id); + bottom_left = + bottom_left + || matrix_definition.corner_of_origin + == QLatin1String("bottomLeft"); + variable_width = + variable_width + || matrix_definition.has_variable_matrix_widths; + definition.matrix_set.matrices.push_back( + matrix_definition.matrix + ); + definition.matrices.push_back(std::move(matrix_definition)); + } + + if (bottom_left) + { + addUnsupported( + source, QStringLiteral("tile-matrix-set.bottom-left.v1"), + path, + QStringLiteral("Bottom-left matrix origins are not supported") + ); + } + if (variable_width) + { + addUnsupported( + source, QStringLiteral("tile-matrix-set.variable-width.v1"), + path, + QStringLiteral("Variable-width tile matrices are not supported") + ); + } + if (matrices.size() > 63) + { + addUnsupported( + source, QStringLiteral("tile-matrix-set.runtime-size.v1"), + path, + QStringLiteral( + "Tile matrix set exceeds the resolved runtime zoom count" + ) + ); + } + + QString dyadic_error; + definition.dyadic_top_left = + !bottom_left && !variable_width + && matrices.size() <= 63 + && definition.matrix_set.validateDyadicTopLeft(&dyadic_error); + if (!definition.dyadic_top_left + && !bottom_left && !variable_width && matrices.size() <= 63) + { + addUnsupported( + source, QStringLiteral("tile-matrix-set.nondyadic.v1"), + path, + QStringLiteral("Tile matrix set is outside the dyadic " + "top-left runtime profile: %1") + .arg(dyadic_error) + ); + } + } + + OicTileMatrixDefinition validateTileMatrix( + const QJsonValue& value, + const QString& path, + int zoom) + { + OicTileMatrixDefinition definition; + definition.matrix.zoom = zoom; + if (!value.isObject()) + { + sourceError( + QStringLiteral("matrix-type"), path, + QStringLiteral("Tile matrix must be an object") + ); + return definition; + } + auto const object = value.toObject(); + definition.original_object = object; + static const QSet fields { + QStringLiteral("id"), QStringLiteral("title"), + QStringLiteral("description"), QStringLiteral("keywords"), + QStringLiteral("scaleDenominator"), QStringLiteral("cellSize"), + QStringLiteral("pointOfOrigin"), + QStringLiteral("cornerOfOrigin"), QStringLiteral("tileWidth"), + QStringLiteral("tileHeight"), QStringLiteral("matrixWidth"), + QStringLiteral("matrixHeight"), + QStringLiteral("variableMatrixWidths"), + }; + checkUnknownFields(object, fields, path, false); + definition.matrix.id = + requiredString(object, QStringLiteral("id"), + path + QStringLiteral(".id"), false); + if (!validMatrixId(definition.matrix.id)) + { + sourceError( + QStringLiteral("invalid-matrix-id"), + path + QStringLiteral(".id"), + QStringLiteral("Invalid tile matrix identifier") + ); + } + optionalText( + object, QStringLiteral("title"), + path + QStringLiteral(".title"), false, 4096 + ); + optionalText( + object, QStringLiteral("description"), + path + QStringLiteral(".description"), false, 4096 + ); + if (object.contains(QStringLiteral("keywords"))) + { + validateStringArray( + object.value(QStringLiteral("keywords")), + path + QStringLiteral(".keywords"), false, false, 64 + ); + } + definition.scale_denominator = + requiredPositiveNumber( + object, QStringLiteral("scaleDenominator"), + path + QStringLiteral(".scaleDenominator") + ); + definition.matrix.cell_size = + requiredPositiveNumber( + object, QStringLiteral("cellSize"), + path + QStringLiteral(".cellSize") + ); + + auto const origin = object.value(QStringLiteral("pointOfOrigin")); + if (!origin.isArray() || origin.toArray().size() != 2 + || !finiteNumber(origin.toArray().at(0)) + || !finiteNumber(origin.toArray().at(1))) + { + sourceError( + QStringLiteral("invalid-origin"), + path + QStringLiteral(".pointOfOrigin"), + QStringLiteral( + "Point of origin must contain two finite numbers" + ) + ); + } + else + { + definition.matrix.point_of_origin = QPointF( + origin.toArray().at(0).toDouble(), + origin.toArray().at(1).toDouble() + ); + } + if (object.contains(QStringLiteral("cornerOfOrigin"))) + { + definition.corner_of_origin = + requiredString( + object, QStringLiteral("cornerOfOrigin"), + path + QStringLiteral(".cornerOfOrigin"), false + ); + } + if (definition.corner_of_origin != QLatin1String("topLeft") + && definition.corner_of_origin != QLatin1String("bottomLeft")) + { + sourceError( + QStringLiteral("invalid-origin-corner"), + path + QStringLiteral(".cornerOfOrigin"), + QStringLiteral("cornerOfOrigin must be topLeft or bottomLeft") + ); + } + + definition.matrix.tile_size.setWidth(int(requiredInteger( + object, QStringLiteral("tileWidth"), + path + QStringLiteral(".tileWidth"), false, 1, 65536 + ))); + definition.matrix.tile_size.setHeight(int(requiredInteger( + object, QStringLiteral("tileHeight"), + path + QStringLiteral(".tileHeight"), false, 1, 65536 + ))); + definition.matrix.matrix_width = + requiredInteger( + object, QStringLiteral("matrixWidth"), + path + QStringLiteral(".matrixWidth"), false, + 1, qint64(maximum_exact_integer) + ); + definition.matrix.matrix_height = + requiredInteger( + object, QStringLiteral("matrixHeight"), + path + QStringLiteral(".matrixHeight"), false, + 1, qint64(maximum_exact_integer) + ); + if (object.contains(QStringLiteral("variableMatrixWidths"))) + { + definition.has_variable_matrix_widths = true; + validateVariableMatrixWidths( + object.value(QStringLiteral("variableMatrixWidths")), + path + QStringLiteral(".variableMatrixWidths"), + definition.matrix.matrix_height + ); + } + return definition; + } + + void validateVariableMatrixWidths( + const QJsonValue& value, + const QString& path, + qint64 matrix_height) + { + if (!value.isArray() || value.toArray().isEmpty()) + { + sourceError( + QStringLiteral("variable-width-type"), path, + QStringLiteral( + "Variable matrix widths must be a nonempty array" + ) + ); + return; + } + auto const array = value.toArray(); + for (qsizetype index = 0; index < array.size(); ++index) + { + auto const item_path = path + QStringLiteral("[%1]").arg(index); + if (!array.at(index).isObject()) + { + sourceError( + QStringLiteral("variable-width-entry"), item_path, + QStringLiteral( + "Variable matrix width entry must be an object" + ) + ); + continue; + } + auto const object = array.at(index).toObject(); + static const QSet fields { + QStringLiteral("coalesce"), QStringLiteral("minTileRow"), + QStringLiteral("maxTileRow"), + }; + checkUnknownFields(object, fields, item_path, false); + requiredInteger( + object, QStringLiteral("coalesce"), + item_path + QStringLiteral(".coalesce"), false, + 2, qint64(maximum_exact_integer) + ); + auto const minimum = requiredInteger( + object, QStringLiteral("minTileRow"), + item_path + QStringLiteral(".minTileRow"), false, + 0, qint64(maximum_exact_integer) + ); + auto const maximum = requiredInteger( + object, QStringLiteral("maxTileRow"), + item_path + QStringLiteral(".maxTileRow"), false, + 0, qint64(maximum_exact_integer) + ); + if (minimum > maximum || maximum >= matrix_height) + { + sourceError( + QStringLiteral("variable-width-bounds"), item_path, + QStringLiteral( + "Variable matrix width rows are outside the matrix" + ) + ); + } + } + } + + void validateTileMatrixLimits( + const QJsonValue& value, + const QString& path, + OicSourceDefinition& source) + { + if (!value.isArray()) + { + sourceError( + QStringLiteral("limits-type"), path, + QStringLiteral("Tile matrix limits must be an array") + ); + return; + } + auto const array = value.toArray(); + if (array.size() > OicCatalogReader::maximum_tile_matrices) + { + sourceError( + QStringLiteral("limits-count"), path, + QStringLiteral("Too many tile matrix limits") + ); + } + QSet ids; + auto const count = std::min( + array.size(), + qsizetype(OicCatalogReader::maximum_tile_matrices) + ); + for (qsizetype index = 0; index < count; ++index) + { + auto const item_path = path + QStringLiteral("[%1]").arg(index); + if (!array.at(index).isObject()) + { + sourceError( + QStringLiteral("limit-type"), item_path, + QStringLiteral("Tile matrix limit must be an object") + ); + continue; + } + auto const object = array.at(index).toObject(); + static const QSet fields { + QStringLiteral("tileMatrix"), QStringLiteral("minTileRow"), + QStringLiteral("maxTileRow"), QStringLiteral("minTileCol"), + QStringLiteral("maxTileCol"), + }; + checkUnknownFields(object, fields, item_path, false); + auto const matrix_id = + requiredString( + object, QStringLiteral("tileMatrix"), + item_path + QStringLiteral(".tileMatrix"), false + ); + if (!validMatrixId(matrix_id)) + { + sourceError( + QStringLiteral("invalid-matrix-id"), + item_path + QStringLiteral(".tileMatrix"), + QStringLiteral("Invalid tile matrix identifier") + ); + } + auto const min_row = requiredInteger( + object, QStringLiteral("minTileRow"), + item_path + QStringLiteral(".minTileRow"), false, + 0, qint64(maximum_exact_integer) + ); + auto const max_row = requiredInteger( + object, QStringLiteral("maxTileRow"), + item_path + QStringLiteral(".maxTileRow"), false, + 0, qint64(maximum_exact_integer) + ); + auto const min_column = requiredInteger( + object, QStringLiteral("minTileCol"), + item_path + QStringLiteral(".minTileCol"), false, + 0, qint64(maximum_exact_integer) + ); + auto const max_column = requiredInteger( + object, QStringLiteral("maxTileCol"), + item_path + QStringLiteral(".maxTileCol"), false, + 0, qint64(maximum_exact_integer) + ); + if (ids.contains(matrix_id)) + { + sourceError( + QStringLiteral("duplicate-limit"), + item_path + QStringLiteral(".tileMatrix"), + QStringLiteral("Duplicate tile matrix limit") + ); + } + ids.insert(matrix_id); + + source.tile_limit_definitions.push_back({ + matrix_id, min_row, max_row, min_column, max_column + }); + if (min_row > max_row || min_column > max_column) + { + sourceError( + QStringLiteral("limit-bounds"), item_path, + QStringLiteral( + "Tile matrix limit minimum exceeds its maximum" + ) + ); + continue; + } + + auto const matrix_index = matrixIndex(source, matrix_id); + if (matrix_index < 0) + { + if (!source.tile_matrix_set.matrix_set.matrices.isEmpty()) + { + sourceError( + QStringLiteral("unknown-limit-matrix"), + item_path + QStringLiteral(".tileMatrix"), + QStringLiteral( + "Limit refers to an unknown tile matrix" + ) + ); + } + continue; + } + auto const& matrix = + source.tile_matrix_set.matrix_set.matrices.at(matrix_index); + if (min_row > max_row || min_column > max_column + || max_row >= matrix.matrix_height + || max_column >= matrix.matrix_width) + { + sourceError( + QStringLiteral("limit-bounds"), item_path, + QStringLiteral( + "Tile matrix limit is outside the matrix dimensions" + ) + ); + continue; + } + source.tile_limits.push_back({ + matrix.zoom, min_column, max_column, min_row, max_row + }); + } + } + + void validateMatrixRange( + OicSourceDefinition& source, + const QString& path) + { + if (source.tile_matrix_set.matrix_set.matrices.isEmpty()) + return; + auto const& matrices = source.tile_matrix_set.matrix_set.matrices; + if (source.min_tile_matrix.isEmpty()) + source.min_tile_matrix = matrices.first().id; + if (source.max_tile_matrix.isEmpty()) + source.max_tile_matrix = matrices.last().id; + auto const minimum = matrixIndex(source, source.min_tile_matrix); + auto const maximum = matrixIndex(source, source.max_tile_matrix); + if (minimum < 0) + { + sourceError( + QStringLiteral("unknown-minimum-matrix"), + path + QStringLiteral(".minTileMatrix"), + QStringLiteral("Unknown minimum tile matrix") + ); + } + if (maximum < 0) + { + sourceError( + QStringLiteral("unknown-maximum-matrix"), + path + QStringLiteral(".maxTileMatrix"), + QStringLiteral("Unknown maximum tile matrix") + ); + } + if (minimum >= 0 && maximum >= 0 && minimum > maximum) + { + sourceError( + QStringLiteral("matrix-range-order"), path, + QStringLiteral( + "Minimum tile matrix follows maximum tile matrix" + ) + ); + } + for (auto const& limit : source.tile_limit_definitions) + { + auto const limit_index = + matrixIndex(source, limit.tile_matrix); + if (minimum >= 0 && maximum >= 0 + && limit_index >= 0 + && (limit_index < minimum || limit_index > maximum)) + { + sourceError( + QStringLiteral("limit-range"), + path + QStringLiteral(".tileMatrixLimits"), + QStringLiteral( + "Tile matrix limit falls outside the usable range" + ) + ); + } + } + } + + void validateRequest( + const QJsonValue& value, + const QString& path, + OicSourceDefinition& source) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("request-type"), path, + QStringLiteral("Request behavior must be an object") + ); + return; + } + auto const object = value.toObject(); + static const QSet fields { + QStringLiteral("referer"), + QStringLiteral("emptyHttpStatusCodes"), + }; + checkUnknownFields(object, fields, path, false); + if (object.contains(QStringLiteral("referer"))) + { + auto const url = httpUrl( + requiredString( + object, QStringLiteral("referer"), + path + QStringLiteral(".referer"), false + ), + path + QStringLiteral(".referer"), false + ); + source.request.referer = url; + } + if (!object.contains(QStringLiteral("emptyHttpStatusCodes"))) + return; + auto const codes = + object.value(QStringLiteral("emptyHttpStatusCodes")); + if (!codes.isArray()) + { + sourceError( + QStringLiteral("status-codes-type"), + path + QStringLiteral(".emptyHttpStatusCodes"), + QStringLiteral("HTTP codes must be an array") + ); + return; + } + auto const array = codes.toArray(); + if (array.size() > OicCatalogReader::maximum_empty_status_codes) + { + sourceError( + QStringLiteral("status-codes-count"), + path + QStringLiteral(".emptyHttpStatusCodes"), + QStringLiteral("Too many empty-tile HTTP codes") + ); + } + source.request.empty_http_status_codes.clear(); + QSet unique; + auto const count = std::min( + array.size(), + qsizetype(OicCatalogReader::maximum_empty_status_codes) + ); + for (qsizetype index = 0; index < count; ++index) + { + auto const code = int(integerValue( + array.at(index), + path + QStringLiteral(".emptyHttpStatusCodes[%1]").arg(index), + false, 100, 599 + )); + if (unique.contains(code)) + { + sourceError( + QStringLiteral("duplicate-status-code"), + path + QStringLiteral(".emptyHttpStatusCodes[%1]") + .arg(index), + QStringLiteral("Duplicate HTTP status code") + ); + } + unique.insert(code); + source.request.empty_http_status_codes.push_back(code); + } + } + + void validatePresentation( + const QJsonObject& object, + const QString& path, + OicSourceDefinition& source) + { + if (object.contains(QStringLiteral("category"))) + { + auto const category = + requiredString( + object, QStringLiteral("category"), + path + QStringLiteral(".category"), false + ); + auto const parsed = categoryFromName(category); + if (!parsed) + { + sourceError( + QStringLiteral("invalid-category"), + path + QStringLiteral(".category"), + QStringLiteral("Unknown source category") + ); + } + else + { + source.metadata.category = *parsed; + } + } + source.metadata.start_date = + optionalDate( + object, QStringLiteral("startDate"), + path + QStringLiteral(".startDate"), false + ); + source.metadata.end_date = + optionalDate( + object, QStringLiteral("endDate"), + path + QStringLiteral(".endDate"), false + ); + if (source.metadata.start_date.isValid() + && source.metadata.end_date.isValid() + && source.metadata.start_date > source.metadata.end_date) + { + sourceError( + QStringLiteral("date-order"), path, + QStringLiteral("Source startDate follows endDate") + ); + } + if (object.contains(QStringLiteral("coverage"))) + { + if (object.value(QStringLiteral("coverage")).isObject()) + { + source.coverage = + object.value(QStringLiteral("coverage")).toObject(); + } + int vertices = 0; + validateGeometry( + object.value(QStringLiteral("coverage")), + path + QStringLiteral(".coverage"), vertices + ); + if (vertices > OicCatalogReader::maximum_coverage_vertices) + { + sourceError( + QStringLiteral("coverage-limit"), + path + QStringLiteral(".coverage"), + QStringLiteral("Coverage exceeds the %1 vertex limit") + .arg(OicCatalogReader::maximum_coverage_vertices) + ); + } + } + if (object.contains(QStringLiteral("notices"))) + { + validateNotices( + object.value(QStringLiteral("notices")), + path + QStringLiteral(".notices"), source.notices + ); + } + } + + void validateNotices( + const QJsonValue& value, + const QString& path, + ImageryNotices& notices) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("notices-type"), path, + QStringLiteral("Notices must be an object") + ); + return; + } + auto const object = value.toObject(); + static const QSet fields { + QStringLiteral("attributionText"), + QStringLiteral("attributionUrl"), + QStringLiteral("sourceUrl"), QStringLiteral("termsUrl"), + QStringLiteral("privacyUrl"), QStringLiteral("notes"), + }; + checkUnknownFields(object, fields, path, false); + notices.attribution_text = + optionalText( + object, QStringLiteral("attributionText"), + path + QStringLiteral(".attributionText"), false, 2048 + ); + notices.notes = + optionalText( + object, QStringLiteral("notes"), + path + QStringLiteral(".notes"), false, 4096 + ); + for (auto const& field : { + QStringLiteral("attributionUrl"), QStringLiteral("sourceUrl"), + QStringLiteral("termsUrl"), QStringLiteral("privacyUrl") + }) + { + if (!object.contains(field)) + continue; + auto const url = + httpUrl( + requiredString( + object, field, path + QLatin1Char('.') + field, + false + ), + path + QLatin1Char('.') + field, false + ); + if (field == QLatin1String("attributionUrl")) + notices.attribution_url = url; + else if (field == QLatin1String("sourceUrl")) + notices.source_url = url; + else if (field == QLatin1String("termsUrl")) + notices.terms_url = url; + else + notices.privacy_url = url; + } + } + + void validateRegistration( + const QJsonValue& value, + const QString& path, + OicSourceDefinition& source) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("registration-type"), path, + QStringLiteral("Registration must be an object") + ); + return; + } + auto const object = value.toObject(); + auto& registration = source.registration; + registration.original_object = object; + static const QSet fields { + QStringLiteral("direction"), QStringLiteral("sourceFrame"), + QStringLiteral("targetFrame"), QStringLiteral("operation"), + QStringLiteral("provenance"), + }; + checkUnknownFields(object, fields, path, false); + registration.direction = + requiredString( + object, QStringLiteral("direction"), + path + QStringLiteral(".direction"), false + ); + if (registration.direction + != QLatin1String("source-to-corrected")) + { + sourceError( + QStringLiteral("registration-direction"), + path + QStringLiteral(".direction"), + QStringLiteral( + "Registration direction must be source-to-corrected" + ) + ); + } + registration.source_crs = + validateFrame( + object.value(QStringLiteral("sourceFrame")), + path + QStringLiteral(".sourceFrame"), nullptr + ); + registration.target_crs = + validateFrame( + object.value(QStringLiteral("targetFrame")), + path + QStringLiteral(".targetFrame"), + ®istration.target_frame_id + ); + if (!source.tile_matrix_set.matrix_set.crs.isEmpty() + && registration.source_crs + != source.tile_matrix_set.matrix_set.crs) + { + sourceError( + QStringLiteral("registration-source-crs"), + path + QStringLiteral(".sourceFrame.crs"), + QStringLiteral( + "Registration source frame must match the tile " + "matrix set CRS" + ) + ); + } + if (registration.source_crs != registration.target_crs) + { + sourceError( + QStringLiteral("registration-target-crs"), + path + QStringLiteral(".targetFrame.crs"), + QStringLiteral( + "Version 1 registration frames must use the same CRS" + ) + ); + } + + auto const operation_value = object.value(QStringLiteral("operation")); + if (!operation_value.isObject()) + { + sourceError( + QStringLiteral("registration-operation"), + path + QStringLiteral(".operation"), + QStringLiteral("Registration operation must be an object") + ); + return; + } + auto const operation = operation_value.toObject(); + auto const operation_path = path + QStringLiteral(".operation"); + auto const type = + requiredString( + operation, QStringLiteral("type"), + operation_path + QStringLiteral(".type"), false + ); + if (type == QLatin1String("translation2d")) + { + validateTranslation(operation, operation_path, source); + } + else if (type == QLatin1String("affine2d")) + { + validateAffine(operation, operation_path, source); + addUnsupported( + source, QStringLiteral("registration.affine2d.v1"), + operation_path, + QStringLiteral( + "Affine registration is parsed but not executable" + ) + ); + } + else if (type == QLatin1String("gridShift")) + { + validateGridShift(operation, operation_path, source); + addUnsupported( + source, QStringLiteral("registration.grid-shift.v1"), + operation_path, + QStringLiteral( + "Grid-shift registration is parsed but not executable" + ) + ); + } + else + { + sourceError( + QStringLiteral("unknown-registration"), + operation_path + QStringLiteral(".type"), + QStringLiteral("Unknown registration operation") + ); + } + if (object.contains(QStringLiteral("provenance"))) + { + validateProvenance( + object.value(QStringLiteral("provenance")), + path + QStringLiteral(".provenance"), + registration.provenance + ); + } + } + + void validateTranslation( + const QJsonObject& object, + const QString& path, + OicSourceDefinition& source) + { + static const QSet fields { + QStringLiteral("type"), QStringLiteral("unit"), + QStringLiteral("dx"), QStringLiteral("dy"), + }; + checkUnknownFields(object, fields, path, false); + auto& registration = source.registration; + registration.kind = OicRegistrationKind::Translation2d; + registration.unit = + requiredString( + object, QStringLiteral("unit"), + path + QStringLiteral(".unit"), false + ); + if (registration.unit != QLatin1String("crs")) + { + sourceError( + QStringLiteral("registration-unit"), + path + QStringLiteral(".unit"), + QStringLiteral("Translation unit must be crs") + ); + } + registration.dx = + requiredNumber( + object, QStringLiteral("dx"), + path + QStringLiteral(".dx") + ); + registration.dy = + requiredNumber( + object, QStringLiteral("dy"), + path + QStringLiteral(".dy") + ); + } + + void validateAffine( + const QJsonObject& object, + const QString& path, + OicSourceDefinition& source) + { + static const QSet fields { + QStringLiteral("type"), QStringLiteral("unit"), + QStringLiteral("xoff"), QStringLiteral("yoff"), + QStringLiteral("s11"), QStringLiteral("s12"), + QStringLiteral("s21"), QStringLiteral("s22"), + }; + checkUnknownFields(object, fields, path, false); + auto& registration = source.registration; + registration.kind = OicRegistrationKind::Affine2d; + registration.unit = + requiredString( + object, QStringLiteral("unit"), + path + QStringLiteral(".unit"), false + ); + if (registration.unit != QLatin1String("crs")) + { + sourceError( + QStringLiteral("registration-unit"), + path + QStringLiteral(".unit"), + QStringLiteral("Affine unit must be crs") + ); + } + registration.xoff = + requiredNumber( + object, QStringLiteral("xoff"), + path + QStringLiteral(".xoff") + ); + registration.yoff = + requiredNumber( + object, QStringLiteral("yoff"), + path + QStringLiteral(".yoff") + ); + registration.s11 = + requiredNumber( + object, QStringLiteral("s11"), + path + QStringLiteral(".s11") + ); + registration.s12 = + requiredNumber( + object, QStringLiteral("s12"), + path + QStringLiteral(".s12") + ); + registration.s21 = + requiredNumber( + object, QStringLiteral("s21"), + path + QStringLiteral(".s21") + ); + registration.s22 = + requiredNumber( + object, QStringLiteral("s22"), + path + QStringLiteral(".s22") + ); + auto const determinant = + registration.s11 * registration.s22 + - registration.s12 * registration.s21; + if (!std::isfinite(determinant) || determinant == 0) + { + sourceError( + QStringLiteral("singular-affine"), path, + QStringLiteral("Affine registration must be invertible") + ); + } + } + + void validateGridShift( + const QJsonObject& object, + const QString& path, + OicSourceDefinition& source) + { + static const QSet fields { + QStringLiteral("type"), QStringLiteral("resource"), + QStringLiteral("domain"), QStringLiteral("gridFrame"), + QStringLiteral("interpolation"), + }; + checkUnknownFields(object, fields, path, false); + auto& registration = source.registration; + registration.kind = OicRegistrationKind::GridShift; + registration.resource_id = + requiredId( + object, QStringLiteral("resource"), + path + QStringLiteral(".resource"), false + ); + if (!result.catalog.resource(registration.resource_id)) + { + sourceError( + QStringLiteral("unknown-grid-resource"), + path + QStringLiteral(".resource"), + QStringLiteral( + "Grid shift refers to an undeclared resource" + ) + ); + } + registration.grid_domain = + requiredString( + object, QStringLiteral("domain"), + path + QStringLiteral(".domain"), false + ); + if (!QSet { + QStringLiteral("horizontal"), QStringLiteral("vertical"), + QStringLiteral("horizontal-and-vertical"), + }.contains(registration.grid_domain)) + { + sourceError( + QStringLiteral("grid-domain"), + path + QStringLiteral(".domain"), + QStringLiteral("Unknown grid-shift domain") + ); + } + registration.grid_crs = + validateFrame( + object.value(QStringLiteral("gridFrame")), + path + QStringLiteral(".gridFrame"), nullptr + ); + registration.interpolation = + requiredString( + object, QStringLiteral("interpolation"), + path + QStringLiteral(".interpolation"), false + ); + if (!QSet { + QStringLiteral("bilinear"), QStringLiteral("biquadratic"), + QStringLiteral("bicubic"), + }.contains(registration.interpolation)) + { + sourceError( + QStringLiteral("grid-interpolation"), + path + QStringLiteral(".interpolation"), + QStringLiteral("Unknown grid-shift interpolation") + ); + } + } + + QString validateFrame( + const QJsonValue& value, + const QString& path, + QString* id) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("frame-type"), path, + QStringLiteral("Frame must be an object") + ); + return {}; + } + auto const object = value.toObject(); + static const QSet fields { + QStringLiteral("crs"), QStringLiteral("id"), + }; + checkUnknownFields(object, fields, path, false); + auto const frame_id = + optionalId( + object, QStringLiteral("id"), + path + QStringLiteral(".id"), false + ); + if (id) + *id = frame_id; + return normalizeCrs( + requiredString( + object, QStringLiteral("crs"), + path + QStringLiteral(".crs"), false + ), + path + QStringLiteral(".crs") + ); + } + + void validateProvenance( + const QJsonValue& value, + const QString& path, + ImageryProvenance& provenance) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("provenance-type"), path, + QStringLiteral("Provenance must be an object") + ); + return; + } + auto const object = value.toObject(); + static const QSet fields { + QStringLiteral("method"), QStringLiteral("observed"), + QStringLiteral("author"), QStringLiteral("rmsError"), + QStringLiteral("notes"), + }; + checkUnknownFields(object, fields, path, false); + provenance.method = + optionalText( + object, QStringLiteral("method"), + path + QStringLiteral(".method"), false, 256 + ); + provenance.observed = + optionalDate( + object, QStringLiteral("observed"), + path + QStringLiteral(".observed"), false + ); + provenance.author = + optionalText( + object, QStringLiteral("author"), + path + QStringLiteral(".author"), false, 512 + ); + provenance.notes = + optionalText( + object, QStringLiteral("notes"), + path + QStringLiteral(".notes"), false, 4096 + ); + if (object.contains(QStringLiteral("rmsError"))) + { + auto const rms = + numberValue( + object.value(QStringLiteral("rmsError")), + path + QStringLiteral(".rmsError"), false + ); + if (rms < 0) + { + sourceError( + QStringLiteral("negative-rms"), + path + QStringLiteral(".rmsError"), + QStringLiteral("RMS error must be nonnegative") + ); + } + provenance.rms_error = rms; + } + } + + void validateResources( + const QJsonValue& value, + const QString& path) + { + if (!value.isObject()) + { + catalogError( + QStringLiteral("resources-type"), path, + QStringLiteral("Resources must be an object") + ); + return; + } + auto const resources = value.toObject(); + if (resources.size() > OicCatalogReader::maximum_resources) + { + catalogError( + QStringLiteral("resource-limit"), path, + QStringLiteral("Catalog exceeds the %1 resource limit") + .arg(OicCatalogReader::maximum_resources) + ); + } + auto processed = 0; + for (auto it = resources.begin(); + it != resources.end() + && processed < OicCatalogReader::maximum_resources; + ++it, ++processed) + { + auto const item_path = path + QLatin1Char('.') + it.key(); + OicResource resource; + resource.id = it.key(); + if (!validId(resource.id)) + { + catalogError( + QStringLiteral("invalid-resource-id"), item_path, + QStringLiteral("Invalid resource ID") + ); + } + if (!it.value().isObject()) + { + catalogError( + QStringLiteral("resource-type"), item_path, + QStringLiteral("Resource must be an object") + ); + continue; + } + auto const object = it.value().toObject(); + resource.original_object = object; + static const QSet fields { + QStringLiteral("href"), QStringLiteral("mediaType"), + QStringLiteral("sha256"), QStringLiteral("size"), + }; + checkUnknownFields(object, fields, item_path, true); + resource.href = + requiredString( + object, QStringLiteral("href"), + item_path + QStringLiteral(".href"), true + ); + validateResourceHref( + resource.href, item_path + QStringLiteral(".href") + ); + resource.media_type = + requiredText( + object, QStringLiteral("mediaType"), + item_path + QStringLiteral(".mediaType"), true, 255 + ); + auto const digest = + requiredString( + object, QStringLiteral("sha256"), + item_path + QStringLiteral(".sha256"), true + ); + static const QRegularExpression digest_pattern( + QStringLiteral("^[0-9a-f]{64}$") + ); + if (!digest_pattern.match(digest).hasMatch()) + { + catalogError( + QStringLiteral("resource-digest"), + item_path + QStringLiteral(".sha256"), + QStringLiteral( + "Resource SHA-256 must contain 64 lowercase " + "hexadecimal characters" + ) + ); + } + resource.sha256 = digest.toLatin1(); + resource.size = + requiredInteger( + object, QStringLiteral("size"), + item_path + QStringLiteral(".size"), true, + 1, 1073741824 + ); + result.catalog.resources.push_back(std::move(resource)); + } + } + + void validateResourceHref( + const QString& href, + const QString& path) + { + if (href.isEmpty() + || href.size() > OicCatalogReader::maximum_url_length + || containsUrlWhitespaceOrControl(href)) + { + catalogError( + QStringLiteral("resource-href"), path, + QStringLiteral("Resource href is empty, too long, or unsafe") + ); + return; + } + auto const url = QUrl(href, QUrl::StrictMode); + if (!url.isValid()) + { + catalogError( + QStringLiteral("resource-href"), path, + QStringLiteral("Resource href is invalid") + ); + return; + } + if (url.hasFragment()) + { + catalogError( + QStringLiteral("resource-fragment"), path, + QStringLiteral("Resource href must not contain a fragment") + ); + return; + } + if (url.isRelative()) + { + auto const decoded_path = url.path(QUrl::FullyDecoded); + auto const segments = decoded_path.split(QLatin1Char('/')); + if (href.startsWith(QLatin1Char('/')) + || segments.contains(QStringLiteral("..")) + || decoded_path.contains(QLatin1Char('\\'))) + { + catalogError( + QStringLiteral("resource-path"), path, + QStringLiteral("Unsafe relative resource path") + ); + } + return; + } + if (url.scheme().toLower() != QLatin1String("https") + || url.host().isEmpty() + || !url.userName().isEmpty() || !url.password().isEmpty()) + { + catalogError( + QStringLiteral("resource-url"), path, + QStringLiteral( + "Remote resources must use HTTPS without user " + "information or fragments" + ) + ); + } + } + + OicPublisher validatePublisher( + const QJsonValue& value, + const QString& path) + { + OicPublisher publisher; + if (!value.isObject()) + { + catalogError( + QStringLiteral("publisher-type"), path, + QStringLiteral("Publisher must be an object") + ); + return publisher; + } + auto const object = value.toObject(); + publisher.original_object = object; + static const QSet fields { + QStringLiteral("name"), QStringLiteral("url"), + QStringLiteral("contactUrl"), + }; + checkUnknownFields(object, fields, path, true); + publisher.name = + requiredText( + object, QStringLiteral("name"), + path + QStringLiteral(".name"), true, 512 + ); + if (object.contains(QStringLiteral("url"))) + { + publisher.url = + httpUrl( + requiredString( + object, QStringLiteral("url"), + path + QStringLiteral(".url"), true + ), + path + QStringLiteral(".url"), true + ); + } + if (object.contains(QStringLiteral("contactUrl"))) + { + publisher.contact_url = + httpUrl( + requiredString( + object, QStringLiteral("contactUrl"), + path + QStringLiteral(".contactUrl"), true + ), + path + QStringLiteral(".contactUrl"), true + ); + } + return publisher; + } + + void validateBoundingBox( + const QJsonValue& value, + const QString& path) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("bounding-box-type"), path, + QStringLiteral("Bounding box must be an object") + ); + return; + } + auto const object = value.toObject(); + static const QSet fields { + QStringLiteral("crs"), QStringLiteral("orderedAxes"), + QStringLiteral("lowerLeft"), QStringLiteral("upperRight"), + }; + checkUnknownFields(object, fields, path, false); + if (object.contains(QStringLiteral("crs"))) + { + normalizeCrs( + requiredString( + object, QStringLiteral("crs"), + path + QStringLiteral(".crs"), false + ), + path + QStringLiteral(".crs") + ); + } + if (object.contains(QStringLiteral("orderedAxes"))) + { + auto const axes = + validateStringArray( + object.value(QStringLiteral("orderedAxes")), + path + QStringLiteral(".orderedAxes"), + false, false, 2 + ); + if (axes.size() != 2) + { + sourceError( + QStringLiteral("axis-count"), + path + QStringLiteral(".orderedAxes"), + QStringLiteral( + "orderedAxes must contain exactly two axes" + ) + ); + } + } + validatePosition( + object.value(QStringLiteral("lowerLeft")), + path + QStringLiteral(".lowerLeft") + ); + validatePosition( + object.value(QStringLiteral("upperRight")), + path + QStringLiteral(".upperRight") + ); + } + + void validateGeometry( + const QJsonValue& value, + const QString& path, + int& vertices) + { + if (!value.isObject()) + { + sourceError( + QStringLiteral("coverage-type"), path, + QStringLiteral("Coverage geometry must be an object") + ); + return; + } + auto const object = value.toObject(); + auto const type = + requiredString( + object, QStringLiteral("type"), + path + QStringLiteral(".type"), false + ); + if (type == QLatin1String("GeometryCollection")) + { + static const QSet fields { + QStringLiteral("type"), QStringLiteral("geometries"), + }; + checkUnknownFields(object, fields, path, false); + auto const geometries = + object.value(QStringLiteral("geometries")); + if (!geometries.isArray()) + { + sourceError( + QStringLiteral("geometry-collection"), + path + QStringLiteral(".geometries"), + QStringLiteral( + "Geometry collection must contain an array" + ) + ); + return; + } + auto const array = geometries.toArray(); + for (qsizetype index = 0; + index < array.size() + && vertices <= OicCatalogReader::maximum_coverage_vertices; + ++index) + { + validateGeometry( + array.at(index), + path + QStringLiteral(".geometries[%1]").arg(index), + vertices + ); + } + return; + } + + static const QSet fields { + QStringLiteral("type"), QStringLiteral("coordinates"), + }; + checkUnknownFields(object, fields, path, false); + if (!object.contains(QStringLiteral("coordinates"))) + { + sourceError( + QStringLiteral("coverage-coordinates"), + path + QStringLiteral(".coordinates"), + QStringLiteral("Geometry coordinates are required") + ); + return; + } + auto coordinate_depth = -1; + if (type == QLatin1String("Point")) + coordinate_depth = 0; + else if (type == QLatin1String("MultiPoint") + || type == QLatin1String("LineString")) + coordinate_depth = 1; + else if (type == QLatin1String("MultiLineString") + || type == QLatin1String("Polygon")) + coordinate_depth = 2; + else if (type == QLatin1String("MultiPolygon")) + coordinate_depth = 3; + else + { + sourceError( + QStringLiteral("geometry-type"), + path + QStringLiteral(".type"), + QStringLiteral("Unknown GeoJSON geometry type") + ); + } + if (coordinate_depth < 0) + return; + auto const coordinates_path = path + QStringLiteral(".coordinates"); + validateCoordinates( + object.value(QStringLiteral("coordinates")), + coordinates_path, coordinate_depth, vertices + ); + if (object.value(QStringLiteral("coordinates")).isArray()) + { + validateGeometryShape( + type, + object.value(QStringLiteral("coordinates")).toArray(), + coordinates_path + ); + } + } + + void validateGeometryShape( + const QString& type, + const QJsonArray& coordinates, + const QString& path) + { + if (type == QLatin1String("LineString") + && coordinates.size() < 2) + { + sourceError( + QStringLiteral("line-size"), path, + QStringLiteral( + "LineString must contain at least two positions" + ) + ); + } + else if (type == QLatin1String("MultiLineString")) + { + for (qsizetype index = 0; index < coordinates.size(); ++index) + { + if (coordinates.at(index).isArray() + && coordinates.at(index).toArray().size() < 2) + { + sourceError( + QStringLiteral("line-size"), + path + QStringLiteral("[%1]").arg(index), + QStringLiteral( + "LineString must contain at least two positions" + ) + ); + } + } + } + else if (type == QLatin1String("Polygon")) + { + validatePolygonRings(coordinates, path); + } + else if (type == QLatin1String("MultiPolygon")) + { + for (qsizetype index = 0; index < coordinates.size(); ++index) + { + if (coordinates.at(index).isArray()) + { + validatePolygonRings( + coordinates.at(index).toArray(), + path + QStringLiteral("[%1]").arg(index) + ); + } + } + } + } + + void validatePolygonRings( + const QJsonArray& rings, + const QString& path) + { + for (qsizetype index = 0; index < rings.size(); ++index) + { + if (!rings.at(index).isArray()) + continue; + auto const ring = rings.at(index).toArray(); + auto const ring_path = + path + QStringLiteral("[%1]").arg(index); + if (ring.size() < 4) + { + sourceError( + QStringLiteral("ring-size"), ring_path, + QStringLiteral( + "Polygon ring must contain at least four positions" + ) + ); + } + else if (ring.first() != ring.last()) + { + sourceError( + QStringLiteral("ring-closure"), ring_path, + QStringLiteral("Polygon ring must be closed") + ); + } + } + } + + void validateCoordinates( + const QJsonValue& value, + const QString& path, + int depth, + int& vertices) + { + if (!value.isArray()) + { + sourceError( + QStringLiteral("coordinate-type"), path, + QStringLiteral("GeoJSON coordinates must be arrays") + ); + return; + } + auto const array = value.toArray(); + if (array.isEmpty()) + { + sourceError( + QStringLiteral("coordinate-empty"), path, + QStringLiteral( + "GeoJSON coordinate array must not be empty" + ) + ); + } + if (depth == 0) + { + if (array.size() < 2 || array.size() > 3 + || !finiteNumber(array.at(0)) + || !finiteNumber(array.at(1)) + || (array.size() == 3 && !finiteNumber(array.at(2)))) + { + sourceError( + QStringLiteral("position"), path, + QStringLiteral( + "GeoJSON position must contain two or three " + "finite numbers" + ) + ); + return; + } + auto const longitude = array.at(0).toDouble(); + auto const latitude = array.at(1).toDouble(); + if (longitude < -180 || longitude > 180 + || latitude < -90 || latitude > 90) + { + sourceError( + QStringLiteral("position-bounds"), path, + QStringLiteral( + "Coverage position is outside WGS84 " + "longitude/latitude bounds" + ) + ); + } + ++vertices; + return; + } + for (qsizetype index = 0; + index < array.size() + && vertices <= OicCatalogReader::maximum_coverage_vertices; + ++index) + { + validateCoordinates( + array.at(index), + path + QStringLiteral("[%1]").arg(index), + depth - 1, vertices + ); + } + } + + void validatePosition( + const QJsonValue& value, + const QString& path) + { + if (!value.isArray() || value.toArray().size() != 2 + || !finiteNumber(value.toArray().at(0)) + || !finiteNumber(value.toArray().at(1))) + { + sourceError( + QStringLiteral("position"), path, + QStringLiteral("Expected a two-number position") + ); + } + } + + void resolveSource( + OicSourceDefinition& definition, + const QString& path) + { + auto const minimum = + matrixIndex(definition, definition.min_tile_matrix); + auto const maximum = + matrixIndex(definition, definition.max_tile_matrix); + if (minimum < 0 || maximum < minimum) + { + sourceError( + QStringLiteral("runtime-matrix-range"), path, + QStringLiteral( + "Supported source has no executable tile matrix range" + ) + ); + definition.valid = false; + definition.supported = false; + return; + } + for (auto index = minimum; index <= maximum; ++index) + { + if (!runtimeSupportsTileSize( + definition.tile_matrix_set.matrix_set.matrices + .at(index).tile_size)) + { + addUnsupported( + definition, + QStringLiteral("tile-size.runtime.v1"), + path + QStringLiteral(".tileMatrixSet"), + QStringLiteral( + "Tile dimensions exceed this build's raster execution profile") + ); + definition.supported = false; + return; + } + } + + ResolvedImagerySource source; + source.metadata = definition.metadata; + source.notices = definition.notices; + source.tile_urls = definition.tile_urls; + source.row_scheme = definition.row_scheme; + source.media_type = definition.media_type; + source.tile_matrix_set = definition.tile_matrix_set.matrix_set; + source.min_zoom = + source.tile_matrix_set.matrices.at(minimum).zoom; + source.max_zoom = + source.tile_matrix_set.matrices.at(maximum).zoom; + source.tile_limits = definition.tile_limits; + source.request = definition.request; + source.catalog_provenance = CatalogSourceProvenance { + result.catalog.id, + result.catalog.revision, + result.catalog.document_sha256, + definition.metadata.id, + definition.full_fingerprint, + definition.operational_fingerprint, + }; + if (definition.registration.kind + == OicRegistrationKind::Translation2d) + { + source.registration = TranslationRegistration { + definition.registration.source_crs, + definition.registration.target_crs, + definition.registration.target_frame_id, + definition.registration.dx, + definition.registration.dy, + definition.registration.provenance, + }; + } + + QString error; + if (!source.validate(&error)) + { + sourceError( + QStringLiteral("runtime-contract"), path, + QStringLiteral( + "Source cannot satisfy the resolved runtime contract: %1" + ).arg(error) + ); + definition.valid = false; + definition.supported = false; + return; + } + definition.resolved_source = std::move(source); + } + + QString normalizeCrs( + const QString& value, + const QString& path) + { + static const QRegularExpression short_form( + QStringLiteral("^EPSG:([1-9][0-9]{0,8})$") + ); + static const QRegularExpression uri_form( + QStringLiteral( + "^https?://www\\.opengis\\.net/def/crs/EPSG/" + "(?:0|[0-9.]+)/([1-9][0-9]{0,8})$" + ) + ); + auto match = short_form.match(value); + if (!match.hasMatch()) + match = uri_form.match(value); + if (!match.hasMatch()) + { + sourceError( + QStringLiteral("invalid-crs"), path, + QStringLiteral( + "CRS must be an EPSG code or equivalent OGC URI" + ) + ); + return {}; + } + return QStringLiteral("EPSG:%1").arg(match.captured(1)); + } + + QStringList validateCapabilities( + const QJsonValue& value, + const QString& path, + bool catalog_level) + { + auto const result = + validateStringArray( + value, path, catalog_level, true, 64 + ); + for (qsizetype index = 0; index < result.size(); ++index) + { + if (!validId(result.at(index))) + { + addError( + catalog_level, + QStringLiteral("invalid-capability"), + path + QStringLiteral("[%1]").arg(index), + QStringLiteral("Invalid capability identifier") + ); + } + } + return result; + } + + QStringList validateStringArray( + const QJsonValue& value, + const QString& path, + bool catalog_level, + bool unique, + int maximum) + { + QStringList result; + if (!value.isArray()) + { + addError( + catalog_level, QStringLiteral("string-array"), path, + QStringLiteral("Expected an array of strings") + ); + return result; + } + auto const array = value.toArray(); + if (array.size() > maximum) + { + addError( + catalog_level, QStringLiteral("array-limit"), path, + QStringLiteral("Array exceeds the %1 item limit").arg(maximum) + ); + } + QSet seen; + auto const count = + std::min(array.size(), qsizetype(maximum)); + for (qsizetype index = 0; index < count; ++index) + { + auto const item_path = path + QStringLiteral("[%1]").arg(index); + if (!array.at(index).isString() + || array.at(index).toString().isEmpty() + || containsControl(array.at(index).toString())) + { + addError( + catalog_level, QStringLiteral("string-array"), + item_path, + QStringLiteral( + "Expected a nonempty string without controls" + ) + ); + continue; + } + auto const text = array.at(index).toString(); + if (unique && seen.contains(text)) + { + addError( + catalog_level, QStringLiteral("duplicate-array-value"), + item_path, QStringLiteral("Duplicate array value") + ); + } + seen.insert(text); + result.push_back(text); + } + return result; + } + + void validateExtensions( + const QJsonObject& object, + const QString& path, + bool catalog_level) + { + if (object.size() > 128) + { + addError( + catalog_level, QStringLiteral("extension-limit"), path, + QStringLiteral("Extensions exceed the 128-key limit") + ); + } + static const QRegularExpression namespaced( + QStringLiteral("^[A-Za-z0-9-]+(?:\\.[A-Za-z0-9-]+)+$") + ); + for (auto it = object.begin(); it != object.end(); ++it) + { + if (!namespaced.match(it.key()).hasMatch()) + { + addError( + catalog_level, QStringLiteral("extension-namespace"), + path + QLatin1Char('.') + it.key(), + QStringLiteral( + "Extension key must be reverse-DNS namespaced" + ) + ); + } + } + } + + void checkUnknownFields( + const QJsonObject& object, + const QSet& allowed, + const QString& path, + bool catalog_level) + { + for (auto it = object.begin(); it != object.end(); ++it) + { + if (!allowed.contains(it.key())) + { + addError( + catalog_level, QStringLiteral("unknown-member"), + path + QLatin1Char('.') + it.key(), + QStringLiteral("Unknown member") + ); + } + } + } + + QString requiredText( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level, + qsizetype maximum) + { + auto const text = + requiredString(object, name, path, catalog_level); + validateText(text, path, catalog_level, maximum); + return text; + } + + QString optionalText( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level, + qsizetype maximum) + { + if (!object.contains(name)) + return {}; + auto const text = + requiredString(object, name, path, catalog_level); + validateText(text, path, catalog_level, maximum); + return text; + } + + void validateText( + const QString& text, + const QString& path, + bool catalog_level, + qsizetype maximum) + { + if (text.isEmpty() || text.size() > maximum || containsControl(text)) + { + addError( + catalog_level, QStringLiteral("invalid-text"), path, + QStringLiteral( + "Text must be nonempty, bounded, and free of controls" + ) + ); + } + } + + QString requiredId( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level) + { + auto const id = + requiredString(object, name, path, catalog_level); + if (!id.isEmpty() && !validId(id)) + { + addError( + catalog_level, QStringLiteral("invalid-id"), path, + QStringLiteral("Invalid identifier") + ); + } + return id; + } + + QString optionalId( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level) + { + if (!object.contains(name)) + return {}; + return requiredId(object, name, path, catalog_level); + } + + bool validId(const QString& value) const + { + static const QRegularExpression pattern( + QStringLiteral( + "^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,253}[A-Za-z0-9])?$" + ) + ); + return pattern.match(value).hasMatch(); + } + + bool validRuntimeId(const QString& value) const + { + static const QRegularExpression pattern( + QStringLiteral("^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + ); + return pattern.match(value).hasMatch(); + } + + bool validMatrixId(const QString& value) const + { + return !value.isEmpty() && value.size() <= 64 + && !containsControl(value); + } + + QString requiredString( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level) + { + if (!object.contains(name) || !object.value(name).isString()) + { + addError( + catalog_level, QStringLiteral("required-string"), path, + QStringLiteral("Required member must be a string") + ); + return {}; + } + return object.value(name).toString(); + } + + QString optionalString( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level) + { + if (!object.contains(name)) + return {}; + auto const value = + requiredString(object, name, path, catalog_level); + if (value.isEmpty()) + { + addError( + catalog_level, QStringLiteral("empty-string"), path, + QStringLiteral("String must not be empty") + ); + } + return value; + } + + QDate optionalDate( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level) + { + if (!object.contains(name)) + return {}; + auto const text = + requiredString(object, name, path, catalog_level); + auto const date = QDate::fromString(text, Qt::ISODate); + if (!date.isValid() || date.toString(Qt::ISODate) != text) + { + addError( + catalog_level, QStringLiteral("invalid-date"), path, + QStringLiteral("Date must use YYYY-MM-DD ISO 8601 form") + ); + } + return date; + } + + QUrl absoluteUrl( + const QString& text, + const QString& path, + bool catalog_level) + { + if (text.isEmpty() + || text.size() > OicCatalogReader::maximum_url_length + || containsUrlWhitespaceOrControl(text) + || urlAuthorityContainsUserInfo(text)) + { + addError( + catalog_level, QStringLiteral("invalid-url"), path, + QStringLiteral("URL is empty, too long, or contains controls") + ); + return {}; + } + auto const url = QUrl(text, QUrl::StrictMode); + if (!url.isValid() || url.isRelative() + || !url.userName().isEmpty() || !url.password().isEmpty() + || url.hasFragment()) + { + addError( + catalog_level, QStringLiteral("invalid-url"), path, + QStringLiteral( + "URL must be absolute without user information " + "or a fragment" + ) + ); + return {}; + } + return url; + } + + QUrl httpUrl( + const QString& text, + const QString& path, + bool catalog_level) + { + auto const url = absoluteUrl(text, path, catalog_level); + if (url.isEmpty()) + return {}; + auto const scheme = url.scheme().toLower(); + if ((scheme != QLatin1String("http") + && scheme != QLatin1String("https")) + || url.host().isEmpty()) + { + addError( + catalog_level, QStringLiteral("invalid-http-url"), path, + QStringLiteral("URL must use HTTP or HTTPS and contain a host") + ); + return {}; + } + return url; + } + + double requiredPositiveNumber( + const QJsonObject& object, + const QString& name, + const QString& path) + { + auto const value = requiredNumber(object, name, path); + if (!(value > 0)) + { + sourceError( + QStringLiteral("positive-number"), path, + QStringLiteral("Number must be positive") + ); + } + return value; + } + + double requiredNumber( + const QJsonObject& object, + const QString& name, + const QString& path) + { + if (!object.contains(name)) + { + sourceError( + QStringLiteral("required-number"), path, + QStringLiteral("Required numeric member is missing") + ); + return 0; + } + return numberValue(object.value(name), path, false); + } + + bool finiteNumber(const QJsonValue& value) const + { + return value.isDouble() && std::isfinite(value.toDouble()); + } + + double numberValue( + const QJsonValue& value, + const QString& path, + bool catalog_level) + { + if (!finiteNumber(value)) + { + addError( + catalog_level, QStringLiteral("finite-number"), path, + QStringLiteral("Expected a finite number") + ); + return 0; + } + return value.toDouble(); + } + + qint64 requiredInteger( + const QJsonObject& object, + const QString& name, + const QString& path, + bool catalog_level, + qint64 minimum, + qint64 maximum) + { + if (!object.contains(name)) + { + addError( + catalog_level, QStringLiteral("required-integer"), path, + QStringLiteral("Required integer member is missing") + ); + return minimum; + } + return integerValue( + object.value(name), path, catalog_level, minimum, maximum + ); + } + + qint64 integerValue( + const QJsonValue& value, + const QString& path, + bool catalog_level, + qint64 minimum, + qint64 maximum) + { + auto const number = numberValue(value, path, catalog_level); + if (std::floor(number) != number + || number < double(minimum) || number > double(maximum) + || std::abs(number) > maximum_exact_integer) + { + addError( + catalog_level, QStringLiteral("integer-range"), path, + QStringLiteral( + "Integer is outside its permitted exact range" + ) + ); + return minimum; + } + return qint64(number); + } + + QJsonObject objectValue( + const QJsonValue& value, + const QString& path, + bool catalog_level) + { + if (!value.isObject()) + { + addError( + catalog_level, QStringLiteral("object-type"), path, + QStringLiteral("Expected an object") + ); + return {}; + } + return value.toObject(); + } + + int matrixIndex( + const OicSourceDefinition& source, + const QString& id) const + { + auto const& matrices = source.tile_matrix_set.matrix_set.matrices; + for (qsizetype index = 0; index < matrices.size(); ++index) + { + if (matrices.at(index).id == id) + return int(index); + } + return -1; + } + + bool hasSourceErrors(qsizetype start) const + { + for (qsizetype index = start; + index < result.diagnostics.size(); ++index) + { + if (result.diagnostics.at(index).kind + == OicDiagnosticKind::SourceError) + { + return true; + } + } + return false; + } + + void addUnsupported( + OicSourceDefinition& source, + const QString& capability, + const QString& path, + const QString& message) + { + if (!source.unsupported_capabilities.contains(capability)) + source.unsupported_capabilities.push_back(capability); + result.diagnostics.push_back({ + OicDiagnosticKind::UnsupportedSource, + QStringLiteral("unsupported-source"), + path, + message + QStringLiteral(": ") + capability, + current_source, + }); + } + + void addError( + bool catalog_level, + const QString& code, + const QString& path, + const QString& message) + { + if (catalog_level) + catalogError(code, path, message); + else + sourceError(code, path, message); + } + + void catalogError( + const QString& code, + const QString& path, + const QString& message) + { + result.diagnostics.push_back({ + OicDiagnosticKind::CatalogError, + code, + path, + message, + current_source, + }); + } + + void sourceError( + const QString& code, + const QString& path, + const QString& message) + { + result.diagnostics.push_back({ + OicDiagnosticKind::SourceError, + code, + path, + message, + current_source, + }); + } + + static const QSet& runtimeCapabilities() + { + static const QSet capabilities { + QStringLiteral("tile-matrix-set.ogc-2.0"), + QStringLiteral("tile-matrix-set.dyadic.v1"), + QStringLiteral("registration.translation2d.v1"), + }; + return capabilities; + } + + OicCatalogReadResult& result; + int current_source = -1; +}; + +} // namespace + +QString OicDiagnostic::displayText() const +{ + if (path.isEmpty()) + return message; + return QStringLiteral("%1: %2").arg(path, message); +} + +const OicResource* OicCatalog::resource(const QString& id) const noexcept +{ + for (auto const& candidate : resources) + { + if (candidate.id == id) + return &candidate; + } + return nullptr; +} + +bool OicCatalogReadResult::accepted() const noexcept +{ + return !hasCatalogErrors() && validSourceCount() > 0; +} + +bool OicCatalogReadResult::hasCatalogErrors() const noexcept +{ + for (auto const& diagnostic : diagnostics) + { + if (diagnostic.kind == OicDiagnosticKind::CatalogError) + return true; + } + return false; +} + +int OicCatalogReadResult::validSourceCount() const noexcept +{ + return int(std::count_if( + catalog.sources.begin(), catalog.sources.end(), + [](auto const& source) { return source.valid; } + )); +} + +int OicCatalogReadResult::supportedSourceCount() const noexcept +{ + return int(std::count_if( + catalog.sources.begin(), catalog.sources.end(), + [](auto const& source) { return source.supported; } + )); +} + +QVector OicCatalogReadResult::resolvedSources() const +{ + QVector sources; + if (hasCatalogErrors()) + return sources; + sources.reserve(supportedSourceCount()); + for (auto const& definition : catalog.sources) + { + if (definition.resolved_source) + sources.push_back(*definition.resolved_source); + } + return sources; +} + +QString OicCatalogReader::fileExtension() +{ + return QStringLiteral("oic"); +} + +OicCatalogReadResult OicCatalogReader::read(const QByteArray& bytes) +{ + OicCatalogReadResult result; + result.catalog.document_sha256 = sha256(bytes); + if (bytes.size() > maximum_document_size) + { + result.diagnostics.push_back({ + OicDiagnosticKind::CatalogError, + QStringLiteral("document-size"), + QStringLiteral("$"), + QStringLiteral("Catalog exceeds the %1-byte limit") + .arg(maximum_document_size), + -1, + }); + return result; + } + + JsonPreflight preflight(bytes); + if (!preflight.validate()) + { + result.diagnostics.push_back({ + OicDiagnosticKind::CatalogError, + QStringLiteral("json-preflight"), + QStringLiteral("$"), + preflight.errorString(), + -1, + }); + return result; + } + + QJsonParseError parse_error; + auto const document = QJsonDocument::fromJson(bytes, &parse_error); + if (parse_error.error != QJsonParseError::NoError + || !document.isObject()) + { + result.diagnostics.push_back({ + OicDiagnosticKind::CatalogError, + QStringLiteral("json-parse"), + QStringLiteral("$"), + parse_error.error == QJsonParseError::NoError + ? QStringLiteral("Catalog root must be an object") + : parse_error.errorString(), + -1, + }); + return result; + } + + CatalogValidator validator(result); + validator.validate(document.object(), bytes); + if (result.hasCatalogErrors()) + { + for (auto& source : result.catalog.sources) + source.resolved_source.reset(); + } + return result; +} + +} // namespace OpenOrienteering::imagery diff --git a/src/imagery/oic_catalog.h b/src/imagery/oic_catalog.h new file mode 100644 index 000000000..f8729da8b --- /dev/null +++ b/src/imagery/oic_catalog.h @@ -0,0 +1,238 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + */ + +#ifndef OPENORIENTEERING_IMAGERY_OIC_CATALOG_H +#define OPENORIENTEERING_IMAGERY_OIC_CATALOG_H + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/imagery_source.h" + +namespace OpenOrienteering::imagery { + +enum class OicDiagnosticKind +{ + CatalogError, + SourceError, + UnsupportedSource, +}; + +struct OicDiagnostic +{ + OicDiagnosticKind kind = OicDiagnosticKind::CatalogError; + QString code; + QString path; + QString message; + int source_index = -1; + + QString displayText() const; + bool operator==(const OicDiagnostic&) const = default; +}; + +struct OicPublisher +{ + QString name; + QUrl url; + QUrl contact_url; + QJsonObject original_object; + + bool operator==(const OicPublisher&) const = default; +}; + +struct OicResource +{ + QString id; + QString href; + QString media_type; + QByteArray sha256; + qint64 size = 0; + QJsonObject original_object; + + bool operator==(const OicResource&) const = default; +}; + +struct OicTileMatrixDefinition +{ + TileMatrix matrix; + double scale_denominator = 0; + QString corner_of_origin = QStringLiteral("topLeft"); + bool has_variable_matrix_widths = false; + QJsonObject original_object; + + bool operator==(const OicTileMatrixDefinition&) const = default; +}; + +struct OicTileMatrixSetDefinition +{ + TileMatrixSet matrix_set; + QStringList ordered_axes; + QVector matrices; + QJsonObject original_object; + bool dyadic_top_left = false; + + bool operator==(const OicTileMatrixSetDefinition&) const = default; +}; + +struct OicTileMatrixLimitDefinition +{ + QString tile_matrix; + qint64 min_row = 0; + qint64 max_row = -1; + qint64 min_column = 0; + qint64 max_column = -1; + + bool operator==(const OicTileMatrixLimitDefinition&) const = default; +}; + +enum class OicRegistrationKind +{ + None, + Translation2d, + Affine2d, + GridShift, +}; + +struct OicRegistrationDefinition +{ + OicRegistrationKind kind = OicRegistrationKind::None; + QString direction; + QString source_crs; + QString target_crs; + QString target_frame_id; + QString unit; + double dx = 0; + double dy = 0; + double xoff = 0; + double yoff = 0; + double s11 = 1; + double s12 = 0; + double s21 = 0; + double s22 = 1; + QString resource_id; + QString grid_domain; + QString grid_crs; + QString interpolation; + ImageryProvenance provenance; + QJsonObject original_object; + + bool operator==(const OicRegistrationDefinition&) const = default; +}; + +/** + * One source definition as published in an OIC catalog. + * + * valid means that the version-1 definition is structurally sound. + * supported means that this build can execute every required operation. + * resolved_source is present when both conditions hold and the containing + * catalog has no catalog-level errors. + */ +struct OicSourceDefinition +{ + ImageryMetadata metadata; + ImageryNotices notices; + QString type; + QVector tile_urls; + TileRowScheme row_scheme = TileRowScheme::Xyz; + QString media_type = QStringLiteral("image/png"); + QString min_tile_matrix; + QString max_tile_matrix; + QString tile_matrix_set_uri; + OicTileMatrixSetDefinition tile_matrix_set; + QVector tile_limit_definitions; + QVector tile_limits; + ImageryRequestPolicy request { QUrl {}, QVector {} }; + OicRegistrationDefinition registration; + QStringList required_capabilities; + QStringList unsupported_capabilities; + QJsonObject coverage; + QJsonObject extensions; + QJsonObject original_object; + QByteArray full_fingerprint; + QByteArray operational_fingerprint; + bool valid = false; + bool supported = false; + std::optional resolved_source; + + bool operator==(const OicSourceDefinition&) const = default; +}; + +struct OicCatalog +{ + QString format; + int version = 0; + QString id; + int revision = 0; + QString name; + QString description; + std::optional publisher; + QDate created; + QDate updated; + QString catalog_license; + QStringList required_capabilities; + QVector resources; + QVector sources; + QJsonObject extensions; + QJsonObject original_object; + QByteArray original_bytes; + QByteArray document_sha256; + + const OicResource* resource(const QString& id) const noexcept; + bool operator==(const OicCatalog&) const = default; +}; + +struct OicCatalogReadResult +{ + OicCatalog catalog; + QVector diagnostics; + + bool accepted() const noexcept; + bool hasCatalogErrors() const noexcept; + int validSourceCount() const noexcept; + int supportedSourceCount() const noexcept; + QVector resolvedSources() const; +}; + +/** + * Strict reader for OpenOrienteering Imagery Catalog version 1 JSON. + * + * A lexical preflight rejects duplicate members, malformed UTF-8, excessive + * nesting, nonfinite numbers, and negative zero before Qt's object parser can + * normalize or discard those distinctions. + */ +class OicCatalogReader +{ +public: + static constexpr qsizetype maximum_document_size = 10 * 1024 * 1024; + static constexpr int maximum_nesting_depth = 64; + static constexpr int maximum_string_length = 16 * 1024; + static constexpr int maximum_url_length = 8192; + static constexpr int maximum_sources = 1000; + static constexpr int maximum_resources = 1000; + static constexpr int maximum_tiles_per_source = 8; + static constexpr int maximum_tile_matrices = 64; + static constexpr int maximum_coverage_vertices = 10000; + static constexpr int maximum_empty_status_codes = 32; + + static QString fileExtension(); + static OicCatalogReadResult read(const QByteArray& bytes); +}; + +} // namespace OpenOrienteering::imagery + +#endif diff --git a/src/imagery/tile_matrix_set.cpp b/src/imagery/tile_matrix_set.cpp index b8f8ed579..acdafe64c 100644 --- a/src/imagery/tile_matrix_set.cpp +++ b/src/imagery/tile_matrix_set.cpp @@ -43,6 +43,24 @@ bool normalizedEpsg(const QString& crs) return pattern.match(crs).hasMatch(); } +bool hasFiniteExtent(const TileMatrix& matrix) +{ + auto const tile_width = + matrix.cell_size * double(matrix.tile_size.width()); + auto const tile_height = + matrix.cell_size * double(matrix.tile_size.height()); + auto const full_width = tile_width * double(matrix.matrix_width); + auto const full_height = tile_height * double(matrix.matrix_height); + auto const east = matrix.point_of_origin.x() + full_width; + auto const south = matrix.point_of_origin.y() - full_height; + return finite(tile_width) && tile_width > 0 + && finite(tile_height) && tile_height > 0 + && finite(full_width) && full_width > 0 + && finite(full_height) && full_height > 0 + && finite(east) && east > matrix.point_of_origin.x() + && finite(south) && south < matrix.point_of_origin.y(); +} + } // namespace bool CrsBounds::isValid() const noexcept @@ -132,6 +150,16 @@ bool TileMatrixSet::validateDyadicTopLeft(QString* error) const } if (matrix.matrix_width <= 0 || matrix.matrix_height <= 0) return fail(error, QStringLiteral("Tile matrix dimensions must be positive")); + if (!hasFiniteExtent(matrix)) + { + return fail( + error, + QStringLiteral( + "Tile matrix spans and full extent must be finite " + "and representable" + ) + ); + } if (index == 0) continue; diff --git a/src/imagery/tile_network_manager.cpp b/src/imagery/tile_network_manager.cpp index 4ebfb0e21..f48ee405e 100644 --- a/src/imagery/tile_network_manager.cpp +++ b/src/imagery/tile_network_manager.cpp @@ -12,14 +12,17 @@ #include "imagery/tile_network_manager.h" #include +#include #include #include #include #include #include +#include #include #include +#include #include #include #include @@ -115,9 +118,117 @@ bool isTransientHttpStatus(int status) QString hostKey(const QUrl& url) { auto const default_port = url.scheme() == QLatin1String("https") ? 443 : 80; - return url.scheme().toLower() + QLatin1String("://") - + QString::fromLatin1(QUrl::toAce(url.host()).toLower()) - + QLatin1Char(':') + QString::number(url.port(default_port)); + QHostAddress address; + auto const host = address.setAddress(url.host()) + ? address.toString().toLower() + : QString::fromLatin1( + QUrl::toAce(url.host()).toLower()); + QUrl origin; + origin.setScheme(url.scheme().toLower()); + origin.setHost(host); + origin.setPort(url.port(default_port)); + return origin.toString(QUrl::FullyEncoded); +} + +bool isPublicDestination(const QHostAddress& candidate) +{ + auto address = candidate; + bool has_ipv4 = false; + auto const ipv4 = address.toIPv4Address(&has_ipv4); + if (has_ipv4) + address = QHostAddress(ipv4); + + auto const in_subnet = [&address]( + const QHostAddress& prefix, + int length) { + return address.isInSubnet(prefix, length); + }; + if (address.protocol() + == QAbstractSocket::IPv4Protocol) + { + static const std::array non_public { + std::pair { QHostAddress(QStringLiteral("0.0.0.0")), 8 }, + std::pair { QHostAddress(QStringLiteral("10.0.0.0")), 8 }, + std::pair { QHostAddress(QStringLiteral("100.64.0.0")), 10 }, + std::pair { QHostAddress(QStringLiteral("127.0.0.0")), 8 }, + std::pair { QHostAddress(QStringLiteral("169.254.0.0")), 16 }, + std::pair { QHostAddress(QStringLiteral("172.16.0.0")), 12 }, + std::pair { QHostAddress(QStringLiteral("192.0.0.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("192.0.2.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("192.31.196.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("192.52.193.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("192.88.99.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("192.168.0.0")), 16 }, + std::pair { QHostAddress(QStringLiteral("192.175.48.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("198.18.0.0")), 15 }, + std::pair { QHostAddress(QStringLiteral("198.51.100.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("203.0.113.0")), 24 }, + std::pair { QHostAddress(QStringLiteral("224.0.0.0")), 4 }, + std::pair { QHostAddress(QStringLiteral("240.0.0.0")), 4 }, + }; + if (std::ranges::any_of( + non_public, + [&in_subnet](auto const& subnet) { + return in_subnet( + subnet.first, subnet.second); + })) + return false; + } + else if (address.protocol() + == QAbstractSocket::IPv6Protocol) + { + // The well-known NAT64 prefix is globally reachable, but its embedded + // IPv4 destination must independently pass this same policy. + static const QHostAddress nat64( + QStringLiteral("64:ff9b::")); + if (address.isInSubnet(nat64, 96)) + { + auto const bytes = address.toIPv6Address(); + auto const embedded = + (quint32(bytes[12]) << 24) + | (quint32(bytes[13]) << 16) + | (quint32(bytes[14]) << 8) + | quint32(bytes[15]); + return isPublicDestination( + QHostAddress(embedded)); + } + + static const QHostAddress global_unicast( + QStringLiteral("2000::")); + if (!address.isInSubnet(global_unicast, 3)) + return false; + static const std::array non_public { + std::pair { QHostAddress(QStringLiteral("2001::")), 23 }, + std::pair { QHostAddress(QStringLiteral("2001:db8::")), 32 }, + std::pair { QHostAddress(QStringLiteral("2002::")), 16 }, + std::pair { QHostAddress(QStringLiteral("2620:4f:8000::")), 48 }, + std::pair { QHostAddress(QStringLiteral("3fff::")), 20 }, + }; + if (std::ranges::any_of( + non_public, + [&in_subnet](auto const& subnet) { + return in_subnet( + subnet.first, subnet.second); + })) + return false; + } + else + { + return false; + } + + // QHostAddress::isGlobal() intentionally includes RFC 1918, IPv6 ULA, + // and deprecated site-local ranges. Online imagery treats all of those as + // permission-gated destinations, while isGlobal() rejects other reserved + // and documentation-only ranges. + return address.isGlobal() + && !address.isNull() + && !address.isLoopback() + && !address.isLinkLocal() + && !address.isMulticast() + && !address.isBroadcast() + && !address.isPrivateUse() + && !address.isSiteLocal(); } bool isLocalHostname(QString host) @@ -133,8 +244,12 @@ bool isLocalHostname(QString host) QString validateHttpUrl( const QUrl& url, - const TileNetworkManager::Config& config) + const TileNetworkManager::Config& config, + QUrl* private_network_rejected_url = nullptr, + bool enforce_private_network_policy = true) { + if (private_network_rejected_url) + private_network_rejected_url->clear(); if (!url.isValid() || url.isRelative()) return TileNetworkManager::tr("The imagery URL is invalid."); auto const scheme = url.scheme().toLower(); @@ -149,9 +264,13 @@ QString validateHttpUrl( auto const encoded = url.toEncoded(); if (encoded.contains('\r') || encoded.contains('\n') || encoded.contains('\0')) return TileNetworkManager::tr("The imagery URL contains unsafe control characters."); + if (encoded.size() > 16 * 1024) + return TileNetworkManager::tr("The imagery URL is too long."); auto const port = url.port(); if (port == 0 || port > 65535) return TileNetworkManager::tr("The imagery URL has an invalid port."); + if (!enforce_private_network_policy) + return {}; if (config.allow_private_networks || config.approved_private_origins.contains(hostKey(url))) @@ -160,30 +279,60 @@ QString validateHttpUrl( QHostAddress address; if (address.setAddress(url.host())) { - if (address.isNull() || address.isLoopback() || address.isLinkLocal() - || address.isMulticast() || address.isPrivateUse()) + if (!isPublicDestination(address)) { + if (private_network_rejected_url) + *private_network_rejected_url = url; return TileNetworkManager::tr( "Private, local, and link-local imagery hosts require explicit permission."); } } else if (isLocalHostname(url.host())) { + if (private_network_rejected_url) + *private_network_rejected_url = url; return TileNetworkManager::tr( "Private, local, and link-local imagery hosts require explicit permission."); } return {}; } +QByteArray negativeCacheKey(const TileNetworkRequest& request) +{ + QCryptographicHash digest(QCryptographicHash::Sha256); + QByteArray representation; + representation.append(static_cast( + request.payload_kind)); + representation.append('\n'); + representation.append( + request.url.toEncoded(QUrl::FullyEncoded)); + representation.append('\n'); + representation.append(request.referer.toUtf8()); + representation.append('\n'); + auto statuses = request.empty_http_status_codes; + std::sort(statuses.begin(), statuses.end()); + for (auto const status : std::as_const(statuses)) + { + representation.append(QByteArray::number(status)); + representation.append(','); + } + digest.addData(representation); + return digest.result(); +} + QString validateRequest( const TileNetworkRequest& request, - const TileNetworkManager::Config& config) + const TileNetworkManager::Config& config, + QUrl* private_network_rejected_url = nullptr) { + if (private_network_rejected_url) + private_network_rejected_url->clear(); if (request.client_id == 0) return TileNetworkManager::tr("The imagery request has no client identity."); if (!std::isfinite(request.distance_priority)) return TileNetworkManager::tr("The imagery request priority is invalid."); - if (auto const error = validateHttpUrl(request.url, config); + if (auto const error = validateHttpUrl( + request.url, config, private_network_rejected_url); !error.isEmpty()) { return error; @@ -191,7 +340,8 @@ QString validateRequest( if (!request.referer.isEmpty()) { auto const referer = QUrl(request.referer); - if (auto const error = validateHttpUrl(referer, config); + if (auto const error = validateHttpUrl( + referer, config, nullptr, false); !error.isEmpty()) { return TileNetworkManager::tr("The imagery Referer is invalid: %1").arg(error); @@ -206,6 +356,24 @@ QString validateRequest( return TileNetworkManager::tr("The empty-tile HTTP status list is invalid."); statuses.insert(status); } + auto const valid_header = [](const QByteArray& value) { + return value.size() <= 8192 + && !value.contains('\r') + && !value.contains('\n') + && !value.contains('\0'); + }; + if (!valid_header(request.if_none_match) + || !valid_header(request.if_modified_since)) + { + return TileNetworkManager::tr( + "The imagery conditional request headers are invalid."); + } + if (request.max_response_bytes < 0 + || request.max_response_bytes > config.max_response_bytes) + { + return TileNetworkManager::tr( + "The imagery response limit is invalid."); + } return {}; } @@ -226,10 +394,12 @@ class TileNetworkManager::Worker final : public QObject public: Worker(Config config, QPointer facade, - std::atomic_bool* offline) - : config_(std::move(config)) - , facade_(std::move(facade)) - , offline_(offline) + std::atomic_bool* offline, + std::atomic* network_mode_generation) + : config_(std::move(config)) + , facade_(std::move(facade)) + , offline_(offline) + , network_mode_generation_(network_mode_generation) {} void initialize() @@ -239,6 +409,15 @@ class TileNetworkManager::Worker final : public QObject wake_timer_ = new QTimer(this); wake_timer_->setSingleShot(true); connect(wake_timer_, &QTimer::timeout, this, [this] { dispatch(); }); + auto* state_prune_timer = new QTimer(this); + state_prune_timer->setInterval(std::chrono::minutes(1)); + connect(state_prune_timer, &QTimer::timeout, this, [this] { + pruneNegativeCache(); + pruneClientHistory(); + pruneHostBackoff(); + pruneDestinationCache(); + }); + state_prune_timer->start(); network_ = new QNetworkAccessManager(this); network_->setCookieJar(new RejectingCookieJar(network_)); @@ -286,7 +465,14 @@ class TileNetworkManager::Worker final : public QObject active_replies_.clear(); active_hosts_.clear(); active_clients_.clear(); + client_entry_counts_.clear(); + client_last_service_.clear(); + host_not_before_.clear(); + negative_cache_.clear(); + destination_cache_.clear(); active_total_ = 0; + outstanding_results_ = 0; + outstanding_response_bytes_ = 0; if (network_) { delete network_; @@ -300,26 +486,47 @@ class TileNetworkManager::Worker final : public QObject if (shutting_down_) return; - if (auto const error = validateRequest(request, config_); + QUrl private_network_rejected_url; + if (auto const error = validateRequest( + request, config_, &private_network_rejected_url); !error.isEmpty()) { TileNetworkResult result; result.outcome = TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = + !private_network_rejected_url.isEmpty(); + result.private_network_rejected_url = + private_network_rejected_url; result.error_string = error; - deliver(token, request, std::move(result)); + deliver( + token, + request, + std::move(result), + false, + 0, + DeliveryGuard {}); return; } auto url = request.url; - auto const negative = negative_cache_.constFind(url); - if (negative != negative_cache_.cend()) + auto const negative_key = negativeCacheKey(request); + auto negative = negative_cache_.find(negative_key); + if (request.payload_kind == NetworkPayloadKind::TileImage + && negative != negative_cache_.end()) { - if (*negative > now()) + if (negative->expires > now()) { + negative->last_access = nextStateAccess(); TileNetworkResult result; result.outcome = TileNetworkResult::Outcome::EmptyTile; result.from_cache = true; - deliver(token, request, std::move(result)); + deliver( + token, + request, + std::move(result), + false, + 0, + DeliveryGuard {}); return; } negative_cache_.erase(negative); @@ -336,10 +543,16 @@ class TileNetworkManager::Worker final : public QObject || pending_for_client >= config_.max_pending_per_client) { TileNetworkResult result; - result.outcome = TileNetworkResult::Outcome::Rejected; + result.outcome = TileNetworkResult::Outcome::Busy; result.error_string = TileNetworkManager::tr( "The imagery request queue is full."); - deliver(token, request, std::move(result)); + deliver( + token, + request, + std::move(result), + false, + 0, + DeliveryGuard {}); return; } @@ -349,6 +562,7 @@ class TileNetworkManager::Worker final : public QObject entry->current_url = std::move(url); entry->sequence = next_sequence_++; entries_.insert(token, entry); + ++client_entry_counts_[entry->request.client_id]; queueAfterDestinationCheck(entry); } @@ -388,6 +602,109 @@ class TileNetworkManager::Worker final : public QObject cancel(token); } + void setOfflineMode(bool offline) + { + Q_ASSERT(QThread::currentThread() == thread()); + if (!offline) + { + dispatch(); + return; + } + + QVector> active; + active.reserve(active_replies_.size()); + for (auto const& entry : std::as_const(active_replies_)) + active.push_back(entry); + for (auto const& entry : std::as_const(active)) + { + if (!entries_.contains(entry->token) || !entry->reply + || entry->cache_only_request) + continue; + entry->offline_abort = true; + entry->body.clear(); + } + for (auto const& entry : std::as_const(active)) + { + if (entry->reply && !entry->cache_only_request) + entry->reply->abort(); + } + + QVector> destination_waiters; + for (auto const& waiters : std::as_const(destination_waiters_)) + destination_waiters.append(waiters); + for (auto const lookup_id : std::as_const(destination_lookups_)) + QHostInfo::abortHostLookup(lookup_id); + destination_lookups_.clear(); + destination_waiters_.clear(); + for (auto const& entry : std::as_const(destination_waiters)) + { + if (!entries_.contains(entry->token)) + continue; + entry->validated_origin.clear(); + entry->destination_valid_until = 0; + enqueue(entry); + } + } + + void setPrivateOriginApproved( + QString origin, + bool approved) + { + Q_ASSERT(QThread::currentThread() == thread()); + if (approved) + config_.approved_private_origins.insert(origin); + else + config_.approved_private_origins.remove(origin); + destination_cache_.remove(origin); + if (approved || config_.allow_private_networks) + return; + + QVector> affected; + affected.reserve(entries_.size()); + for (auto const& entry : std::as_const(entries_)) + { + if (hostKey(entry->request.url) == origin + || hostKey(entry->current_url) == origin) + { + affected.push_back(entry); + } + } + + for (auto const& entry : std::as_const(affected)) + { + if (!entries_.contains(entry->token)) + continue; + entry->permission_revoked = true; + entry->permission_revoked_url = + hostKey(entry->current_url) == origin + ? entry->current_url + : entry->request.url; + entry->body.clear(); + } + for (auto const& entry : std::as_const(affected)) + { + if (!entries_.contains(entry->token)) + continue; + if (entry->reply) + { + entry->reply->abort(); + continue; + } + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = true; + result.private_network_rejected_url = + entry->permission_revoked_url; + result.private_network_permission_revoked = true; + result.error_string = TileNetworkManager::tr( + "Permission for the private imagery origin was revoked."); + finish(entry, std::move(result)); + } + + pruneDestinationWaiters(); + dispatch(); + } + private: struct Entry { @@ -406,6 +723,27 @@ class TileNetworkManager::Worker final : public QObject bool absolute_timeout = false; bool authentication_rejected = false; bool received_metadata = false; + bool offline_abort = false; + bool permission_revoked = false; + QUrl permission_revoked_url; + bool cache_only_request = false; + bool result_slot_reserved = false; + qint64 reserved_response_bytes = 0; + quint64 network_mode_generation = 0; + QString private_permission_origin; + QUrl private_permission_url; + quint64 private_permission_generation = 0; + QString validated_origin; + qint64 destination_valid_until = 0; + }; + + struct DeliveryGuard + { + quint64 network_mode_generation = 0; + bool cache_only_request = false; + QString private_permission_origin; + QUrl private_permission_url; + quint64 private_permission_generation = 0; }; struct DestinationDecision @@ -414,6 +752,13 @@ class TileNetworkManager::Worker final : public QObject bool transient_failure = false; QString error; qint64 expires = 0; + quint64 last_access = 0; + }; + + struct NegativeCacheEntry + { + qint64 expires = 0; + quint64 last_access = 0; }; qint64 now() const @@ -421,6 +766,266 @@ class TileNetworkManager::Worker final : public QObject return clock_.elapsed(); } + qint64 responseLimit(const Entry& entry) const + { + return entry.request.max_response_bytes > 0 + ? entry.request.max_response_bytes + : config_.max_response_bytes; + } + + quint64 privateOriginGeneration(const QString& origin) const + { + auto const facade = facade_; + return facade ? facade->privateOriginGeneration(origin) : 0; + } + + TileNetworkManager::NetworkModeSnapshot networkModeSnapshot() const + { + auto const facade = facade_; + if (facade) + return facade->networkModeSnapshot(); + return { + offline_->load(), + network_mode_generation_->load(), + }; + } + + bool privatePermissionChanged(const Entry& entry) const + { + return entry.private_permission_generation != 0 + && privateOriginGeneration(entry.private_permission_origin) + != entry.private_permission_generation; + } + + bool networkModeChanged(const Entry& entry) const + { + return !entry.cache_only_request + && entry.network_mode_generation != 0 + && entry.network_mode_generation + != networkModeSnapshot().generation; + } + + DeliveryGuard deliveryGuard(const Entry& entry) const + { + return { + entry.network_mode_generation, + entry.cache_only_request, + entry.private_permission_origin, + entry.private_permission_url, + entry.private_permission_generation, + }; + } + + quint64 nextStateAccess() + { + if (next_state_access_ == std::numeric_limits::max()) + { + for (auto& entry : negative_cache_) + entry.last_access = 0; + for (auto& entry : destination_cache_) + entry.last_access = 0; + next_state_access_ = 1; + } + return next_state_access_++; + } + + void pruneNegativeCache() + { + auto const current = now(); + negative_cache_.removeIf([current]( + QHash::iterator it) { + return it->expires <= current; + }); + while (negative_cache_.size() > config_.max_negative_cache_entries) + { + auto victim = negative_cache_.end(); + for (auto it = negative_cache_.begin(); + it != negative_cache_.end(); ++it) + { + if (victim == negative_cache_.end() + || it->last_access < victim->last_access + || (it->last_access == victim->last_access + && it.key() < victim.key())) + { + victim = it; + } + } + if (victim == negative_cache_.end()) + break; + negative_cache_.erase(victim); + } + } + + void pruneClientHistory() + { + while (client_last_service_.size() + > config_.max_client_history_entries) + { + auto victim = client_last_service_.end(); + for (auto it = client_last_service_.begin(); + it != client_last_service_.end(); ++it) + { + if (victim == client_last_service_.end() + || it.value() < victim.value() + || (it.value() == victim.value() + && it.key() < victim.key())) + { + victim = it; + } + } + if (victim == client_last_service_.end()) + break; + client_last_service_.erase(victim); + } + } + + void pruneHostBackoff() + { + auto const current = now(); + host_not_before_.removeIf([current]( + QHash::iterator it) { + return it.value() <= current; + }); + while (host_not_before_.size() > config_.max_host_backoff_entries) + { + auto victim = host_not_before_.end(); + for (auto it = host_not_before_.begin(); + it != host_not_before_.end(); ++it) + { + if (victim == host_not_before_.end() + || it.value() < victim.value() + || (it.value() == victim.value() + && it.key() < victim.key())) + { + victim = it; + } + } + if (victim == host_not_before_.end()) + break; + host_not_before_.erase(victim); + } + } + + void pruneDestinationCache() + { + auto const current = now(); + destination_cache_.removeIf([current]( + QHash::iterator it) { + return it->expires <= current; + }); + while (destination_cache_.size() + > config_.max_destination_cache_entries) + { + auto victim = destination_cache_.end(); + for (auto it = destination_cache_.begin(); + it != destination_cache_.end(); ++it) + { + if (victim == destination_cache_.end() + || it->last_access < victim->last_access + || (it->last_access == victim->last_access + && it.key() < victim.key())) + { + victim = it; + } + } + if (victim == destination_cache_.end()) + break; + destination_cache_.erase(victim); + } + } + + void pruneDestinationWaiters() + { + for (auto it = destination_waiters_.begin(); + it != destination_waiters_.end();) + { + it.value().removeIf([this](auto const& entry) { + return !entries_.contains(entry->token); + }); + if (it.value().isEmpty()) + { + if (auto const lookup = destination_lookups_.take(it.key()); + lookup != 0) + { + QHostInfo::abortHostLookup(lookup); + } + it = destination_waiters_.erase(it); + } + else + { + ++it; + } + } + } + + bool destinationNeedsPreflight( + const Entry& entry, + bool offline) const + { + // AlwaysCache never opens a connection, so offline cache reads must not + // depend on DNS being available. URL syntax/private-literal policy was + // still enforced by validateRequest(). + if (offline) + return false; + auto const origin = hostKey(entry.current_url); + if (config_.allow_private_networks + || config_.approved_private_origins.contains(origin)) + { + return false; + } + QHostAddress literal; + return !literal.setAddress(entry.current_url.host()); + } + + bool destinationNeedsPreflight(const Entry& entry) const + { + return destinationNeedsPreflight(entry, offline_->load()); + } + + bool hasResultCapacity(const Entry& entry) const + { + if (entry.result_slot_reserved) + return true; + auto const response_bytes = responseLimit(entry); + return outstanding_results_ < config_.max_outstanding_results + && response_bytes + <= config_.max_outstanding_response_bytes + - outstanding_response_bytes_; + } + + void reserveResultCapacity(const std::shared_ptr& entry) + { + if (entry->result_slot_reserved) + return; + Q_ASSERT(hasResultCapacity(*entry)); + entry->result_slot_reserved = true; + entry->reserved_response_bytes = responseLimit(*entry); + ++outstanding_results_; + outstanding_response_bytes_ += entry->reserved_response_bytes; + } + + void releaseResultCapacity(const std::shared_ptr& entry) + { + if (!entry->result_slot_reserved) + return; + entry->result_slot_reserved = false; + --outstanding_results_; + outstanding_response_bytes_ -= entry->reserved_response_bytes; + entry->reserved_response_bytes = 0; + Q_ASSERT(outstanding_results_ >= 0); + Q_ASSERT(outstanding_response_bytes_ >= 0); + } + + void acknowledgeResult(qint64 reserved_response_bytes) + { + Q_ASSERT(QThread::currentThread() == thread()); + --outstanding_results_; + outstanding_response_bytes_ -= reserved_response_bytes; + Q_ASSERT(outstanding_results_ >= 0); + Q_ASSERT(outstanding_response_bytes_ >= 0); + dispatch(); + } + void enqueue(const std::shared_ptr& entry) { if (!entries_.contains(entry->token) || entry->cancelled) @@ -440,6 +1045,10 @@ class TileNetworkManager::Worker final : public QObject result.outcome = decision.transient_failure ? TileNetworkResult::Outcome::TransientError : TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = + !decision.transient_failure; + if (result.private_network_rejected) + result.private_network_rejected_url = entry->current_url; result.error_string = decision.error; finish(entry, std::move(result)); } @@ -447,31 +1056,32 @@ class TileNetworkManager::Worker final : public QObject void queueAfterDestinationCheck(const std::shared_ptr& entry) { auto const origin = hostKey(entry->current_url); - if (config_.allow_private_networks - || config_.approved_private_origins.contains(origin)) + if (offline_->load()) { + // This is cache-only admission, not a reusable network decision. If + // online mode resumes before dispatch, start() will preflight again. + entry->validated_origin.clear(); + entry->destination_valid_until = 0; enqueue(entry); return; } - - QHostAddress literal; - if (literal.setAddress(entry->current_url.host())) + if (!destinationNeedsPreflight(*entry)) { - // validateHttpUrl() already rejected non-global literals. + entry->validated_origin = origin; + entry->destination_valid_until = + std::numeric_limits::max(); enqueue(entry); return; } - auto const cached = destination_cache_.constFind(origin); - if (cached != destination_cache_.cend() && cached->expires > now()) + auto cached = destination_cache_.find(origin); + if (cached != destination_cache_.end() && cached->expires > now()) { - if (cached->allowed) - enqueue(entry); - else - destinationFailure(entry, *cached); + cached->last_access = nextStateAccess(); + destinationFailure(entry, *cached); return; } - if (cached != destination_cache_.cend()) + if (cached != destination_cache_.end()) destination_cache_.erase(cached); auto& waiters = destination_waiters_[origin]; @@ -479,9 +1089,15 @@ class TileNetworkManager::Worker final : public QObject if (destination_lookups_.contains(origin)) return; - auto const lookup_id = QHostInfo::lookupHost( + auto const lookup_id = std::make_shared(0); + *lookup_id = QHostInfo::lookupHost( entry->current_url.host(), this, - [this, origin](QHostInfo info) { + [this, origin, lookup_id](QHostInfo info) { + auto const current_lookup = + destination_lookups_.constFind(origin); + if (current_lookup == destination_lookups_.cend() + || *current_lookup != *lookup_id) + return; destination_lookups_.remove(origin); auto waiters = destination_waiters_.take(origin); DestinationDecision decision; @@ -494,9 +1110,11 @@ class TileNetworkManager::Worker final : public QObject } else { - decision.allowed = std::ranges::all_of( - info.addresses(), - [](auto const& address) { return address.isGlobal(); }); + decision.allowed = std::ranges::all_of( + info.addresses(), + [](auto const& address) { + return isPublicDestination(address); + }); if (!decision.allowed) { decision.error = TileNetworkManager::tr( @@ -504,22 +1122,39 @@ class TileNetworkManager::Worker final : public QObject } decision.expires = now() + 5 * 60 * 1000; } - destination_cache_.insert(origin, decision); + auto const origin_is_approved = + config_.allow_private_networks + || config_.approved_private_origins.contains(origin); + if (!decision.allowed && !origin_is_approved) + { + decision.last_access = nextStateAccess(); + destination_cache_.insert(origin, decision); + pruneDestinationCache(); + } for (auto const& waiter : std::as_const(waiters)) { - if (decision.allowed) + if (decision.allowed || origin_is_approved) + { + waiter->validated_origin = origin; + // Keep the DNS decision close to QNAM's own resolution. + // Long scheduler waits force another preflight. + waiter->destination_valid_until = now() + 1000; enqueue(waiter); + } else destinationFailure(waiter, decision); } dispatch(); }); - destination_lookups_.insert(origin, lookup_id); + destination_lookups_.insert(origin, *lookup_id); } bool eligible(const std::shared_ptr& entry, qint64 current) const { - if (entry->cancelled || entry->not_before > current) + if (entry->cancelled || entry->permission_revoked + || entry->not_before > current) + return false; + if (!hasResultCapacity(*entry)) return false; if (active_clients_.value(entry->request.client_id) >= config_.max_active_per_client) @@ -593,6 +1228,7 @@ class TileNetworkManager::Worker final : public QObject break; auto entry = queue_.takeAt(*selected); client_last_service_[entry->request.client_id] = next_service_++; + pruneClientHistory(); start(entry); } scheduleWake(); @@ -622,7 +1258,52 @@ class TileNetworkManager::Worker final : public QObject void start(const std::shared_ptr& entry) { - if (offline_->load() && !entry->request.referer.isEmpty()) + auto const origin = hostKey(entry->current_url); + auto const network_mode = networkModeSnapshot(); + auto const needs_preflight = + destinationNeedsPreflight(*entry, network_mode.offline); + if (needs_preflight + && (entry->validated_origin != origin + || entry->destination_valid_until <= now())) + { + queueAfterDestinationCheck(entry); + return; + } + if (needs_preflight) + entry->destination_valid_until = 0; + + auto const cache_only = network_mode.offline; + entry->cache_only_request = cache_only; + entry->network_mode_generation = + network_mode.generation; + entry->private_permission_origin.clear(); + entry->private_permission_url.clear(); + entry->private_permission_generation = 0; + if (!config_.allow_private_networks + && config_.approved_private_origins.contains(origin)) + { + auto const permission_generation = + privateOriginGeneration(origin); + if (permission_generation == 0) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = true; + result.private_network_rejected_url = entry->current_url; + result.private_network_permission_revoked = true; + result.error_string = TileNetworkManager::tr( + "Permission for the private imagery origin was revoked."); + finish(entry, std::move(result)); + return; + } + entry->private_permission_origin = origin; + entry->private_permission_url = entry->current_url; + entry->private_permission_generation = + permission_generation; + } + reserveResultCapacity(entry); + + if (cache_only && !entry->request.referer.isEmpty()) { TileNetworkResult result; result.outcome = TileNetworkResult::Outcome::OfflineMiss; @@ -652,7 +1333,36 @@ class TileNetworkManager::Worker final : public QObject QByteArrayLiteral("Referer"), entry->request.referer.toUtf8()); } - request.setRawHeader(QByteArrayLiteral("Accept"), QByteArrayLiteral("image/*")); + if (entry->request.payload_kind + == NetworkPayloadKind::JsonDocument) + { + request.setRawHeader( + QByteArrayLiteral("Accept"), + QByteArrayLiteral( + "application/json, application/*+json;q=0.9, " + "application/octet-stream;q=0.5")); + if (entry->redirects == 0) + { + if (!entry->request.if_none_match.isEmpty()) + { + request.setRawHeader( + QByteArrayLiteral("If-None-Match"), + entry->request.if_none_match); + } + if (!entry->request.if_modified_since.isEmpty()) + { + request.setRawHeader( + QByteArrayLiteral("If-Modified-Since"), + entry->request.if_modified_since); + } + } + } + else + { + request.setRawHeader( + QByteArrayLiteral("Accept"), + QByteArrayLiteral("image/*")); + } request.setPriority( entry->request.priority == TileRequestPriority::Coverage ? QNetworkRequest::HighPriority @@ -677,15 +1387,21 @@ class TileNetworkManager::Worker final : public QObject QNetworkRequest::CacheLoadControlAttribute, referer_dependent ? QNetworkRequest::AlwaysNetwork - : offline_->load() + : cache_only ? QNetworkRequest::AlwaysCache - : QNetworkRequest::PreferNetwork); + : entry->request.payload_kind + == NetworkPayloadKind::JsonDocument + ? QNetworkRequest::AlwaysNetwork + : QNetworkRequest::PreferNetwork); request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, - !referer_dependent); + !referer_dependent + && entry->request.payload_kind + == NetworkPayloadKind::TileImage); request.setMaximumRedirectsAllowed(config_.max_redirects); request.setTransferTimeout(config_.transfer_timeout); - request.setDecompressedSafetyCheckThreshold(config_.max_response_bytes); + auto const response_limit = responseLimit(*entry); + request.setDecompressedSafetyCheckThreshold(response_limit); auto* reply = network_->get(request); entry->reply = reply; @@ -696,17 +1412,20 @@ class TileNetworkManager::Worker final : public QObject entry->received_metadata = true; auto const length = entry->reply->header( QNetworkRequest::ContentLengthHeader).toLongLong(); - if (length > config_.max_response_bytes) + auto const response_limit = responseLimit(*entry); + if (length > response_limit) { entry->too_large = true; entry->reply->abort(); } }); connect(reply, &QIODevice::readyRead, this, [this, entry] { - if (!entry->reply || entry->too_large) + if (!entry->reply || entry->too_large || entry->cancelled + || entry->offline_abort || entry->permission_revoked) return; auto chunk = entry->reply->readAll(); - if (chunk.size() > config_.max_response_bytes - entry->body.size()) + auto const response_limit = responseLimit(*entry); + if (chunk.size() > response_limit - entry->body.size()) { entry->too_large = true; entry->body.clear(); @@ -762,10 +1481,13 @@ class TileNetworkManager::Worker final : public QObject if (!entry->reply) return; auto* reply = entry->reply.data(); - if (!entry->too_large) + if (!entry->too_large && !entry->cancelled + && !entry->offline_abort && !entry->permission_revoked + && reply->isReadable()) { auto tail = reply->readAll(); - if (tail.size() > config_.max_response_bytes - entry->body.size()) + auto const response_limit = responseLimit(*entry); + if (tail.size() > response_limit - entry->body.size()) { entry->too_large = true; entry->body.clear(); @@ -787,12 +1509,76 @@ class TileNetworkManager::Worker final : public QObject auto const from_cache = reply->attribute( QNetworkRequest::SourceIsFromCacheAttribute).toBool(); auto const retry_after = reply->rawHeader(QByteArrayLiteral("Retry-After")); + auto const etag = reply->rawHeader(QByteArrayLiteral("ETag")); + auto const last_modified = + reply->rawHeader(QByteArrayLiteral("Last-Modified")); + auto const final_url = reply->url(); releaseActive(entry); + auto const add_response_metadata = + [&](TileNetworkResult& result) { + result.final_url = final_url; + result.etag = etag; + result.last_modified = last_modified; + }; + if (entry->cancelled) { TileNetworkResult result; result.outcome = TileNetworkResult::Outcome::Cancelled; + add_response_metadata(result); + finish(entry, std::move(result)); + dispatch(); + return; + } + if (entry->permission_revoked) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = true; + result.private_network_rejected_url = + entry->permission_revoked_url; + result.private_network_permission_revoked = true; + result.error_string = TileNetworkManager::tr( + "Permission for the private imagery origin was revoked."); + add_response_metadata(result); + finish(entry, std::move(result)); + dispatch(); + return; + } + if (privatePermissionChanged(*entry)) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = true; + result.private_network_rejected_url = + entry->private_permission_url; + result.private_network_permission_revoked = true; + result.error_string = TileNetworkManager::tr( + "Permission for the private imagery origin changed while the request was active."); + add_response_metadata(result); + finish(entry, std::move(result)); + dispatch(); + return; + } + if (networkModeChanged(*entry)) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::OfflineMiss; + result.error_string = TileNetworkManager::tr( + "The imagery request was stopped because offline mode was enabled."); + add_response_metadata(result); + finish(entry, std::move(result)); + dispatch(); + return; + } + if (entry->offline_abort) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::OfflineMiss; + result.error_string = TileNetworkManager::tr( + "The imagery request was stopped because offline mode was enabled."); + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -803,7 +1589,11 @@ class TileNetworkManager::Worker final : public QObject result.outcome = TileNetworkResult::Outcome::PermanentError; result.error_string = TileNetworkManager::tr( "The imagery response exceeded the %1 MB safety limit.") - .arg(config_.max_response_bytes / (1024 * 1024)); + .arg((entry->request.max_response_bytes > 0 + ? entry->request.max_response_bytes + : config_.max_response_bytes) + / (1024 * 1024)); + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -814,6 +1604,7 @@ class TileNetworkManager::Worker final : public QObject result.outcome = TileNetworkResult::Outcome::PermanentError; result.error_string = TileNetworkManager::tr( "Imagery sources requiring HTTP authentication are not supported."); + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -823,7 +1614,9 @@ class TileNetworkManager::Worker final : public QObject { auto const next = entry->current_url.resolved(redirect) .adjusted(QUrl::RemoveFragment); - auto error = validateHttpUrl(next, config_); + QUrl private_network_rejected_url; + auto error = validateHttpUrl( + next, config_, &private_network_rejected_url); if (error.isEmpty() && entry->current_url.scheme() == QLatin1String("https") && next.scheme() == QLatin1String("http") @@ -840,9 +1633,16 @@ class TileNetworkManager::Worker final : public QObject if (!error.isEmpty()) { TileNetworkResult result; - result.outcome = TileNetworkResult::Outcome::PermanentError; + result.outcome = private_network_rejected_url.isEmpty() + ? TileNetworkResult::Outcome::PermanentError + : TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = + !private_network_rejected_url.isEmpty(); + result.private_network_rejected_url = + private_network_rejected_url; result.http_status = status; result.error_string = error; + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -850,20 +1650,41 @@ class TileNetworkManager::Worker final : public QObject ++entry->redirects; entry->current_url = next; entry->not_before = now(); + entry->body.clear(); + releaseResultCapacity(entry); queueAfterDestinationCheck(entry); + dispatch(); + return; + } + + if (entry->request.payload_kind + == NetworkPayloadKind::JsonDocument + && status == 304) + { + TileNetworkResult result; + result.outcome = TileNetworkResult::Outcome::NotModified; + result.http_status = status; + result.content_type = content_type; + result.from_cache = from_cache; + add_response_metadata(result); + finish(entry, std::move(result)); + dispatch(); return; } if (entry->request.empty_http_status_codes.contains(status)) { negative_cache_.insert( - entry->request.url.adjusted(QUrl::RemoveFragment), - now() + config_.negative_cache_ttl_ms); + negativeCacheKey(entry->request), + { now() + config_.negative_cache_ttl_ms, + nextStateAccess() }); + pruneNegativeCache(); TileNetworkResult result; result.outcome = TileNetworkResult::Outcome::EmptyTile; result.http_status = status; result.content_type = content_type; result.from_cache = from_cache; + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -877,6 +1698,7 @@ class TileNetworkManager::Worker final : public QObject result.http_status = status; result.content_type = content_type; result.from_cache = from_cache; + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -891,6 +1713,7 @@ class TileNetworkManager::Worker final : public QObject result.http_status = status; result.error_string = TileNetworkManager::tr( "The imagery tile is not available in the offline cache."); + add_response_metadata(result); finish(entry, std::move(result)); dispatch(); return; @@ -907,10 +1730,14 @@ class TileNetworkManager::Worker final : public QObject auto const host = hostKey(entry->current_url); host_not_before_[host] = std::max( host_not_before_.value(host), now() + delay); + pruneHostBackoff(); } ++entry->retries; entry->not_before = now() + delay; - enqueue(entry); + entry->body.clear(); + releaseResultCapacity(entry); + queueAfterDestinationCheck(entry); + dispatch(); return; } @@ -921,6 +1748,7 @@ class TileNetworkManager::Worker final : public QObject result.http_status = status; result.content_type = content_type; result.from_cache = from_cache; + add_response_metadata(result); result.error_string = entry->absolute_timeout ? TileNetworkManager::tr("The imagery request timed out.") : network_error_string; @@ -974,10 +1802,36 @@ class TileNetworkManager::Worker final : public QObject { eraseQueued(entry); entries_.remove(entry->token); - deliver(entry->token, entry->request, std::move(result)); + pruneDestinationWaiters(); + auto client_count = client_entry_counts_.find( + entry->request.client_id); + if (client_count != client_entry_counts_.end() + && --(*client_count) <= 0) + { + client_entry_counts_.erase(client_count); + client_last_service_.remove(entry->request.client_id); + } + auto const reserved = entry->result_slot_reserved; + auto const reserved_response_bytes = entry->reserved_response_bytes; + auto const guard = deliveryGuard(*entry); + entry->result_slot_reserved = false; + entry->reserved_response_bytes = 0; + deliver( + entry->token, + entry->request, + std::move(result), + reserved, + reserved_response_bytes, + guard); } - void deliver(Token token, const TileNetworkRequest& request, TileNetworkResult result) + void deliver( + Token token, + const TileNetworkRequest& request, + TileNetworkResult result, + bool reserved, + qint64 reserved_response_bytes, + DeliveryGuard guard) { if (shutting_down_) return; @@ -986,12 +1840,66 @@ class TileNetworkManager::Worker final : public QObject result.user_data = request.user_data; auto facade = facade_; if (!facade) + { + if (reserved) + acknowledgeResult(reserved_response_bytes); return; + } + QPointer worker(this); QMetaObject::invokeMethod( facade, - [facade, token, result = std::move(result)] { + [facade, worker, token, result = std::move(result), + reserved, reserved_response_bytes, + guard = std::move(guard)]() mutable { if (facade) + { + if (result.outcome + != TileNetworkResult::Outcome::Cancelled + && result.outcome + != TileNetworkResult::Outcome::Rejected) + { + auto const private_permission_changed = + guard.private_permission_generation != 0 + && facade->privateOriginGeneration( + guard.private_permission_origin) + != guard.private_permission_generation; + if (private_permission_changed) + { + result.body.clear(); + result.outcome = + TileNetworkResult::Outcome::Rejected; + result.private_network_rejected = true; + result.private_network_rejected_url = + guard.private_permission_url; + result.private_network_permission_revoked = true; + result.error_string = TileNetworkManager::tr( + "Permission for the private imagery origin changed before the response was delivered."); + } + else if (!guard.cache_only_request + && guard.network_mode_generation != 0 + && facade->networkModeSnapshot().generation + != guard.network_mode_generation) + { + result.body.clear(); + result.outcome = + TileNetworkResult::Outcome::OfflineMiss; + result.error_string = TileNetworkManager::tr( + "The imagery request was stopped because offline mode was enabled."); + } + } emit facade->finished(token, result); + } + if (reserved && worker) + { + QMetaObject::invokeMethod( + worker, + [worker, reserved_response_bytes] { + if (worker) + worker->acknowledgeResult( + reserved_response_bytes); + }, + Qt::QueuedConnection); + } }, Qt::QueuedConnection); } @@ -999,6 +1907,7 @@ class TileNetworkManager::Worker final : public QObject Config config_; QPointer facade_; std::atomic_bool* offline_ = nullptr; + std::atomic* network_mode_generation_ = nullptr; QElapsedTimer clock_; QNetworkAccessManager* network_ = nullptr; QTimer* wake_timer_ = nullptr; @@ -1006,17 +1915,21 @@ class TileNetworkManager::Worker final : public QObject quint64 next_sequence_ = 1; quint64 next_service_ = 1; int active_total_ = 0; + int outstanding_results_ = 0; + qint64 outstanding_response_bytes_ = 0; QHash> entries_; QVector> queue_; QHash> active_replies_; QHash active_hosts_; QHash active_clients_; + QHash client_entry_counts_; QHash client_last_service_; QHash host_not_before_; - QHash negative_cache_; + QHash negative_cache_; QHash destination_cache_; QHash destination_lookups_; QHash>> destination_waiters_; + quint64 next_state_access_ = 1; }; TileNetworkManager::TileNetworkManager(QObject* parent) @@ -1040,15 +1953,39 @@ TileNetworkManager::TileNetworkManager(Config config, QObject* parent) config_.max_active_per_client = std::max(1, config_.max_active_per_client); config_.max_pending_total = std::max(1, config_.max_pending_total); config_.max_pending_per_client = std::max(1, config_.max_pending_per_client); + config_.max_negative_cache_entries = + std::max(0, config_.max_negative_cache_entries); + config_.max_client_history_entries = + std::max(0, config_.max_client_history_entries); + config_.max_host_backoff_entries = + std::max(0, config_.max_host_backoff_entries); + config_.max_destination_cache_entries = + std::max(0, config_.max_destination_cache_entries); config_.max_redirects = std::max(0, config_.max_redirects); config_.max_retries = std::max(0, config_.max_retries); + config_.negative_cache_ttl_ms = + std::max(0, config_.negative_cache_ttl_ms); config_.max_response_bytes = std::max(1, config_.max_response_bytes); + config_.max_outstanding_results = + std::max(1, config_.max_outstanding_results); + config_.max_outstanding_response_bytes = std::max( + config_.max_response_bytes, + config_.max_outstanding_response_bytes); config_.disk_cache_bytes = std::max(0, config_.disk_cache_bytes); + approved_private_origins_ = config_.approved_private_origins; + for (auto const& origin : std::as_const(approved_private_origins_)) + { + auto const generation = next_private_origin_generation_++; + if (generation == 0) + qFatal("Imagery private-origin generation space exhausted"); + private_origin_generations_.insert(origin, generation); + } qRegisterMetaType(); network_thread_ = new QThread(this); network_thread_->setObjectName(QStringLiteral("Mapper imagery network")); - worker_ = new Worker(config_, this, &offline_); + worker_ = new Worker( + config_, this, &offline_, &network_mode_generation_); worker_->moveToThread(network_thread_); network_thread_->start(); QMetaObject::invokeMethod( @@ -1093,6 +2030,12 @@ QString TileNetworkManager::canonicalOrigin(const QUrl& url) return hostKey(url); } +bool TileNetworkManager::isPublicDestinationAddress( + const QHostAddress& address) +{ + return isPublicDestination(address); +} + TileNetworkManager::Token TileNetworkManager::submit(TileNetworkRequest request) { auto const token = next_token_.fetch_add(1); @@ -1128,7 +2071,54 @@ void TileNetworkManager::cancelClient( void TileNetworkManager::setOfflineMode(bool offline) { - offline_.store(offline); + { + QMutexLocker lock(&network_mode_mutex_); + if (offline_.load() == offline) + return; + auto const advance_generation = [this] { + if (network_mode_generation_.fetch_add(1) + == std::numeric_limits::max()) + { + qFatal("Imagery network-mode generation space exhausted"); + } + }; + // A worker which observes the transition without taking this mutex must + // see either the old online mode or a cache-only state. The generation + // makes every already-active online request stale. + if (offline) + { + offline_.store(true); + advance_generation(); + } + else + { + advance_generation(); + offline_.store(false); + } + // Queue under the same mutex so concurrent callers cannot reorder the + // worker's transition callbacks after publishing facade state. + QMetaObject::invokeMethod( + worker_, + [worker = worker_, offline] { + worker->setOfflineMode(offline); + }, + Qt::QueuedConnection); + } + if (QThread::currentThread() == thread()) + { + emit offlineModeChanged(offline); + } + else + { + QPointer self(this); + QMetaObject::invokeMethod( + this, + [self, offline] { + if (self) + emit self->offlineModeChanged(offline); + }, + Qt::QueuedConnection); + } } bool TileNetworkManager::offlineMode() const noexcept @@ -1136,4 +2126,95 @@ bool TileNetworkManager::offlineMode() const noexcept return offline_.load(); } +TileNetworkManager::NetworkModeSnapshot +TileNetworkManager::networkModeSnapshot() const +{ + QMutexLocker lock(&network_mode_mutex_); + return { + offline_.load(), + network_mode_generation_.load(), + }; +} + +bool TileNetworkManager::approvePrivateOrigin( + const QUrl& url) +{ + auto const scheme = url.scheme().toLower(); + if (!url.isValid() || url.isRelative() || url.host().isEmpty() + || !url.userInfo().isEmpty() + || (scheme != QLatin1String("http") + && scheme != QLatin1String("https"))) + return false; + auto const origin = canonicalOrigin(url); + { + QMutexLocker lock(&permissions_mutex_); + if (approved_private_origins_.contains(origin)) + return true; + approved_private_origins_.insert(origin); + auto const generation = next_private_origin_generation_++; + if (generation == 0) + qFatal("Imagery private-origin generation space exhausted"); + private_origin_generations_.insert(origin, generation); + // Preserve mutation order when approvals are changed concurrently. + QMetaObject::invokeMethod( + worker_, + [worker = worker_, origin] { + worker->setPrivateOriginApproved(origin, true); + }, + Qt::QueuedConnection); + } + QPointer self(this); + QMetaObject::invokeMethod( + this, + [self, origin] { + if (self) + emit self->privateOriginApprovalChanged(origin, true); + }, + Qt::QueuedConnection); + return true; +} + +bool TileNetworkManager::revokePrivateOrigin( + const QUrl& url) +{ + auto const origin = canonicalOrigin(url); + { + QMutexLocker lock(&permissions_mutex_); + if (!approved_private_origins_.remove(origin)) + return false; + private_origin_generations_.remove(origin); + // Preserve mutation order when approvals are changed concurrently. + QMetaObject::invokeMethod( + worker_, + [worker = worker_, origin] { + worker->setPrivateOriginApproved(origin, false); + }, + Qt::QueuedConnection); + } + QPointer self(this); + QMetaObject::invokeMethod( + this, + [self, origin] { + if (self) + emit self->privateOriginApprovalChanged(origin, false); + }, + Qt::QueuedConnection); + return true; +} + +bool TileNetworkManager::isPrivateOriginApproved( + const QUrl& url) const +{ + QMutexLocker lock(&permissions_mutex_); + return approved_private_origins_.contains( + canonicalOrigin(url)); +} + +quint64 TileNetworkManager::privateOriginGeneration( + const QString& origin) const +{ + QMutexLocker lock(&permissions_mutex_); + return private_origin_generations_.value(origin); +} + } // namespace OpenOrienteering::imagery diff --git a/src/imagery/tile_network_manager.h b/src/imagery/tile_network_manager.h index a6854f907..cfcc3c94a 100644 --- a/src/imagery/tile_network_manager.h +++ b/src/imagery/tile_network_manager.h @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include #include @@ -24,6 +26,7 @@ #include class QThread; +class QHostAddress; namespace OpenOrienteering::imagery { @@ -34,6 +37,12 @@ enum class TileRequestPriority Background, }; +enum class NetworkPayloadKind +{ + TileImage, + JsonDocument, +}; + struct TileNetworkRequest { QUrl url; @@ -41,9 +50,14 @@ struct TileNetworkRequest quint64 generation = 0; quint64 user_data = 0; TileRequestPriority priority = TileRequestPriority::Visible; + NetworkPayloadKind payload_kind = NetworkPayloadKind::TileImage; double distance_priority = 0; QString referer; QVector empty_http_status_codes = { 204, 404 }; + QByteArray if_none_match; + QByteArray if_modified_since; + /** Zero uses Config::max_response_bytes. */ + qint64 max_response_bytes = 0; }; struct TileNetworkResult @@ -51,11 +65,15 @@ struct TileNetworkResult enum class Outcome { Success, + NotModified, EmptyTile, Cancelled, OfflineMiss, + /** A bounded scheduler queue is full; the request may be retried later. */ + Busy, TransientError, PermanentError, + /** The request is invalid or disallowed by network policy. */ Rejected, }; @@ -63,15 +81,24 @@ struct TileNetworkResult QByteArray body; QString content_type; QString error_string; + QUrl final_url; + QByteArray etag; + QByteArray last_modified; int http_status = 0; bool from_cache = false; + /** Rejected because a network destination was private/non-global. */ + bool private_network_rejected = false; + /** Exact URL whose origin failed the private-network policy. */ + QUrl private_network_rejected_url; + /** Rejection was caused by an explicit user revocation, not discovery. */ + bool private_network_permission_revoked = false; quint64 client_id = 0; quint64 generation = 0; quint64 user_data = 0; }; /** - * Application-scoped, bounded HTTP scheduler for tiled imagery. + * Application-scoped, bounded HTTP scheduler for online imagery resources. * * A single QNetworkAccessManager and QNetworkDiskCache live on a dedicated * event-loop thread. Public methods are thread-safe. Results are emitted on @@ -81,10 +108,21 @@ struct TileNetworkResult * each client's coverage, visible, then background work is ordered by distance. * The manager enforces total, per-host, per-client, and pending limits. * - * Only HTTP(S) URLs without embedded credentials are accepted. Cookies and - * HTTP authentication are disabled. Redirects are validated explicitly and - * HTTPS downgrades are rejected by default. Response bodies and time are - * bounded before image decoding. + * Tile images and OIC catalogs share one connection pool and disk cache while + * retaining resource-specific Accept, cache, conditional-request, and body + * limit behavior. Only HTTP(S) URLs without embedded credentials are accepted. + * Cookies and HTTP authentication are disabled. Redirects are validated + * explicitly and HTTPS downgrades are rejected by default. Response bodies and + * time are bounded before parsing or image decoding. Active requests reserve + * bounded result-delivery slots and body bytes until the application thread + * consumes their completion, preventing a blocked UI event loop from growing + * an unbounded cross-thread body backlog. + * + * Unapproved hostnames re-enter DNS preflight for retries and redirects, and + * successful decisions expire after one second of scheduler waiting. + * QNetworkAccessManager performs the eventual connection resolution itself, + * so this substantially narrows but cannot entirely remove the DNS-rebinding + * interval without bypassing Qt's TLS and HTTP cache stack. */ class TileNetworkManager final : public QObject { @@ -100,14 +138,27 @@ Q_OBJECT qint64 disk_cache_bytes = qint64(128) << 20; int max_active_total = 6; int max_active_per_client = 4; + /** Active requests plus results awaiting application-thread delivery. */ + int max_outstanding_results = 16; + /** Reserved response limits for those outstanding requests/results. */ + qint64 max_outstanding_response_bytes = qint64(48) << 20; #else qint64 disk_cache_bytes = qint64(512) << 20; int max_active_total = 12; int max_active_per_client = 6; + /** Active requests plus results awaiting application-thread delivery. */ + int max_outstanding_results = 32; + /** Reserved response limits for those outstanding requests/results. */ + qint64 max_outstanding_response_bytes = qint64(192) << 20; #endif int max_active_per_host = 6; int max_pending_total = 2048; int max_pending_per_client = 256; + /** Bounds for long-lived scheduler/cache bookkeeping. Zero retains none. */ + int max_negative_cache_entries = 4096; + int max_client_history_entries = 4096; + int max_host_backoff_entries = 1024; + int max_destination_cache_entries = 1024; int max_redirects = 5; int max_retries = 2; int retry_base_delay_ms = 400; @@ -134,6 +185,8 @@ Q_OBJECT static TileNetworkManager& instance(); static quint64 nextClientId(); static QString canonicalOrigin(const QUrl& url); + /** True only for destinations which may be contacted without approval. */ + static bool isPublicDestinationAddress(const QHostAddress& address); Token submit(TileNetworkRequest request); void cancel(Token token); @@ -143,20 +196,40 @@ Q_OBJECT void setOfflineMode(bool offline); bool offlineMode() const noexcept; + bool approvePrivateOrigin(const QUrl& url); + bool revokePrivateOrigin(const QUrl& url); + bool isPrivateOriginApproved(const QUrl& url) const; signals: + void offlineModeChanged(bool offline); + void privateOriginApprovalChanged( + const QString& origin, + bool approved); void finished( OpenOrienteering::imagery::TileNetworkManager::Token token, const OpenOrienteering::imagery::TileNetworkResult& result); private: class Worker; + struct NetworkModeSnapshot + { + bool offline = false; + quint64 generation = 0; + }; + NetworkModeSnapshot networkModeSnapshot() const; + quint64 privateOriginGeneration(const QString& origin) const; Config config_; QThread* network_thread_ = nullptr; Worker* worker_ = nullptr; std::atomic next_token_ { 1 }; std::atomic_bool offline_ { false }; + std::atomic network_mode_generation_ { 1 }; + mutable QMutex network_mode_mutex_; + mutable QMutex permissions_mutex_; + QSet approved_private_origins_; + QHash private_origin_generations_; + quint64 next_private_origin_generation_ = 1; }; } // namespace OpenOrienteering::imagery diff --git a/src/render/overlay_scene.cpp b/src/render/overlay_scene.cpp index 17b82bf70..9ec93c5a0 100644 --- a/src/render/overlay_scene.cpp +++ b/src/render/overlay_scene.cpp @@ -399,10 +399,11 @@ std::shared_ptr OverlaySceneBuilder::image(const QImage& source } auto data = std::make_shared(ImageData { std::uint32_t(converted.width()), - std::uint32_t(converted.height()), - std::uint32_t(row_bytes), - std::move(bytes), - }); + std::uint32_t(converted.height()), + std::uint32_t(row_bytes), + std::move(bytes), + {}, + }); if (images_.size() >= 128) images_.clear(); images_.emplace(stable_key, data); diff --git a/src/render/render_ir.h b/src/render/render_ir.h index a1b942393..76fa98666 100644 --- a/src/render/render_ir.h +++ b/src/render/render_ir.h @@ -168,6 +168,8 @@ struct ImageData std::uint32_t height = 0; std::uint32_t bytes_per_row = 0; std::shared_ptr> rgba8; + /** Optional accounting lease retained for the pixel buffer's lifetime. */ + std::shared_ptr memory_keepalive; }; struct PushTransform diff --git a/src/render/template_layer_planner.cpp b/src/render/template_layer_planner.cpp index fd603090e..5db0bda7a 100644 --- a/src/render/template_layer_planner.cpp +++ b/src/render/template_layer_planner.cpp @@ -52,6 +52,7 @@ struct TileKey Rect source; Transform image_to_scene; bool direct_to_map = false; + bool provisional = false; bool operator==(const TileKey& other) const { @@ -66,7 +67,8 @@ struct TileKey && image_to_scene.m22 == other.image_to_scene.m22 && image_to_scene.dx == other.image_to_scene.dx && image_to_scene.dy == other.image_to_scene.dy - && direct_to_map == other.direct_to_map; + && direct_to_map == other.direct_to_map + && provisional == other.provisional; } }; @@ -120,25 +122,97 @@ bool imageIsOpaque(const QImage& source) return true; } -std::shared_ptr snapshotImage(const QImage& source) +std::shared_ptr snapshotImage( + const QImage& source, + const RasterMemoryReserver& reserve_memory = {}, + bool shrink_memory = true) { - auto const image = source.convertToFormat(QImage::Format_RGBA8888); - if (image.isNull()) + auto const retained_bytes = + qint64(source.width()) * source.height() * 4; + if (retained_bytes <= 0 + || retained_bytes > std::numeric_limits::max() / 2) + return {}; + auto const direct_format = + source.format() == QImage::Format_RGBA8888 + || source.format() + == QImage::Format_RGBA8888_Premultiplied; + auto memory = reserve_memory + ? reserve_memory( + direct_format + ? retained_bytes + : 2 * retained_bytes) + : std::shared_ptr {}; + if (reserve_memory && !memory) return {}; auto bytes = std::make_shared>(); - auto const row_bytes = std::size_t(image.width()) * 4; - bytes->reserve(row_bytes * std::size_t(image.height())); - for (int y = 0; y < image.height(); ++y) + auto const width = source.width(); + auto const height = source.height(); + auto const row_bytes = std::size_t(width) * 4; + bytes->resize(row_bytes * std::size_t(height)); + if (source.format() == QImage::Format_RGBA8888) { - auto const* row = image.constScanLine(y); - bytes->insert(bytes->end(), row, row + row_bytes); + for (int y = 0; y < height; ++y) + { + std::copy_n( + source.constScanLine(y), + row_bytes, + bytes->data() + + std::size_t(y) * row_bytes); + } + } + else if (source.format() + == QImage::Format_RGBA8888_Premultiplied) + { + for (int y = 0; y < height; ++y) + { + auto const* input = source.constScanLine(y); + auto* output = + bytes->data() + std::size_t(y) * row_bytes; + for (int x = 0; x < width; ++x) + { + auto const alpha = int(input[4 * x + 3]); + output[4 * x + 3] = + std::uint8_t(alpha); + for (int channel = 0; channel < 3; ++channel) + { + output[4 * x + channel] = + alpha == 0 + ? 0 + : std::uint8_t(std::min( + 255, + (int(input[4 * x + channel]) + * 255 + alpha / 2) + / alpha)); + } + } + } } + else + { + auto image = + source.convertToFormat( + QImage::Format_RGBA8888); + if (image.isNull()) + return {}; + for (int y = 0; y < height; ++y) + { + std::copy_n( + image.constScanLine(y), + row_bytes, + bytes->data() + + std::size_t(y) * row_bytes); + } + image = {}; + } + if (memory && shrink_memory) + memory->shrinkTo(retained_bytes); return std::make_shared(ImageData { - std::uint32_t(image.width()), - std::uint32_t(image.height()), + std::uint32_t(width), + std::uint32_t(height), std::uint32_t(row_bytes), std::move(bytes), + std::move(memory), }); } @@ -146,6 +220,8 @@ struct SourceTile { TileKey key; QImage image; + std::shared_ptr pixel_memory; + RasterMemoryReserver reserve_render_memory; }; struct RasterMosaic @@ -184,10 +260,19 @@ RasterMosaic transparentMosaic(const std::vector& tiles) return {}; } + #ifdef Q_OS_ANDROID + constexpr double max_dimension = 4096; + constexpr double max_pixels = 4.0 * 1024 * 1024; + #else constexpr double max_dimension = 8192; - constexpr double max_pixels = 64.0 * 1024 * 1024; + constexpr double max_pixels = 16.0 * 1024 * 1024; + #endif auto desired_width = std::ceil(target_width * density_x); auto desired_height = std::ceil(target_height * density_y); + if (!std::isfinite(desired_width) + || !std::isfinite(desired_height) + || desired_width <= 0 || desired_height <= 0) + return {}; auto reduction = std::min({ 1.0, max_dimension / desired_width, @@ -196,6 +281,29 @@ RasterMosaic transparentMosaic(const std::vector& tiles) }); auto const width = std::max(1, int(std::floor(desired_width * reduction))); auto const height = std::max(1, int(std::floor(desired_height * reduction))); + auto const retained_bytes = qint64(width) * height * 4; + if (retained_bytes <= 0 + || retained_bytes > std::numeric_limits::max() / 2) + return {}; + + RasterMemoryReserver reserve_memory; + for (auto const& tile : tiles) + { + if (tile.reserve_render_memory) + { + reserve_memory = tile.reserve_render_memory; + break; + } + } + std::shared_ptr peak_memory; + if (reserve_memory) + { + // The premultiplied mosaic and immutable straight-RGBA snapshot coexist + // during conversion. Admit both before either large allocation exists. + peak_memory = reserve_memory(2 * retained_bytes); + if (!peak_memory) + return {}; + } QImage mosaic(width, height, QImage::Format_RGBA8888_Premultiplied); if (mosaic.isNull()) @@ -218,8 +326,23 @@ RasterMosaic transparentMosaic(const std::vector& tiles) } painter.end(); + RasterMemoryReserver admitted_snapshot; + if (peak_memory) + { + admitted_snapshot = + [memory = peak_memory, retained_bytes](qint64 bytes) { + return bytes > 0 && bytes <= retained_bytes + ? memory + : std::shared_ptr {}; + }; + } + auto snapshot = snapshotImage( + mosaic, admitted_snapshot, false); + mosaic = {}; + if (peak_memory) + peak_memory->shrinkTo(retained_bytes); return { - snapshotImage(mosaic), + std::move(snapshot), { left, top, target_width, target_height }, }; } @@ -441,30 +564,47 @@ class TemplateLayerPlanner::Impl ? fromQTransform(tile.image_to_map) : Transform {}, tile.has_image_to_map, + tile.provisional, }; full_key.tiles.push_back(tile_key); - source_images.push_back({ std::move(tile_key), tile.image }); + source_images.push_back({ + std::move(tile_key), + tile.image, + tile.pixel_memory, + tile.reserve_render_memory, + }); } - auto const found = layers_.find(&source); + auto found = layers_.find(&source); if (found != layers_.end() && found->second.key == full_key) return found->second.scene; + if (!on_screen && found != layers_.end()) + { + // Exact multi-page output does not need the previous page as a + // fallback. Release its immutable pixel snapshots before admitting + // the next page so both pages never count against the global raster + // budget at once. + layers_.erase(found); + found = layers_.end(); + } if (source_images.empty()) { layers_.erase(&source); return {}; } - auto const has_transparency = std::ranges::any_of( - source_images, - [this](auto const& tile) { - return !isOpaque(tile.key.image, tile.image); - } - ); auto const has_direct_tiles = std::ranges::any_of( source_images, [](auto const& tile) { return tile.key.direct_to_map; } ); + auto const has_transparency = + !has_direct_tiles + && std::ranges::any_of( + source_images, + [this](auto const& tile) { + return !isOpaque(tile.key.image, tile.image); + } + ); if (has_transparency && !has_direct_tiles) { if (on_screen && result.newly_resident_images >= max_new_images_per_frame) @@ -497,7 +637,12 @@ class TemplateLayerPlanner::Impl tiles.reserve(source_images.size()); for (auto const& tile : source_images) { - auto image = imageFor(tile.key.image, tile.image, result, on_screen); + auto image = imageFor( + tile.key.image, + tile.image, + tile.reserve_render_memory, + result, + on_screen); if (!image) { result.complete = false; @@ -513,6 +658,11 @@ class TemplateLayerPlanner::Impl layers_.erase(&source); return {}; } + std::stable_sort( + tiles.begin(), tiles.end(), + [](auto const& lhs, auto const& rhs) { + return lhs.first.provisional && !rhs.first.provisional; + }); if (next_revision_ == std::numeric_limits::max()) qFatal("Raster layer revision space exhausted"); @@ -546,6 +696,7 @@ class TemplateLayerPlanner::Impl std::shared_ptr imageFor(const ImageKey& key, const QImage& image, + const RasterMemoryReserver& reserve_memory, TemplateLayerPlan& result, bool on_screen) { @@ -558,7 +709,7 @@ class TemplateLayerPlanner::Impl if (on_screen && result.newly_resident_images >= max_new_images_per_frame) return {}; - auto snapshot = snapshotImage(image); + auto snapshot = snapshotImage(image, reserve_memory); if (!snapshot) return {}; images_[key] = snapshot; @@ -579,6 +730,11 @@ TemplateLayerPlanner::TemplateLayerPlanner() TemplateLayerPlanner::~TemplateLayerPlanner() = default; +void TemplateLayerPlanner::clear() +{ + impl_ = std::make_unique(); +} + TemplateLayerPlan TemplateLayerPlanner::plan(const Map& map, const MapView& view, Rect visible_map_rect, diff --git a/src/render/template_layer_planner.h b/src/render/template_layer_planner.h index 2749c8d0f..6d435d61e 100644 --- a/src/render/template_layer_planner.h +++ b/src/render/template_layer_planner.h @@ -44,6 +44,9 @@ class TemplateLayerPlanner TemplateLayerPlanner(const TemplateLayerPlanner&) = delete; TemplateLayerPlanner& operator=(const TemplateLayerPlanner&) = delete; + /** Releases all retained template scenes and immutable image snapshots. */ + void clear(); + TemplateLayerPlan plan(const Map& map, const MapView& view, Rect visible_map_rect, diff --git a/src/templates/online_raster_template.cpp b/src/templates/online_raster_template.cpp new file mode 100644 index 000000000..04cfcfff8 --- /dev/null +++ b/src/templates/online_raster_template.cpp @@ -0,0 +1,3890 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +#include "templates/online_raster_template.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/georeferencing.h" +#include "core/map.h" +#include "core/map_coord.h" +#include "gui/util_gui.h" +#include "imagery/imagery_network_permissions.h" +#include "util/transformation.h" +#include "util/util.h" + +namespace OpenOrienteering { + +namespace { + +// One payload is valid. Retaining a second preserves a useful malformed-file +// diagnostic without allowing an attacker to accumulate an unbounded number +// of near-limit strings in one template record. +constexpr qsizetype maximum_embedded_snapshot_payloads = 2; +constexpr qsizetype maximum_snapshot_version_size = 32; +constexpr qsizetype maximum_snapshot_checksum_size = 64; +constexpr qsizetype maximum_snapshot_encoding_size = 32; + +struct BoundedElementText +{ + QString text; + bool too_large = false; +}; + +BoundedElementText readBoundedElementText( + QXmlStreamReader& xml, + qsizetype maximum_size) +{ + BoundedElementText result; + while (!xml.atEnd()) + { + auto const token = xml.readNext(); + if (token == QXmlStreamReader::EndElement) + break; + if (token == QXmlStreamReader::StartElement) + { + xml.raiseError( + OnlineRasterTemplate::tr( + "The embedded imagery source contains an unexpected element.")); + break; + } + if (token != QXmlStreamReader::Characters + && token != QXmlStreamReader::EntityReference) + continue; + auto const text = xml.text(); + if (result.too_large + || text.size() > maximum_size - result.text.size()) + { + result.too_large = true; + continue; + } + result.text.append(text); + } + return result; +} + +int cacheCostKiB(const QImage& image) +{ + auto const bytes = qsizetype(image.bytesPerLine()) * image.height(); + return int(std::min(std::numeric_limits::max(), (bytes + 1023) / 1024)); +} + +bool imageIsOpaque( + const QImage& source, + const RasterResourceManager::CancellationToken& cancellation, + const std::shared_ptr& source_cancelled) +{ + if (!source.hasAlphaChannel()) + return true; + auto const image = source.convertToFormat(QImage::Format_RGBA8888); + for (int y = 0; y < image.height(); ++y) + { + if (cancellation.isCancelled() + || source_cancelled->load(std::memory_order_relaxed)) + { + return false; + } + auto const* row = image.constScanLine(y); + for (int x = 0; x < image.width(); ++x) + { + if (row[4 * x + 3] != 255) + return false; + } + } + return true; +} + +QImage addGutter(QImage source) +{ + if (source.isNull()) + return {}; + if (source.format() != QImage::Format_RGBA8888_Premultiplied) + source = source.convertToFormat(QImage::Format_RGBA8888_Premultiplied); + auto const& core = source; + QImage padded(core.width() + 2, core.height() + 2, QImage::Format_RGBA8888); + if (padded.isNull()) + return {}; + + QPainter painter(&padded); + painter.setCompositionMode(QPainter::CompositionMode_Source); + painter.drawImage(1, 1, core); + painter.drawImage(QRect(0, 1, 1, core.height()), core, QRect(0, 0, 1, core.height())); + painter.drawImage(QRect(core.width() + 1, 1, 1, core.height()), core, + QRect(core.width() - 1, 0, 1, core.height())); + painter.drawImage(QRect(1, 0, core.width(), 1), core, QRect(0, 0, core.width(), 1)); + painter.drawImage(QRect(1, core.height() + 1, core.width(), 1), core, + QRect(0, core.height() - 1, core.width(), 1)); + painter.drawImage(QPoint(0, 0), core, QRect(0, 0, 1, 1)); + painter.drawImage(QPoint(core.width() + 1, 0), core, QRect(core.width() - 1, 0, 1, 1)); + painter.drawImage(QPoint(0, core.height() + 1), core, QRect(0, core.height() - 1, 1, 1)); + painter.drawImage(QPoint(core.width() + 1, core.height() + 1), core, + QRect(core.width() - 1, core.height() - 1, 1, 1)); + painter.end(); + return padded; +} + +double pointDistance(const QPointF& first, const QPointF& second) +{ + return std::hypot(first.x() - second.x(), first.y() - second.y()); +} + +QRectF boundsOf(std::initializer_list points) +{ + QRectF result; + for (auto const& point : points) + rectIncludeSafe(result, point); + return result; +} + +qint64 boundedFloor(double value) +{ + if (!std::isfinite(value)) + return 0; + if (value <= double(std::numeric_limits::min())) + return std::numeric_limits::min(); + if (value >= double(std::numeric_limits::max())) + return std::numeric_limits::max(); + return qint64(std::floor(value)); +} + +quint64 mixSignature(quint64 value, quint64 input) +{ + value ^= input + 0x9e3779b97f4a7c15ULL + (value << 6) + (value >> 2); + return value; +} + +std::optional rgbaImageBytes(qint64 width, qint64 height) +{ + if (width <= 0 || height <= 0 || width > std::numeric_limits::max() / height + || width * height > std::numeric_limits::max() / 4) + { + return std::nullopt; + } + return width * height * 4; +} + +std::shared_ptr> sharedDecodeBytesInFlight() +{ + static auto counter = std::make_shared>(0); + return counter; +} + +std::shared_ptr> sharedRetainedRasterBytes() +{ + static auto counter = std::make_shared>(0); + return counter; +} + +QSet& liveOnlineRasterTemplates() +{ + static QSet templates; + return templates; +} + +quint64 nextRetainedAccess() +{ + static quint64 next = 1; + if (next == std::numeric_limits::max()) + qFatal("Online imagery retained-access space exhausted"); + return next++; +} + +} // namespace + +bool OnlineRasterTemplate::TileWindow::isEmpty() const noexcept +{ + return zoom < 0 || min_column > max_column || min_row > max_row; +} + +bool OnlineRasterTemplate::TileWindow::intersects(const TileWindow& other) const noexcept +{ + return zoom == other.zoom && !isEmpty() && !other.isEmpty() && min_column <= other.max_column + && max_column >= other.min_column && min_row <= other.max_row + && max_row >= other.min_row; +} + +bool OnlineRasterTemplate::TileWindow::contains(const TileWindow& other) const noexcept +{ + return zoom == other.zoom && !isEmpty() && !other.isEmpty() && min_column <= other.min_column + && max_column >= other.max_column && min_row <= other.min_row + && max_row >= other.max_row; +} + +qint64 OnlineRasterTemplate::TileWindow::width() const noexcept +{ + return isEmpty() ? 0 : max_column - min_column + 1; +} + +qint64 OnlineRasterTemplate::TileWindow::height() const noexcept +{ + return isEmpty() ? 0 : max_row - min_row + 1; +} + +void OnlineRasterTemplate::AtlasCache::clear() +{ + window = {}; + signature.clear(); + image = {}; + image_to_map = {}; + map_bounds = {}; + pixels_per_map_unit = 0; + provisional = false; + output_owned = false; + memory.reset(); + render_memory.reset(); +} + +OnlineRasterTemplate::MemoryReservation::~MemoryReservation() +{ + if (counter && bytes > 0) + counter->fetch_sub(bytes, std::memory_order_relaxed); +} + +void OnlineRasterTemplate::MemoryReservation::shrinkTo( + qint64 retained_bytes) noexcept +{ + retained_bytes = std::clamp( + retained_bytes, 0, bytes); + auto const released = bytes - retained_bytes; + bytes = retained_bytes; + if (counter && released > 0) + counter->fetch_sub(released, std::memory_order_relaxed); +} + +OnlineRasterTemplate::EncodedTilePayload::~EncodedTilePayload() +{ + if (bytes_in_flight && byte_count > 0) + bytes_in_flight->fetch_sub(byte_count, std::memory_order_relaxed); +} + +OnlineRasterTemplate::OnlineRasterTemplate(imagery::ImagerySourceSnapshot snapshot, Map* map, + imagery::TileNetworkManager* network) + : OnlineRasterTemplate(QString{}, map, network) +{ + setSnapshot(std::move(snapshot)); +} + +OnlineRasterTemplate::OnlineRasterTemplate(const QString& path, Map* map, + imagery::TileNetworkManager* network) + : TemplateImage(path, map), + network_(network ? network : &imagery::TileNetworkManager::instance()), + network_client_id_(imagery::TileNetworkManager::nextClientId()), + decode_bytes_in_flight_(sharedDecodeBytesInFlight()) +{ + if (!network) + (void)imagery::ImageryNetworkPermissions::instance(); + initializeConnections(); + liveOnlineRasterTemplates().insert(this); +} + +OnlineRasterTemplate::OnlineRasterTemplate(const OnlineRasterTemplate& prototype) + : TemplateImage(prototype), + snapshot_(prototype.snapshot_), + stored_snapshot_json_(prototype.stored_snapshot_json_), + stored_snapshot_sha256_(prototype.stored_snapshot_sha256_), + stored_snapshot_version_(prototype.stored_snapshot_version_), + stored_snapshot_payloads_(prototype.stored_snapshot_payloads_), + stored_version_attribute_(prototype.stored_version_attribute_), + stored_checksum_attribute_(prototype.stored_checksum_attribute_), + snapshot_error_(prototype.snapshot_error_), + source_(prototype.source_), + network_(prototype.network_), + network_client_id_(imagery::TileNetworkManager::nextClientId()), + decode_bytes_in_flight_(sharedDecodeBytesInFlight()) +{ + if (source_) + source_projection_ = std::make_unique(source_->tile_matrix_set.crs); + image = QImage(1, 1, QImage::Format_RGBA8888); + image.fill(Qt::transparent); + initializeConnections(); + liveOnlineRasterTemplates().insert(this); +} + +OnlineRasterTemplate::~OnlineRasterTemplate() +{ + liveOnlineRasterTemplates().remove(this); + if (template_state == Loaded) + unloadTemplateFile(); + network_->cancelClient(network_client_id_); + decode_owner_.invalidate(); +} + +void OnlineRasterTemplate::initializeConnections() +{ + auto& georeferencing = map->getGeoreferencing(); + disconnect(&georeferencing, &Georeferencing::projectionChanged, this, + &TemplateImage::updateGeoreferencing); + disconnect(&georeferencing, &Georeferencing::transformationChanged, this, + &TemplateImage::updateGeoreferencing); + connect(&georeferencing, &Georeferencing::projectionChanged, this, + &OnlineRasterTemplate::onMapGeoreferencingChanged); + connect(&georeferencing, &Georeferencing::transformationChanged, this, + &OnlineRasterTemplate::onMapGeoreferencingChanged); + connect(&georeferencing, &Georeferencing::stateChanged, this, + &OnlineRasterTemplate::onMapGeoreferencingChanged); + connect(network_, &imagery::TileNetworkManager::finished, this, + &OnlineRasterTemplate::onNetworkFinished); + connect(network_, &imagery::TileNetworkManager::privateOriginApprovalChanged, this, + [this](const QString& origin, bool approved) { + if (!approved || !source_) + return; + QVector recovered; + for (auto found = failed_tiles_.begin(); + found != failed_tiles_.end(); ++found) + { + QVector endpoints; + for (auto rejected = + found->policy_rejected_origins.cbegin(); + rejected + != found->policy_rejected_origins.cend(); + ++rejected) + { + if (rejected.value() == origin) + endpoints.push_back(rejected.key()); + } + if (endpoints.isEmpty()) + continue; + for (auto const endpoint : std::as_const(endpoints)) + { + found->policy_rejected_origins.remove(endpoint); + found->terminal_endpoints.remove(endpoint); + } + found->permanent = false; + found->retry = QDeadlineTimer(0); + recovered.push_back(found.key()); + } + if (!recovered.isEmpty()) + { + queueWindow(wanted_window_, false); + if (output_preparation_active_) + queueWindow(output_window_, false); + } + }); + connect(network_, &imagery::TileNetworkManager::offlineModeChanged, this, + [this](bool offline) { + if (offline || offline_tiles_.isEmpty()) + return; + offline_tiles_.clear(); + queueWindow(wanted_window_, false); + if (output_preparation_active_ && output_window_ != wanted_window_) + queueWindow(output_window_, false); + auto const bounds = calculateTemplateBoundingBox(); + if (bounds.isValid()) + map->setTemplateAreaDirty(this, bounds, getTemplateBoundingBoxPixelBorder()); + }); + retry_timer_.setSingleShot(true); + connect(&retry_timer_, &QTimer::timeout, this, [this] { + queueWindow(wanted_window_, false); + scheduleNextRetry(); + }); + atlas_retry_timer_.setSingleShot(true); + connect(&atlas_retry_timer_, &QTimer::timeout, this, [this] { + atlas_queue_busy_ = false; + auto const bounds = last_render_bounds_.isValid() ? last_render_bounds_ + : calculateTemplateBoundingBox(); + if (bounds.isValid()) + map->setTemplateAreaDirty(this, bounds, getTemplateBoundingBoxPixelBorder()); + }); +} + +std::unique_ptr OnlineRasterTemplate::createForType(const QString& path, + Map* map) +{ + return std::unique_ptr(new OnlineRasterTemplate(path, map, nullptr)); +} + +OnlineRasterTemplate* OnlineRasterTemplate::duplicate() const +{ + return new OnlineRasterTemplate(*this); +} + +const char* OnlineRasterTemplate::getTemplateType() const +{ + return "OnlineRasterTemplate"; +} + +bool OnlineRasterTemplate::fileExists() const +{ + return true; +} + +Template::LookupResult OnlineRasterTemplate::tryToFindTemplateFile(const QString&) +{ + if (template_state != Loaded) + template_state = Unloaded; + return FoundByAbsPath; +} + +const imagery::ImagerySourceSnapshot* OnlineRasterTemplate::sourceSnapshot() const noexcept +{ + return snapshot_ ? &*snapshot_ : nullptr; +} + +void OnlineRasterTemplate::setDisplayName(const QString& name) +{ + auto const trimmed = name.trimmed(); + template_file = trimmed.isEmpty() && source_ ? source_->metadata.name : trimmed; +} + +bool OnlineRasterTemplate::sourceReady() const noexcept +{ + return source_ && source_projection_ && source_projection_->isValid() + && map->getGeoreferencing().getState() == Georeferencing::Geospatial; +} + +const imagery::ResolvedImagerySource* OnlineRasterTemplate::source() const noexcept +{ + return source_.get(); +} + +const imagery::TileMatrix* OnlineRasterTemplate::matrix(int zoom) const noexcept +{ + return source_ ? source_->tile_matrix_set.matrixForZoom(zoom) : nullptr; +} + +const imagery::TileMatrixLimits* OnlineRasterTemplate::limits(int zoom) const noexcept +{ + return source_ ? source_->limitsForZoom(zoom) : nullptr; +} + +bool OnlineRasterTemplate::tileAllowed(const OnlineRasterTileKey& key) const noexcept +{ + auto const* tile_matrix = matrix(key.zoom); + if (!tile_matrix || !tile_matrix->contains(key.column, key.row)) + return false; + auto const* tile_limits = limits(key.zoom); + return !tile_limits || tile_limits->contains(key.column, key.row); +} + +imagery::CrsBounds OnlineRasterTemplate::tileBounds(const OnlineRasterTileKey& key) const noexcept +{ + auto const* tile_matrix = matrix(key.zoom); + return tile_matrix ? tile_matrix->tileBounds(key.column, key.row) : imagery::CrsBounds{}; +} + +void OnlineRasterTemplate::setSnapshot(imagery::ImagerySourceSnapshot snapshot) +{ + stored_snapshot_json_ = snapshot.canonical_json; + stored_snapshot_sha256_ = snapshot.sha256; + stored_snapshot_version_ = QString::number(imagery::ImagerySourceSnapshotCodec::version); + stored_snapshot_payloads_ = { + { + QStringLiteral("base64"), + QString::fromLatin1(stored_snapshot_json_.toBase64(QByteArray::Base64Encoding)), + }, + }; + stored_version_attribute_ = true; + stored_checksum_attribute_ = true; + source_ = std::make_shared(snapshot.source); + snapshot_ = std::move(snapshot); + snapshot_error_.clear(); + if (template_file.isEmpty()) + template_file = source_->metadata.name; +} + +bool OnlineRasterTemplate::decodeStoredSnapshot() +{ + if (!stored_version_attribute_ || stored_snapshot_version_.isEmpty()) + { + snapshot_error_ = tr("The embedded imagery source version is missing."); + return false; + } + if (stored_snapshot_version_ != QString::number(imagery::ImagerySourceSnapshotCodec::version)) + { + snapshot_error_ = tr("The embedded imagery source version %1 is not supported.") + .arg(stored_snapshot_version_); + return false; + } + if (stored_snapshot_payloads_.size() != 1) + { + snapshot_error_ = stored_snapshot_payloads_.isEmpty() + ? tr("The embedded imagery source is missing.") + : tr("The embedded imagery source contains multiple payloads."); + return false; + } + auto const& payload = stored_snapshot_payloads_.front(); + if (payload.encoding != QLatin1String("base64")) + { + snapshot_error_ = + tr("The embedded imagery source encoding %1 is not supported.").arg(payload.encoding); + return false; + } + if (payload.text.size() + > imagery::ImagerySourceSnapshotCodec::maximum_base64_size) + { + snapshot_error_ = + tr("The embedded imagery source exceeds the size limit."); + return false; + } + auto decoded_payload = QByteArray::fromBase64Encoding(payload.text.toLatin1(), + QByteArray::AbortOnBase64DecodingErrors); + if (!decoded_payload) + { + snapshot_error_ = tr("The embedded imagery source is not valid base64."); + return false; + } + if (decoded_payload.decoded.size() + > imagery::ImagerySourceSnapshotCodec::maximum_size) + { + snapshot_error_ = + tr("The embedded imagery source exceeds the size limit."); + return false; + } + stored_snapshot_json_ = std::move(decoded_payload.decoded); + if (stored_snapshot_json_.isEmpty()) + { + snapshot_error_ = tr("The embedded imagery source is missing."); + return false; + } + if (!stored_checksum_attribute_ || stored_snapshot_sha256_.isEmpty()) + { + snapshot_error_ = tr("The embedded imagery source checksum is missing."); + return false; + } + static const QRegularExpression sha_pattern(QStringLiteral("^[0-9a-f]{64}$")); + if (!sha_pattern.match(QString::fromLatin1(stored_snapshot_sha256_)).hasMatch()) + { + snapshot_error_ = tr("The embedded imagery source checksum is not valid SHA-256."); + return false; + } + auto const actual_sha = + QCryptographicHash::hash(stored_snapshot_json_, QCryptographicHash::Sha256).toHex(); + if (!stored_snapshot_sha256_.isEmpty() && actual_sha != stored_snapshot_sha256_) + { + snapshot_error_ = tr("The embedded imagery source checksum does not match."); + return false; + } + QString error; + auto decoded = imagery::ImagerySourceSnapshotCodec::decode(stored_snapshot_json_, &error); + if (!decoded) + { + snapshot_error_ = error; + return false; + } + setSnapshot(std::move(*decoded)); + return true; +} + +void OnlineRasterTemplate::saveTypeSpecificTemplateConfiguration(QXmlStreamWriter& xml) const +{ + xml.writeStartElement(QStringLiteral("online_source")); + if (stored_version_attribute_) + xml.writeAttribute(QStringLiteral("snapshot_version"), stored_snapshot_version_); + if (stored_checksum_attribute_) + xml.writeAttribute(QStringLiteral("sha256"), QString::fromLatin1(stored_snapshot_sha256_)); + for (auto const& payload : stored_snapshot_payloads_) + { + xml.writeStartElement(QStringLiteral("snapshot_json")); + xml.writeAttribute(QStringLiteral("encoding"), payload.encoding); + xml.writeCharacters(payload.text); + xml.writeEndElement(); + } + xml.writeEndElement(); +} + +bool OnlineRasterTemplate::loadTypeSpecificTemplateConfiguration(QXmlStreamReader& xml) +{ + if (xml.name() != QLatin1String("online_source")) + { + xml.skipCurrentElement(); + return true; + } + + auto const attributes = xml.attributes(); + stored_version_attribute_ = attributes.hasAttribute(QLatin1String("snapshot_version")); + stored_checksum_attribute_ = attributes.hasAttribute(QLatin1String("sha256")); + auto const version = attributes.value(QLatin1String("snapshot_version")); + auto const checksum = attributes.value(QLatin1String("sha256")); + stored_snapshot_version_.clear(); + stored_snapshot_sha256_.clear(); + stored_snapshot_payloads_.clear(); + stored_snapshot_json_.clear(); + snapshot_.reset(); + source_.reset(); + snapshot_error_.clear(); + if (version.size() > maximum_snapshot_version_size + || checksum.size() > maximum_snapshot_checksum_size) + { + snapshot_error_ = + tr("The embedded imagery source metadata exceeds the size limit."); + } + else + { + stored_snapshot_version_ = version.toString(); + stored_snapshot_sha256_ = checksum.toLatin1(); + } + while (xml.readNextStartElement()) + { + if (xml.name() == QLatin1String("snapshot_json")) + { + auto const encoding = + xml.attributes() + .value(QLatin1String("encoding")) + .toString(); + auto payload = readBoundedElementText( + xml, + imagery::ImagerySourceSnapshotCodec::maximum_base64_size); + if (payload.too_large + || encoding.size() > maximum_snapshot_encoding_size) + { + if (snapshot_error_.isEmpty()) + snapshot_error_ = tr( + "The embedded imagery source exceeds the size limit."); + continue; + } + if (stored_snapshot_payloads_.size() + >= maximum_embedded_snapshot_payloads) + { + if (snapshot_error_.isEmpty()) + snapshot_error_ = tr( + "The embedded imagery source contains too many payloads."); + continue; + } + stored_snapshot_payloads_.push_back({ + encoding, + std::move(payload.text), + }); + } + else + { + xml.skipCurrentElement(); + } + } + is_georeferenced = true; + return true; +} + +bool OnlineRasterTemplate::finishTypeSpecificTemplateConfiguration() +{ + if (snapshot_error_.isEmpty()) + decodeStoredSnapshot(); + is_georeferenced = true; + return true; +} + +bool OnlineRasterTemplate::loadTemplateFileImpl() +{ + if (!source_ && !decodeStoredSnapshot()) + { + setErrorString(snapshot_error_); + return false; + } + QString error; + if (!source_->validate(&error)) + { + setErrorString(error); + return false; + } + if (map->getGeoreferencing().getState() != Georeferencing::Geospatial) + { + setErrorString(tr("Online imagery requires a georeferenced map.")); + return false; + } + source_projection_ = std::make_unique(source_->tile_matrix_set.crs); + if (!source_projection_->isValid()) + { + setErrorString(tr("The imagery coordinate reference system is not usable: %1") + .arg(source_->tile_matrix_set.crs)); + source_projection_.reset(); + return false; + } + + resetRuntime(false); + image = QImage(1, 1, QImage::Format_RGBA8888); + image.fill(Qt::transparent); + is_georeferenced = true; + transform = TemplateTransform{}; + updateTransformationMatrices(); + return true; +} + +bool OnlineRasterTemplate::postLoadSetup(QWidget*, bool& out_center_in_view) +{ + out_center_in_view = false; + return true; +} + +void OnlineRasterTemplate::unloadTemplateFileImpl() +{ + resetRuntime(true); + source_projection_.reset(); + image = {}; +} + +void OnlineRasterTemplate::resetRuntime(bool clear_cache) +{ + auto const previous_generation = generation_; + if (generation_ == std::numeric_limits::max()) + qFatal("Online imagery generation space exhausted"); + ++generation_; + network_->cancelClient(network_client_id_, previous_generation); + decode_owner_.invalidate(); + cancelAtlasBuild(); + pending_fetches_.clear(); + for (auto const& pending : std::as_const(pending_decodes_)) + pending.cancelled->store(true, std::memory_order_relaxed); + pending_decodes_.clear(); + queued_tiles_.clear(); + failed_tiles_.clear(); + offline_tiles_.clear(); + retry_timer_.stop(); + atlas_retry_timer_.stop(); + atlas_queue_busy_ = false; + wanted_window_ = {}; + output_window_ = {}; + output_preparation_active_ = false; + output_preparation_error_.clear(); + output_preparation_scale_ = 0; + output_source_tiles_released_ = false; + output_keys_.clear(); + output_tiles_.clear(); + output_render_memory_.clear(); + atlas_.clear(); + output_atlases_.clear(); + output_uses_atlases_ = false; + last_render_bounds_ = {}; + if (clear_cache) + tile_cache_.clear(); + setResourceStatus({}); +} + +QSize OnlineRasterTemplate::getRasterPixelSize() const +{ + if (!source_) + return {}; + auto const* deepest = matrix(source_->max_zoom); + if (!deepest) + return {}; + auto const width = std::min(std::numeric_limits::max(), + static_cast(deepest->matrix_width) + * deepest->tile_size.width()); + auto const height = std::min(std::numeric_limits::max(), + static_cast(deepest->matrix_height) + * deepest->tile_size.height()); + return { int(width), int(height) }; +} + +std::optional OnlineRasterTemplate::mapToNominalSource(const QPointF& map_point) const +{ + if (!sourceReady()) + return std::nullopt; + bool ok = false; + auto const lat_lon = map->getGeoreferencing().toGeographicCoords(MapCoordF(map_point), &ok); + if (!ok) + return std::nullopt; + auto projected = source_projection_->forward(lat_lon, &ok); + if (!ok || !std::isfinite(projected.x()) || !std::isfinite(projected.y())) + return std::nullopt; + if (source_->registration) + { + projected.rx() -= source_->registration->dx; + projected.ry() -= source_->registration->dy; + } + return projected; +} + +std::optional OnlineRasterTemplate::nominalSourceToMap(const QPointF& source_point) const +{ + if (!sourceReady()) + return std::nullopt; + auto corrected = source_point; + if (source_->registration) + { + corrected.rx() += source_->registration->dx; + corrected.ry() += source_->registration->dy; + } + bool ok = false; + auto const lat_lon = source_projection_->inverse(corrected, &ok); + if (!ok) + return std::nullopt; + auto const result = map->getGeoreferencing().toMapCoordF(lat_lon, &ok); + if (!ok || !std::isfinite(result.x()) || !std::isfinite(result.y())) + return std::nullopt; + return QPointF(result); +} + +std::optional OnlineRasterTemplate::imagePointToMap(const OnlineRasterTileKey& image_key, + const QPointF& image_point) const +{ + auto const* tile_matrix = matrix(image_key.zoom); + auto const bounds = tileBounds(image_key); + if (!tile_matrix || !bounds.isValid()) + return std::nullopt; + auto const source_point = + QPointF(bounds.west + (image_point.x() - 1) * tile_matrix->cell_size, + bounds.north - (image_point.y() - 1) * tile_matrix->cell_size); + return nominalSourceToMap(source_point); +} + +std::optional OnlineRasterTemplate::imageRectToMap(const OnlineRasterTileKey& image_key, + const QRectF& source_rect, + QRectF* map_bounds, + double* residual_map_units) const +{ + if (!source_rect.isValid() || source_rect.isEmpty()) + return std::nullopt; + auto const top_left = imagePointToMap(image_key, source_rect.topLeft()); + auto const top_right = imagePointToMap(image_key, source_rect.topRight()); + auto const bottom_left = imagePointToMap(image_key, source_rect.bottomLeft()); + auto const bottom_right = imagePointToMap(image_key, source_rect.bottomRight()); + auto const center = imagePointToMap(image_key, source_rect.center()); + if (!top_left || !top_right || !bottom_left || !bottom_right || !center) + return std::nullopt; + + auto const width = source_rect.width(); + auto const height = source_rect.height(); + QTransform transform( + (top_right->x() - top_left->x()) / width, (top_right->y() - top_left->y()) / width, + (bottom_left->x() - top_left->x()) / height, (bottom_left->y() - top_left->y()) / height, + top_left->x() - source_rect.x() * (top_right->x() - top_left->x()) / width + - source_rect.y() * (bottom_left->x() - top_left->x()) / height, + top_left->y() - source_rect.x() * (top_right->y() - top_left->y()) / width + - source_rect.y() * (bottom_left->y() - top_left->y()) / height); + if (!transform.isInvertible()) + return std::nullopt; + + if (map_bounds) + { + *map_bounds = boundsOf({ *top_left, *top_right, *bottom_left, *bottom_right }); + } + if (residual_map_units) + { + double residual = 0; + for (int y = 0; y <= 4; ++y) + { + for (int x = 0; x <= 4; ++x) + { + auto const sample = QPointF(source_rect.left() + source_rect.width() * x / 4.0, + source_rect.top() + source_rect.height() * y / 4.0); + auto const actual = imagePointToMap(image_key, sample); + if (!actual) + return std::nullopt; + residual = std::max(residual, pointDistance(transform.map(sample), *actual)); + } + } + *residual_map_units = residual; + } + return transform; +} + +OnlineRasterTemplate::TileWindow OnlineRasterTemplate::tileWindowForMapRect(const QRectF& map_rect, + int zoom, + bool exact_output, + bool* projection_complete) const +{ + if (projection_complete) + *projection_complete = false; + TileWindow window; + window.zoom = zoom; + auto const* tile_matrix = matrix(zoom); + if (!tile_matrix || !map_rect.isValid() || map_rect.isEmpty()) + return window; + + QRectF source_bounds; + bool complete = true; + double uncertainty = 0; + if (!exact_output) + { + for (int y = 0; y <= 4; ++y) + { + for (int x = 0; x <= 4; ++x) + { + auto const point = + QPointF(map_rect.left() + map_rect.width() * x / 4.0, + map_rect.top() + map_rect.height() * y / 4.0); + if (auto projected = mapToNominalSource(point)) + rectIncludeSafe(source_bounds, *projected); + } + } + } + else + { + std::function sample_cell; + sample_cell = [this, tile_matrix, &source_bounds, &complete, &uncertainty, + &sample_cell](const QRectF& cell, int depth) { + if (!complete) + return; + std::array map_points { + cell.topLeft(), + cell.topRight(), + cell.bottomLeft(), + cell.bottomRight(), + QPointF(cell.center().x(), cell.top()), + QPointF(cell.center().x(), cell.bottom()), + QPointF(cell.left(), cell.center().y()), + QPointF(cell.right(), cell.center().y()), + cell.center(), + }; + std::array projected; + for (std::size_t index = 0; index < map_points.size(); ++index) + { + auto value = mapToNominalSource(map_points.at(index)); + if (!value) + { + complete = false; + return; + } + projected.at(index) = *value; + rectIncludeSafe(source_bounds, *value); + } + + auto error = 0.0; + error = std::max( + error, pointDistance(projected.at(4), + (projected.at(0) + projected.at(1)) / 2)); + error = std::max( + error, pointDistance(projected.at(5), + (projected.at(2) + projected.at(3)) / 2)); + error = std::max( + error, pointDistance(projected.at(6), + (projected.at(0) + projected.at(2)) / 2)); + error = std::max( + error, pointDistance(projected.at(7), + (projected.at(1) + projected.at(3)) / 2)); + error = std::max( + error, pointDistance( + projected.at(8), + (projected.at(0) + projected.at(1) + projected.at(2) + + projected.at(3)) + / 4)); + auto const span = std::max( + { pointDistance(projected.at(0), projected.at(1)), + pointDistance(projected.at(0), projected.at(2)), + pointDistance(projected.at(1), projected.at(3)), + pointDistance(projected.at(2), projected.at(3)) }); + auto const tolerance = + std::max(tile_matrix->cell_size * 0.05, span * 1.0e-8); + if (depth < 2 || error > tolerance) + { + if (depth >= 7) + { + complete = false; + return; + } + auto const half_width = cell.width() / 2; + auto const half_height = cell.height() / 2; + sample_cell( + QRectF(cell.left(), cell.top(), half_width, half_height), depth + 1); + sample_cell( + QRectF(cell.center().x(), cell.top(), half_width, half_height), depth + 1); + sample_cell( + QRectF(cell.left(), cell.center().y(), half_width, half_height), depth + 1); + sample_cell( + QRectF(cell.center(), QSizeF(half_width, half_height)), depth + 1); + return; + } + uncertainty = std::max(uncertainty, error); + }; + sample_cell(map_rect, 0); + } + if (!source_bounds.isValid() || source_bounds.isEmpty()) + return window; + auto const tile_width = tile_matrix->cell_size * tile_matrix->tile_size.width(); + auto const tile_height = tile_matrix->cell_size * tile_matrix->tile_size.height(); + auto const numerical_uncertainty = + 128 * std::numeric_limits::epsilon() + * std::max({ + 1.0, + std::abs(source_bounds.left()), + std::abs(source_bounds.right()), + std::abs(source_bounds.top()), + std::abs(source_bounds.bottom()), + std::abs(tile_width), + std::abs(tile_height), + }); + if (uncertainty <= numerical_uncertainty) + uncertainty = 0; + if (uncertainty > 0) + source_bounds.adjust(-uncertainty, -uncertainty, uncertainty, uncertainty); + if (projection_complete) + *projection_complete = complete; + + auto min_column = + boundedFloor((source_bounds.left() - tile_matrix->point_of_origin.x()) / tile_width); + auto max_column = boundedFloor( + std::nextafter((source_bounds.right() - tile_matrix->point_of_origin.x()) / tile_width, + -std::numeric_limits::infinity())); + auto min_row = + boundedFloor((tile_matrix->point_of_origin.y() - source_bounds.bottom()) / tile_height); + auto max_row = boundedFloor( + std::nextafter((tile_matrix->point_of_origin.y() - source_bounds.top()) / tile_height, + -std::numeric_limits::infinity())); + + qint64 allowed_min_column = 0; + qint64 allowed_max_column = tile_matrix->matrix_width - 1; + qint64 allowed_min_row = 0; + qint64 allowed_max_row = tile_matrix->matrix_height - 1; + if (auto const* tile_limits = limits(zoom)) + { + allowed_min_column = tile_limits->min_column; + allowed_max_column = tile_limits->max_column; + allowed_min_row = tile_limits->min_row; + allowed_max_row = tile_limits->max_row; + } + window.min_column = std::max(min_column, allowed_min_column); + window.max_column = std::min(max_column, allowed_max_column); + window.min_row = std::max(min_row, allowed_min_row); + window.max_row = std::min(max_row, allowed_max_row); + return window; +} + +int OnlineRasterTemplate::chooseZoom(const QRectF& map_rect, double pixels_per_map_unit, + bool exact_output) const +{ + if (exact_output) + exact_projection_failed_ = false; + if (!source_) + return -1; + auto result = source_->max_zoom; + if (!(pixels_per_map_unit > 0) || !std::isfinite(pixels_per_map_unit)) + return result; + + auto minimum_scale = std::numeric_limits::infinity(); + auto const subdivisions = exact_output ? 8 : 2; + auto const largest_dimension = std::max(map_rect.width(), map_rect.height()); + auto const derivative_step = std::clamp(largest_dimension / 4096.0, 1.0e-3, 1.0); + bool scale_complete = true; + for (int y = 0; y <= subdivisions; ++y) + { + for (int x = 0; x <= subdivisions; ++x) + { + auto const point = + QPointF(map_rect.left() + map_rect.width() * x / subdivisions, + map_rect.top() + map_rect.height() * y / subdivisions); + auto const source = mapToNominalSource(point); + auto const source_x = + mapToNominalSource(point + QPointF(derivative_step, 0)); + auto const source_y = + mapToNominalSource(point + QPointF(0, derivative_step)); + if (!source || !source_x || !source_y) + { + scale_complete = false; + continue; + } + + auto const x_vector = (*source_x - *source) / derivative_step; + auto const y_vector = (*source_y - *source) / derivative_step; + auto const trace = QPointF::dotProduct(x_vector, x_vector) + + QPointF::dotProduct(y_vector, y_vector); + auto const determinant = + x_vector.x() * y_vector.y() - x_vector.y() * y_vector.x(); + auto const discriminant = + std::max(0.0, trace * trace - 4.0 * determinant * determinant); + auto const singular_value = + std::sqrt(std::max(0.0, 0.5 * (trace - std::sqrt(discriminant)))); + if (singular_value > 0 && std::isfinite(singular_value)) + minimum_scale = std::min(minimum_scale, singular_value); + else + scale_complete = false; + } + } + if (exact_output && !scale_complete) + { + exact_projection_failed_ = true; + return -1; + } + if (std::isfinite(minimum_scale)) + { + auto const desired_cell = minimum_scale / pixels_per_map_unit; + // Interactive rendering may tolerate modest resampling to keep panning + // fluid. Exact output requires source pixels at least as fine as output + // pixels, with a small numerical safety margin. + auto const maximum_cell = desired_cell * (exact_output ? 0.98 : 1.5); + for (int zoom = source_->min_zoom; zoom <= source_->max_zoom; ++zoom) + { + auto const* candidate = matrix(zoom); + if (candidate && candidate->cell_size <= maximum_cell) + { + result = zoom; + break; + } + } + } + + bool projection_complete = true; + auto const selected_window = + tileWindowForMapRect( + map_rect, result, exact_output, + &projection_complete); + if (exact_output) + { + if (!projection_complete) + { + exact_projection_failed_ = true; + return -1; + } + // Never hide a resource-limit failure by exporting a coarser imagery + // level than the requested output resolution. + if (!selected_window.isEmpty() && !workingSetFits(selected_window)) + return -1; + return result; + } + + while (result > source_->min_zoom) + { + auto const window = withOverscan(tileWindowForMapRect(map_rect, result), 1); + if (window.isEmpty() || workingSetFits(window)) + break; + --result; + } + auto const final_window = withOverscan(tileWindowForMapRect(map_rect, result), 1); + if (!final_window.isEmpty() && !workingSetFits(final_window)) + return -1; + return result; +} + +OnlineRasterTemplate::TileWindow OnlineRasterTemplate::withOverscan(TileWindow window, + qint64 tiles) const +{ + if (window.isEmpty()) + return window; + auto const* tile_matrix = matrix(window.zoom); + if (!tile_matrix) + return {}; + qint64 min_column = 0; + qint64 max_column = tile_matrix->matrix_width - 1; + qint64 min_row = 0; + qint64 max_row = tile_matrix->matrix_height - 1; + if (auto const* tile_limits = limits(window.zoom)) + { + min_column = tile_limits->min_column; + max_column = tile_limits->max_column; + min_row = tile_limits->min_row; + max_row = tile_limits->max_row; + } + window.min_column = std::max(min_column, window.min_column - tiles); + window.max_column = std::min(max_column, window.max_column + tiles); + window.min_row = std::max(min_row, window.min_row - tiles); + window.max_row = std::min(max_row, window.max_row + tiles); + return window; +} + +std::optional OnlineRasterTemplate::tileCount(const TileWindow& window) const noexcept +{ + auto const width = window.width(); + auto const height = window.height(); + if (width <= 0 || height <= 0 || width > std::numeric_limits::max() / height) + { + return std::nullopt; + } + return width * height; +} + +bool OnlineRasterTemplate::workingSetFits(const TileWindow& window) const noexcept +{ + auto const count = tileCount(window); + auto const* tile_matrix = matrix(window.zoom); + if (!count || !tile_matrix || *count > max_window_tiles) + return false; + auto const width = qint64(tile_matrix->tile_size.width()) + 2; + auto const height = qint64(tile_matrix->tile_size.height()) + 2; + if (width <= 0 || height <= 0 || width > std::numeric_limits::max() / height + || width * height > std::numeric_limits::max() / 4) + { + return false; + } + auto const tile_bytes = width * height * 4; + // Reserve half of the budget again for coarse coverage, prior windows, + // and cache bookkeeping so the exact working set cannot churn itself out. + auto const reserved_tiles = *count + (*count + 1) / 2; + return reserved_tiles <= max_working_set_bytes / tile_bytes; +} + +bool OnlineRasterTemplate::keyNeededForWindow(const OnlineRasterTileKey& key, + const TileWindow& window) const noexcept +{ + if (window.isEmpty() || key.zoom < source_->min_zoom || key.zoom > window.zoom) + { + return false; + } + auto const shift = window.zoom - key.zoom; + return key.column >= (window.min_column >> shift) && key.column <= (window.max_column >> shift) + && key.row >= (window.min_row >> shift) && key.row <= (window.max_row >> shift); +} + +void OnlineRasterTemplate::cancelUnwantedWork(const TileWindow& window) +{ + QVector cancelled; + cancelled.reserve(pending_fetches_.size()); + for (auto found = pending_fetches_.cbegin(); found != pending_fetches_.cend(); ++found) + { + if (!keyNeededForWindow(found->key, window) && !output_keys_.contains(found->key)) + cancelled.push_back(found.key()); + } + for (auto token : cancelled) + { + auto found = pending_fetches_.find(token); + if (found == pending_fetches_.end()) + continue; + queued_tiles_.remove(found->key); + pending_fetches_.erase(found); + network_->cancel(token); + } + QVector cancelled_decodes; + cancelled_decodes.reserve(pending_decodes_.size()); + for (auto found = pending_decodes_.cbegin(); found != pending_decodes_.cend(); ++found) + { + if (!keyNeededForWindow(found.key(), window) && !output_keys_.contains(found.key())) + cancelled_decodes.push_back(found.key()); + } + for (auto const& key : cancelled_decodes) + { + auto found = pending_decodes_.find(key); + if (found == pending_decodes_.end()) + continue; + found->cancelled->store(true, std::memory_order_relaxed); + // Keep the key admitted until the worker completion releases its + // encoded payload. This prevents a quick pan back from duplicating a + // still-resident decode allocation. + } +} + +void OnlineRasterTemplate::updateRenderContext(const ViewRenderContext& context) +{ + if (template_state != Loaded || !sourceReady()) + return; + auto const pixels_per_map_unit = Util::mmToPixelPhysical(context.view_zoom); + auto const zoom = chooseZoom(context.visible_map_rect, pixels_per_map_unit); + auto window = withOverscan(tileWindowForMapRect(context.visible_map_rect, zoom), 1); + auto const replace_pending = wanted_window_ != window; + wanted_window_ = window; + queueWindow(window, replace_pending); +} + +OutputRenderPreparation OnlineRasterTemplate::prepareForOutput(const QRectF& map_rect, + double pixels_per_map_unit) +{ + if (template_state != Loaded || !sourceReady()) + { + return { + OutputRenderPreparation::State::Failed, + 0, + 0, + tr("Online imagery requires a loaded source and a georeferenced map."), + }; + } + if (output_preparation_active_ + && !output_preparation_error_.isEmpty()) + { + return { + OutputRenderPreparation::State::Failed, + 0, + 0, + output_preparation_error_, + }; + } + auto const zoom = chooseZoom(map_rect, pixels_per_map_unit, true); + if (zoom < 0) + { + return { + OutputRenderPreparation::State::Failed, + 0, + 0, + exact_projection_failed_ + ? tr("The requested area cannot be completely reprojected into " + "the imagery coordinate reference system.") + : tr("The requested imagery output exceeds the bounded working set."), + }; + } + bool projection_complete = false; + auto const window = tileWindowForMapRect(map_rect, zoom, true, &projection_complete); + if (!projection_complete) + { + return { + OutputRenderPreparation::State::Failed, + 0, + 0, + tr("The requested area cannot be completely reprojected into " + "the imagery coordinate reference system."), + }; + } + if (window.isEmpty()) + { + if (output_preparation_active_) + finishOutputPreparation(true); + output_preparation_active_ = true; + output_window_ = {}; + output_preparation_scale_ = + pixels_per_map_unit; + updateResourceStatus(); + return {}; + } + if (!workingSetFits(window)) + { + return { + OutputRenderPreparation::State::Failed, + 0, + 0, + tr("The requested imagery output exceeds the bounded working set."), + }; + } + auto const scale_reference = std::max({ + 1.0, + std::abs(output_preparation_scale_), + std::abs(pixels_per_map_unit), + }); + auto const output_scale_changed = + output_preparation_active_ + && std::abs( + output_preparation_scale_ + - pixels_per_map_unit) + > scale_reference * 1.0e-9; + if (output_preparation_active_ + && (output_window_ != window + || output_scale_changed)) + finishOutputPreparation(true); + output_preparation_active_ = true; + output_window_ = window; + output_preparation_scale_ = pixels_per_map_unit; + + auto const count = tileCount(window); + if (!count || *count > std::numeric_limits::max()) + { + return { + OutputRenderPreparation::State::Failed, + 0, + 0, + tr("The requested imagery tile range is invalid."), + }; + } + + OutputRenderPreparation result; + result.state = OutputRenderPreparation::State::Pending; + result.total_resources = qsizetype(*count); + if (output_source_tiles_released_) + { + result.total_resources += output_atlases_.size(); + result.ready_resources = result.total_resources; + for (auto const& cached : std::as_const(output_atlases_)) + { + auto const scale = std::max({ + 1.0, + std::abs(cached.pixels_per_map_unit), + std::abs(pixels_per_map_unit), + }); + if (!cached.output_owned + || cached.image.isNull() + || cached.provisional + || !cached.render_memory + || std::abs( + cached.pixels_per_map_unit + - pixels_per_map_unit) + > scale * 1.0e-9) + { + result.state = + OutputRenderPreparation::State::Failed; + result.error = tr( + "The prepared translucent imagery is no longer " + "available for exact output."); + updateResourceStatus(); + return result; + } + } + result.state = OutputRenderPreparation::State::Ready; + updateResourceStatus(); + return result; + } + auto available = std::max(0, max_queued_tiles - queued_tiles_.size()); + auto const center_column = 0.5 * (window.min_column + window.max_column); + auto const center_row = 0.5 * (window.min_row + window.max_row); + for (qint64 row = window.min_row; row <= window.max_row; ++row) + { + for (qint64 column = window.min_column; column <= window.max_column; ++column) + { + OnlineRasterTileKey key{ window.zoom, column, row }; + output_keys_.insert(key); + if (output_tiles_.contains(key)) + { + ++result.ready_resources; + continue; + } + if (auto const* cached = tile_cache_.object(key)) + { + output_tiles_.insert(key, *cached); + ++result.ready_resources; + continue; + } + if (offline_tiles_.contains(key)) + { + result.state = OutputRenderPreparation::State::Failed; + result.error = + tr("An exact imagery tile is not available in the offline cache. " + "Turn off offline imagery mode and try again."); + updateResourceStatus(); + return result; + } + auto const failure = failed_tiles_.constFind(key); + if (failure != failed_tiles_.cend() && failure->permanent) + { + result.state = OutputRenderPreparation::State::Failed; + result.error = failure->message.isEmpty() + ? tr("An online imagery tile cannot be loaded.") + : failure->message; + updateResourceStatus(); + return result; + } + if (available <= 0 || queued_tiles_.contains(key) || !retryAllowed(key)) + { + continue; + } + auto const dx = double(column) - center_column; + auto const dy = double(row) - center_row; + queueTile(key, imagery::TileRequestPriority::Coverage, dx * dx + dy * dy); + --available; + } + } + if (result.ready_resources == result.total_resources) + { + bool has_transparency = false; + bool has_missing = false; + bool has_pixels = false; + auto const visuals = + visualTiles(window, false, &has_transparency, &has_missing, &has_pixels); + if (has_missing) + return result; + if (has_transparency && has_pixels) + { + output_uses_atlases_ = true; + output_render_memory_.clear(); + auto queued_build = false; + for (auto const& chunk : + atlasChunks(window, pixels_per_map_unit)) + { + auto source_window = withOverscan(chunk, 1); + source_window.min_column = std::max( + source_window.min_column, + window.min_column); + source_window.max_column = std::min( + source_window.max_column, + window.max_column); + source_window.min_row = std::max( + source_window.min_row, + window.min_row); + source_window.max_row = std::min( + source_window.max_row, + window.max_row); + bool chunk_transparency = false; + bool chunk_missing = false; + bool chunk_pixels = false; + auto const chunk_visuals = + visualTiles( + source_window, false, + &chunk_transparency, + &chunk_missing, + &chunk_pixels); + Q_UNUSED(chunk_transparency); + if (chunk_missing) + return result; + if (!chunk_pixels) + continue; + + ++result.total_resources; + auto const signature = + atlasSignature( + chunk, chunk_visuals, false); + auto const ready = std::ranges::find_if( + output_atlases_, + [&](auto const& cached) { + auto const scale = std::max({ + 1.0, + std::abs(cached.pixels_per_map_unit), + std::abs(pixels_per_map_unit), + }); + return cached.window == chunk + && cached.signature == signature + && !cached.image.isNull() + && !cached.provisional + && std::abs( + cached.pixels_per_map_unit + - pixels_per_map_unit) + <= scale * 1.0e-9; + }); + if (ready != output_atlases_.cend()) + { + ++result.ready_resources; + continue; + } + if (queued_build) + continue; + queued_build = true; + if (!queueAtlasBuild( + chunk, chunk_visuals, false, + pixels_per_map_unit, signature, true)) + { + if (atlas_queue_busy_) + { + updateResourceStatus(); + return result; + } + result.state = + OutputRenderPreparation::State::Failed; + result.error = tr( + "The translucent imagery cannot be " + "reprojected within the bounded output policy."); + updateResourceStatus(); + return result; + } + } + } + else + { + output_uses_atlases_ = false; + output_atlases_.clear(); + if (result.ready_resources == result.total_resources) + { + for (auto const& visual : visuals) + { + if (!visual.tile || visual.complete_empty) + continue; + auto const bytes = rgbaImageBytes( + visual.tile->image.width(), + visual.tile->image.height()); + auto const existing = + output_render_memory_.constFind( + visual.cached); + if (bytes && existing != output_render_memory_.cend() + && *existing + && (*existing)->bytes >= *bytes) + { + continue; + } + auto reservation = bytes + ? reserveRetainedMemory(*bytes) + : std::shared_ptr {}; + if (!reservation) + { + output_render_memory_.clear(); + result.state = + OutputRenderPreparation::State::Failed; + result.error = tr( + "The application-wide raster memory budget " + "cannot reserve exact renderer snapshots."); + output_preparation_error_ = result.error; + updateResourceStatus(); + return result; + } + output_render_memory_.insert( + visual.cached, + std::move(reservation)); + } + } + } + if (result.ready_resources == result.total_resources) + { + if (output_uses_atlases_) + { + qint64 render_bytes = 0; + bool render_bytes_valid = true; + for (auto const& cached : + std::as_const(output_atlases_)) + { + auto const bytes = qint64( + cached.image.bytesPerLine()) + * cached.image.height(); + if (bytes <= 0 + || render_bytes + > max_retained_raster_bytes + - bytes) + { + render_bytes_valid = false; + break; + } + render_bytes += bytes; + } + if (!render_bytes_valid + || render_bytes + > max_retained_raster_bytes / 2) + { + result.state = + OutputRenderPreparation::State::Failed; + result.error = tr( + "The translucent imagery and its renderer " + "snapshot exceed the bounded raster memory " + "policy."); + updateResourceStatus(); + return result; + } + + // The completed atlases are self-contained. Drop their + // source-tile pins and ordinary cache entries before + // reserving the straight-RGBA snapshots consumed by the + // renderer. This makes a Ready result an actual memory + // guarantee rather than a best-effort promise. + for (auto const& key : + std::as_const(output_keys_)) + tile_cache_.remove(key); + output_tiles_.clear(); + output_source_tiles_released_ = true; + + auto render_memory_ready = true; + for (auto& cached : output_atlases_) + { + auto const bytes = qint64( + cached.image.bytesPerLine()) + * cached.image.height(); + cached.render_memory = + reserveRetainedMemory(bytes); + if (!cached.render_memory) + { + render_memory_ready = false; + break; + } + } + if (!render_memory_ready) + { + for (auto& cached : output_atlases_) + cached.render_memory.reset(); + result.state = + OutputRenderPreparation::State::Failed; + result.error = tr( + "The application-wide raster memory budget " + "cannot reserve exact renderer snapshots."); + updateResourceStatus(); + return result; + } + } + result.state = OutputRenderPreparation::State::Ready; + } + } + updateResourceStatus(); + return result; +} + +void OnlineRasterTemplate::finishOutputPreparation(bool cancelled) +{ + if (!output_preparation_active_) + return; + if (atlas_pending_for_output_) + cancelAtlasBuild(); + if (cancelled) + { + QVector network_tokens; + for (auto found = pending_fetches_.cbegin(); found != pending_fetches_.cend(); ++found) + { + if (output_keys_.contains(found->key) + && !keyNeededForWindow(found->key, wanted_window_)) + { + network_tokens.push_back(found.key()); + } + } + for (auto token : network_tokens) + { + auto found = pending_fetches_.find(token); + if (found == pending_fetches_.end()) + continue; + queued_tiles_.remove(found->key); + pending_fetches_.erase(found); + network_->cancel(token); + } + + QVector decode_keys; + for (auto found = pending_decodes_.cbegin(); found != pending_decodes_.cend(); ++found) + { + if (output_keys_.contains(found.key()) + && !keyNeededForWindow(found.key(), wanted_window_)) + { + decode_keys.push_back(found.key()); + } + } + for (auto const& key : decode_keys) + { + auto found = pending_decodes_.find(key); + if (found == pending_decodes_.end()) + continue; + found->cancelled->store(true, std::memory_order_relaxed); + } + } + QRectF released_output_bounds; + for (auto const& output_atlas : std::as_const(output_atlases_)) + rectIncludeSafe( + released_output_bounds, + output_atlas.map_bounds); + output_atlases_.clear(); + output_uses_atlases_ = false; + output_preparation_active_ = false; + output_preparation_error_.clear(); + output_window_ = {}; + output_preparation_scale_ = 0; + output_source_tiles_released_ = false; + output_keys_.clear(); + output_tiles_.clear(); + output_render_memory_.clear(); + updateResourceStatus(); + if (released_output_bounds.isValid()) + { + map->setTemplateAreaDirty( + this, released_output_bounds, + getTemplateBoundingBoxPixelBorder()); + } +} + +bool OnlineRasterTemplate::retryAllowed(const OnlineRasterTileKey& key) const +{ + if (offline_tiles_.contains(key)) + return false; + auto const found = failed_tiles_.constFind(key); + return found == failed_tiles_.cend() || (!found->permanent && found->retry.hasExpired()); +} + +void OnlineRasterTemplate::recordFailure( + const OnlineRasterTileKey& key, + bool permanent, + QString message) +{ + offline_tiles_.remove(key); + auto& failure = failed_tiles_[key]; + failure.attempts = std::min(failure.attempts + 1, 20); + failure.permanent = permanent; + if (!message.isEmpty()) + failure.message = std::move(message); + auto const base = permanent ? 30'000 : 1000; + auto const cap = permanent ? 5 * 60 * 1000 : 60'000; + auto const delay = std::min(cap, base * (1 << std::min(failure.attempts - 1, 10))); + failure.retry = QDeadlineTimer(delay); + trimFailureHistory(); + scheduleNextRetry(); +} + +void OnlineRasterTemplate::recordEndpointFailure( + const OnlineRasterTileKey& key, + int endpoint, + quint64 endpoint_offset, + bool terminal, + QString message, + const QUrl& policy_rejected_url) +{ + auto& failure = failed_tiles_[key]; + failure.next_endpoint_offset = + endpoint_offset == std::numeric_limits::max() + ? 0 + : endpoint_offset + 1; + if (terminal) + { + failure.terminal_endpoints.insert(endpoint); + if (!policy_rejected_url.isEmpty()) + { + failure.policy_rejected_origins.insert( + endpoint, + imagery::TileNetworkManager::canonicalOrigin( + policy_rejected_url)); + } + else + { + failure.policy_rejected_origins.remove(endpoint); + } + } + auto const endpoint_count = + source_ ? int(source_->tile_urls.size()) : 1; + auto const permanent = + terminal + && failure.terminal_endpoints.size() >= endpoint_count; + recordFailure( + key, permanent, std::move(message)); +} + +void OnlineRasterTemplate::clearFailure(const OnlineRasterTileKey& key) +{ + failed_tiles_.remove(key); + offline_tiles_.remove(key); + scheduleNextRetry(); +} + +void OnlineRasterTemplate::trimFailureHistory() +{ + if (failed_tiles_.size() <= max_failure_records) + return; + + QVector removable; + removable.reserve(failed_tiles_.size() - max_failure_records); + for (auto found = failed_tiles_.cbegin(); found != failed_tiles_.cend(); ++found) + { + auto const needed_for_view = + source_ && !wanted_window_.isEmpty() && keyNeededForWindow(found.key(), wanted_window_); + if (!needed_for_view && !output_keys_.contains(found.key())) + removable.push_back(found.key()); + } + for (auto const& key : std::as_const(removable)) + { + if (failed_tiles_.size() <= max_failure_records) + break; + failed_tiles_.remove(key); + } +} + +qint64 OnlineRasterTemplate::encodedTileResponseLimit(const OnlineRasterTileKey& key) const noexcept +{ + auto const* tile_matrix = matrix(key.zoom); + if (!tile_matrix) + return 1024 * 1024; + auto const decoded = + rgbaImageBytes(qint64(tile_matrix->tile_size.width()) + 2, + qint64(tile_matrix->tile_size.height()) + 2); + if (!decoded) + return max_encoded_tile_response_bytes; + return std::clamp(*decoded * 2, qint64(1024 * 1024), + max_encoded_tile_response_bytes); +} + +std::shared_ptr +OnlineRasterTemplate::reserveEncodedTilePayload(QByteArray bytes) +{ + auto const byte_count = qint64(bytes.size()); + if (byte_count <= 0 || byte_count > max_encoded_tile_response_bytes) + return {}; + auto current = decode_bytes_in_flight_->load(std::memory_order_relaxed); + while (current <= max_decode_encoded_bytes - byte_count) + { + if (decode_bytes_in_flight_->compare_exchange_weak( + current, current + byte_count, std::memory_order_relaxed)) + { + auto payload = std::make_shared(); + payload->bytes = std::move(bytes); + payload->bytes_in_flight = decode_bytes_in_flight_; + payload->byte_count = byte_count; + return payload; + } + } + return {}; +} + +std::shared_ptr +OnlineRasterTemplate::reserveRetainedMemory(qint64 bytes) const +{ + if (bytes <= 0 || bytes > max_retained_raster_bytes) + return {}; + auto counter = sharedRetainedRasterBytes(); + auto try_reserve = [&]() -> std::shared_ptr { + auto current = counter->load(std::memory_order_relaxed); + while (current <= max_retained_raster_bytes - bytes) + { + if (counter->compare_exchange_weak( + current, current + bytes, std::memory_order_relaxed)) + { + auto reservation = std::make_shared(); + reservation->counter = counter; + reservation->bytes = bytes; + return reservation; + } + } + return {}; + }; + if (auto reservation = try_reserve()) + return reservation; + + // Admission must be able to replace stale cache entries when the shared + // budget is full. All online templates and their retained images live on + // this application thread, so reclaim least-recently-used source tiles + // across templates before treating the pressure as transient. + auto const current = counter->load(std::memory_order_relaxed); + auto remaining = + std::max(1, current - (max_retained_raster_bytes - bytes)); + auto candidates = liveOnlineRasterTemplates().values(); + std::sort( + candidates.begin(), + candidates.end(), + [](auto* first, auto* second) { + if (first->retained_access_ != second->retained_access_) + { + return first->retained_access_ + < second->retained_access_; + } + return std::less {}( + first, second); + }); + for (auto* candidate : std::as_const(candidates)) + { + if (!candidate) + continue; + remaining -= candidate->evictRetainedMemory(remaining); + if (remaining <= 0) + break; + } + return try_reserve(); +} + +qint64 OnlineRasterTemplate::evictRetainedMemory(qint64 target_bytes) +{ + if (target_bytes <= 0) + return 0; + auto const counter = sharedRetainedRasterBytes(); + auto const before = counter->load(std::memory_order_relaxed); + + auto const original_max_cost = tile_cache_.maxCost(); + auto released = qint64(0); + while (released < target_bytes + && tile_cache_.totalCost() > 0) + { + auto const remaining = target_bytes - released; + auto const target_kib = std::min( + std::numeric_limits::max(), + std::max(1, (remaining + 1023) / 1024)); + auto const next_cost = std::max( + 0, + tile_cache_.totalCost() + - std::min( + tile_cache_.totalCost(), + qsizetype(target_kib))); + tile_cache_.setMaxCost(next_cost); + released = + before - counter->load(std::memory_order_relaxed); + } + tile_cache_.setMaxCost(original_max_cost); + + if (released < target_bytes && atlas_.memory + && !atlas_.output_owned && !atlas_cancelled_) + { + auto const dirty_bounds = atlas_.map_bounds; + atlas_.clear(); + if (dirty_bounds.isValid()) + { + map->setTemplateAreaDirty( + this, dirty_bounds, + getTemplateBoundingBoxPixelBorder()); + } + released = + before - counter->load(std::memory_order_relaxed); + } + return std::max(0, released); +} + +void OnlineRasterTemplate::scheduleNextRetry() +{ + int next_delay = -1; + for (auto const& failure : std::as_const(failed_tiles_)) + { + if (failure.permanent || failure.retry.hasExpired()) + continue; + auto const remaining = int( + std::clamp(failure.retry.remainingTime(), 0, std::numeric_limits::max())); + if (next_delay < 0 || remaining < next_delay) + next_delay = remaining; + } + if (next_delay < 0) + retry_timer_.stop(); + else if (!retry_timer_.isActive() || retry_timer_.remainingTime() > next_delay) + retry_timer_.start(next_delay); +} + +void OnlineRasterTemplate::updateResourceStatus() +{ + TemplateResourceStatus status; + auto const window = + output_preparation_active_ && !output_window_.isEmpty() ? output_window_ : wanted_window_; + if (!sourceReady() || window.isEmpty()) + { + setResourceStatus(std::move(status)); + return; + } + + QString permanent_message; + QString transient_message; + for (qint64 row = window.min_row; row <= window.max_row; ++row) + { + for (qint64 column = window.min_column; column <= window.max_column; ++column) + { + OnlineRasterTileKey key{ window.zoom, column, row }; + if (tile_cache_.contains(key)) + { + ++status.ready_resources; + continue; + } + if (offline_tiles_.contains(key)) + { + ++status.offline_resources; + continue; + } + auto const failure = failed_tiles_.constFind(key); + if (failure != failed_tiles_.cend()) + { + if (failure->permanent) + { + ++status.permanent_failures; + if (permanent_message.isEmpty()) + permanent_message = failure->message; + } + else + { + ++status.transient_failures; + if (transient_message.isEmpty()) + transient_message = failure->message; + } + continue; + } + // Missing demand counts as loading even before admission to the + // bounded network/decode queues. + ++status.loading_resources; + } + } + if (atlas_cancelled_ || atlas_queue_busy_) + ++status.loading_resources; + else if (!atlas_failed_signature_.isEmpty()) + { + ++status.permanent_failures; + permanent_message = + tr("The translucent imagery cannot be reprojected within the bounded policy."); + } + + auto countForTranslation = [](qsizetype count) { + return int(std::min(count, std::numeric_limits::max())); + }; + if (status.permanent_failures > 0) + { + status.message = + permanent_message.isEmpty() + ? tr("%n imagery tile(s) cannot be loaded.", nullptr, + countForTranslation(status.permanent_failures)) + : permanent_message; + } + else if (status.offline_resources > 0) + { + status.message = tr("%n imagery tile(s) are waiting for network access.", nullptr, + countForTranslation(status.offline_resources)); + } + else if (status.transient_failures > 0) + { + status.message = + transient_message.isEmpty() + ? tr("%n imagery tile(s) will be retried.", nullptr, + countForTranslation(status.transient_failures)) + : transient_message; + } + else if (status.loading_resources > 0) + { + status.message = tr("Loading online imagery..."); + } + setResourceStatus(std::move(status)); +} + +void OnlineRasterTemplate::queueWindow(const TileWindow& window, bool replace_pending) +{ + if (replace_pending) + cancelUnwantedWork(window); + if (window.isEmpty()) + { + updateResourceStatus(); + return; + } + + struct Missing + { + OnlineRasterTileKey key; + imagery::TileRequestPriority priority = imagery::TileRequestPriority::Visible; + double distance = 0; + }; + QVector missing; + QSet planned; + auto const center_column = 0.5 * (window.min_column + window.max_column); + auto const center_row = 0.5 * (window.min_row + window.max_row); + + auto plan = [this, &missing, &planned](const OnlineRasterTileKey& key, + imagery::TileRequestPriority priority, double distance) { + if (!tileAllowed(key) || tile_cache_.contains(key) || queued_tiles_.contains(key) + || planned.contains(key) || !retryAllowed(key)) + { + return; + } + planned.insert(key); + missing.push_back({ key, priority, distance }); + }; + + for (qint64 row = window.min_row; row <= window.max_row; ++row) + { + for (qint64 column = window.min_column; column <= window.max_column; ++column) + { + OnlineRasterTileKey key{ window.zoom, column, row }; + if (tile_cache_.contains(key)) + continue; + auto const dx = double(column) - center_column; + auto const dy = double(row) - center_row; + auto const distance = dx * dx + dy * dy; + OnlineRasterTileKey cached_key; + if (!bestCachedTile(key, &cached_key, nullptr)) + { + for (int zoom = source_->min_zoom; zoom < key.zoom; ++zoom) + { + auto const shift = key.zoom - zoom; + plan({ zoom, column >> shift, row >> shift }, + imagery::TileRequestPriority::Coverage, distance); + } + } + plan(key, imagery::TileRequestPriority::Visible, distance); + } + } + + std::stable_sort(missing.begin(), missing.end(), [](auto const& first, auto const& second) { + if (first.priority != second.priority) + return first.priority < second.priority; + if (first.priority == imagery::TileRequestPriority::Coverage + && first.key.zoom != second.key.zoom) + { + return first.key.zoom < second.key.zoom; + } + return first.distance < second.distance; + }); + + auto available = std::max(0, max_queued_tiles - queued_tiles_.size()); + if (decode_bytes_in_flight_->load(std::memory_order_relaxed) >= max_decode_encoded_bytes) + available = 0; + for (auto const& item : missing) + { + if (available == 0) + break; + queueTile(item.key, item.priority, item.distance); + --available; + } + updateResourceStatus(); +} + +void OnlineRasterTemplate::queueTile(const OnlineRasterTileKey& key, + imagery::TileRequestPriority priority, + double distance_priority) +{ + if (!source_ || source_->tile_urls.isEmpty()) + return; + auto signature = quint64(key.zoom); + signature = mixSignature(signature, quint64(key.column)); + signature = mixSignature(signature, quint64(key.row)); + auto const failure = failed_tiles_.constFind(key); + auto endpoint_offset = + failure == failed_tiles_.cend() + ? quint64(0) + : failure->next_endpoint_offset; + auto endpoint = -1; + for (qsizetype probe = 0; + probe < source_->tile_urls.size(); + ++probe) + { + auto const candidate_offset = + endpoint_offset + quint64(probe); + auto const candidate = int( + (signature + candidate_offset) + % quint64(source_->tile_urls.size())); + if (failure == failed_tiles_.cend() + || !failure->terminal_endpoints.contains(candidate)) + { + endpoint = candidate; + endpoint_offset = candidate_offset; + break; + } + } + if (endpoint < 0) + { + recordFailure( + key, true, + tr("All equivalent imagery endpoints failed.")); + return; + } + QString error; + auto const url = source_->tileUrl(endpoint, key.zoom, key.column, key.row, &error); + if (!url.isValid() || url.isEmpty()) + { + recordEndpointFailure( + key, endpoint, endpoint_offset, true, error); + return; + } + + imagery::TileNetworkRequest request; + request.url = url; + request.client_id = network_client_id_; + request.generation = generation_; + request.priority = priority; + request.distance_priority = distance_priority; + request.referer = source_->request.referer.toString(QUrl::FullyEncoded); + request.empty_http_status_codes = source_->request.empty_http_status_codes; + request.max_response_bytes = encodedTileResponseLimit(key); + auto const token = network_->submit(std::move(request)); + pending_fetches_.insert( + token, + { key, generation_, endpoint, endpoint_offset }); + queued_tiles_.insert(key); +} + +void OnlineRasterTemplate::onNetworkFinished(imagery::TileNetworkManager::Token token, + const imagery::TileNetworkResult& result) +{ + auto const found = pending_fetches_.find(token); + if (found == pending_fetches_.end()) + return; + auto const pending = *found; + pending_fetches_.erase(found); + if (pending.generation != generation_ || result.generation != pending.generation) + { + queued_tiles_.remove(pending.key); + return; + } + + switch (result.outcome) + { + case imagery::TileNetworkResult::Outcome::Success: + queueDecode( + pending.key, + result.body, + pending.generation, + pending.endpoint, + pending.endpoint_offset); + return; + case imagery::TileNetworkResult::Outcome::NotModified: + queued_tiles_.remove(pending.key); + recordEndpointFailure( + pending.key, + pending.endpoint, + pending.endpoint_offset, + true, + tr("The imagery server returned an unusable not-modified response.")); + queueWindow(wanted_window_, false); + return; + case imagery::TileNetworkResult::Outcome::EmptyTile: + insertEmptyTile(pending.key); + return; + case imagery::TileNetworkResult::Outcome::Cancelled: + queued_tiles_.remove(pending.key); + queueWindow(wanted_window_, false); + return; + case imagery::TileNetworkResult::Outcome::OfflineMiss: + queued_tiles_.remove(pending.key); + failed_tiles_.remove(pending.key); + offline_tiles_.insert(pending.key); + if (offline_tiles_.size() > max_failure_records) + { + for (auto found = offline_tiles_.begin(); + found != offline_tiles_.end() && offline_tiles_.size() > max_failure_records;) + { + auto const needed_for_view = + !wanted_window_.isEmpty() && keyNeededForWindow(*found, wanted_window_); + if (!needed_for_view && !output_keys_.contains(*found)) + found = offline_tiles_.erase(found); + else + ++found; + } + } + scheduleNextRetry(); + updateResourceStatus(); + return; + case imagery::TileNetworkResult::Outcome::Busy: + queued_tiles_.remove(pending.key); + recordFailure(pending.key, false, result.error_string); + queueWindow(wanted_window_, false); + return; + case imagery::TileNetworkResult::Outcome::TransientError: + queued_tiles_.remove(pending.key); + recordEndpointFailure( + pending.key, + pending.endpoint, + pending.endpoint_offset, + false, + result.error_string); + queueWindow(wanted_window_, false); + return; + case imagery::TileNetworkResult::Outcome::PermanentError: + queued_tiles_.remove(pending.key); + recordEndpointFailure( + pending.key, + pending.endpoint, + pending.endpoint_offset, + true, + result.error_string); + queueWindow(wanted_window_, false); + return; + case imagery::TileNetworkResult::Outcome::Rejected: + queued_tiles_.remove(pending.key); + recordEndpointFailure( + pending.key, + pending.endpoint, + pending.endpoint_offset, + true, + result.error_string, + result.private_network_rejected + ? result.private_network_rejected_url + : QUrl {}); + queueWindow(wanted_window_, false); + return; + } +} + +std::optional +OnlineRasterTemplate::decodeTile(const QByteArray& bytes, const QSize& expected_size, + const RasterResourceManager::CancellationToken& cancellation, + const std::shared_ptr& source_cancelled) +{ + auto const is_cancelled = [&] { + return cancellation.isCancelled() + || source_cancelled->load(std::memory_order_relaxed); + }; + if (is_cancelled() || bytes.isEmpty() || expected_size.isEmpty()) + return std::nullopt; + if (expected_size.width() > max_tile_dimension || expected_size.height() > max_tile_dimension + || qint64(expected_size.width()) * expected_size.height() > max_tile_pixels) + { + return std::nullopt; + } + QBuffer buffer; + buffer.setData(bytes); + if (!buffer.open(QIODevice::ReadOnly)) + return std::nullopt; + QImageReader reader(&buffer); + reader.setAutoTransform(false); + reader.setDecideFormatFromContent(true); + auto const advertised_size = reader.size(); + if (!advertised_size.isValid() || advertised_size != expected_size) + return std::nullopt; + if (is_cancelled()) + return std::nullopt; + auto decoded = reader.read(); + if (decoded.size() != expected_size || is_cancelled()) + return std::nullopt; + auto const opaque = imageIsOpaque(decoded, cancellation, source_cancelled); + if (is_cancelled()) + return std::nullopt; + auto padded = addGutter(std::move(decoded)); + if (padded.isNull() || is_cancelled()) + return std::nullopt; + return CachedTile{ std::move(padded), opaque, false, {} }; +} + +void OnlineRasterTemplate::queueDecode( + const OnlineRasterTileKey& key, + QByteArray bytes, + quint64 generation, + int endpoint, + quint64 endpoint_offset) +{ + auto const* tile_matrix = matrix(key.zoom); + if (!tile_matrix) + { + queued_tiles_.remove(key); + return; + } + if (bytes.isEmpty() || qint64(bytes.size()) > encodedTileResponseLimit(key)) + { + queued_tiles_.remove(key); + recordEndpointFailure( + key, + endpoint, + endpoint_offset, + true, + tr("The imagery tile response is empty or exceeds the " + "bounded decode policy.")); + queueWindow(wanted_window_, false); + return; + } + auto payload = reserveEncodedTilePayload(std::move(bytes)); + if (!payload) + { + queued_tiles_.remove(key); + recordFailure(key, false, tr("The imagery decode memory budget is temporarily full.")); + queueWindow(wanted_window_, false); + return; + } + auto const expected_size = tile_matrix->tile_size; + auto const core_bytes = rgbaImageBytes( + expected_size.width(), expected_size.height()); + auto const padded_bytes = rgbaImageBytes( + qint64(expected_size.width()) + 2, + qint64(expected_size.height()) + 2); + auto working_memory = + core_bytes && padded_bytes + ? reserveRetainedMemory( + *core_bytes + *padded_bytes) + : std::shared_ptr {}; + if (!working_memory) + { + queued_tiles_.remove(key); + recordFailure( + key, false, + tr("The imagery decode memory budget is temporarily full.")); + queueWindow(wanted_window_, false); + return; + } + auto cancelled = std::make_shared(false); + pending_decodes_.insert( + key, + { cancelled, endpoint, endpoint_offset }); + auto const accepted = RasterResourceManager::instance().submit( + decode_owner_, RasterResourceManager::Lane::Decode, + RasterResourceManager::Priority::Visible, this, + [payload = std::move(payload), expected_size, key, generation, cancelled, + working_memory = std::move(working_memory), + receiver = this](const RasterResourceManager::CancellationToken& cancellation) mutable { + auto tile = cancelled->load(std::memory_order_relaxed) + ? std::optional{} + : decodeTile( + payload->bytes, expected_size, cancellation, cancelled); + if (tile) + { + auto const retained_bytes = + qint64(tile->image.bytesPerLine()) + * tile->image.height(); + working_memory->shrinkTo(retained_bytes); + tile->memory = std::move(working_memory); + } + return RasterResourceManager::Completion{ [receiver, key, tile = std::move(tile), + generation, cancelled]() mutable { + receiver->finishDecode(key, std::move(tile), generation, cancelled); + } }; + }); + if (!accepted) + { + pending_decodes_.remove(key); + queued_tiles_.remove(key); + recordFailure(key, false, tr("The imagery decode queue is temporarily full.")); + queueWindow(wanted_window_, false); + } +} + +void OnlineRasterTemplate::finishDecode(const OnlineRasterTileKey& key, + std::optional tile, quint64 generation, + const std::shared_ptr& cancelled) +{ + auto found = pending_decodes_.find(key); + if (found == pending_decodes_.end() + || found->cancelled != cancelled) + return; + auto const endpoint = found->endpoint; + auto const endpoint_offset = found->endpoint_offset; + pending_decodes_.erase(found); + if (generation != generation_) + { + queued_tiles_.remove(key); + return; + } + if (cancelled->load(std::memory_order_relaxed)) + { + queued_tiles_.remove(key); + queueWindow(wanted_window_, false); + return; + } + if (!tile) + { + queued_tiles_.remove(key); + recordEndpointFailure( + key, + endpoint, + endpoint_offset, + true, + tr("The tile image is invalid or has unexpected dimensions.")); + queueWindow(wanted_window_, false); + return; + } + insertTile(key, std::move(*tile)); +} + +void OnlineRasterTemplate::insertTile(const OnlineRasterTileKey& key, CachedTile tile) +{ + queued_tiles_.remove(key); + clearFailure(key); + if (!tile.empty && !tile.memory) + { + auto const bytes = qint64(tile.image.bytesPerLine()) * tile.image.height(); + tile.memory = reserveRetainedMemory(bytes); + if (!tile.memory) + { + if (output_preparation_active_ + && output_keys_.contains(key)) + { + output_preparation_error_ = tr( + "Exact imagery cannot fit within the " + "application-wide raster memory budget."); + updateResourceStatus(); + return; + } + recordFailure( + key, false, + tr("The application-wide imagery memory budget is temporarily full.")); + queueWindow(wanted_window_, false); + return; + } + } + auto const cost = tile.empty ? 1 : cacheCostKiB(tile.image); + if (cost <= 0 || cost > tile_cache_.maxCost() + || !tile_cache_.insert(key, new CachedTile(std::move(tile)), cost)) + { + recordFailure(key, true, tr("The decoded tile exceeds the imagery memory-cache policy.")); + return; + } + retained_access_ = nextRetainedAccess(); + if (output_preparation_active_ + && !output_source_tiles_released_ + && output_keys_.contains(key)) + { + if (auto const* cached = tile_cache_.object(key)) + output_tiles_.insert(key, *cached); + } + if (!(output_preparation_active_ && (atlas_.output_owned || atlas_pending_for_output_))) + { + cancelAtlasBuild(); + atlas_.clear(); + } + markTileDirty(key); + queueWindow(wanted_window_, false); +} + +void OnlineRasterTemplate::insertEmptyTile(const OnlineRasterTileKey& key) +{ + auto const* tile_matrix = matrix(key.zoom); + if (!tile_matrix) + { + queued_tiles_.remove(key); + return; + } + insertTile(key, { {}, true, true, {} }); +} + +const OnlineRasterTemplate::CachedTile* +OnlineRasterTemplate::bestCachedTile(const OnlineRasterTileKey& requested, + OnlineRasterTileKey* cached_key, QRectF* source_rect) const +{ + auto const* requested_matrix = matrix(requested.zoom); + if (!requested_matrix) + return nullptr; + if (output_preparation_active_) + { + auto const pinned = output_tiles_.constFind(requested); + if (pinned != output_tiles_.cend()) + { + if (cached_key) + *cached_key = requested; + if (source_rect) + { + *source_rect = pinned->empty + ? QRectF{} + : QRectF(1, 1, requested_matrix->tile_size.width(), + requested_matrix->tile_size.height()); + } + return &*pinned; + } + } + if (auto const* exact = tile_cache_.object(requested)) + { + if (cached_key) + *cached_key = requested; + if (source_rect) + { + *source_rect = exact->empty ? QRectF{} + : QRectF(1, 1, requested_matrix->tile_size.width(), + requested_matrix->tile_size.height()); + } + return exact; + } + for (int zoom = requested.zoom - 1; zoom >= source_->min_zoom; --zoom) + { + auto const shift = requested.zoom - zoom; + OnlineRasterTileKey candidate{ zoom, requested.column >> shift, requested.row >> shift }; + auto const* cached = tile_cache_.object(candidate); + auto const* candidate_matrix = matrix(zoom); + if (!cached || cached->empty || !candidate_matrix) + continue; + auto const divisor = double(qint64(1) << shift); + auto const width = candidate_matrix->tile_size.width() / divisor; + auto const height = candidate_matrix->tile_size.height() / divisor; + auto const local_column = requested.column - (candidate.column << shift); + auto const local_row = requested.row - (candidate.row << shift); + if (cached_key) + *cached_key = candidate; + if (source_rect) + { + *source_rect = QRectF(1 + local_column * width, 1 + local_row * height, width, height); + } + return cached; + } + return nullptr; +} + +QVector +OnlineRasterTemplate::visualTiles(const TileWindow& window, bool allow_provisional, + bool* has_transparency, bool* has_missing, bool* has_pixels) const +{ + QVector result; + if (has_transparency) + *has_transparency = false; + if (has_missing) + *has_missing = false; + if (has_pixels) + *has_pixels = false; + if (window.isEmpty()) + return result; + auto const count = tileCount(window); + if (!count || *count > std::numeric_limits::max()) + return result; + result.reserve(int(*count)); + auto touched_pixels = false; + for (qint64 row = window.min_row; row <= window.max_row; ++row) + { + for (qint64 column = window.min_column; column <= window.max_column; ++column) + { + OnlineRasterTileKey requested{ window.zoom, column, row }; + OnlineRasterTileKey cached; + QRectF source_rect; + auto const pinned = output_preparation_active_ + ? output_tiles_.constFind(requested) + : output_tiles_.cend(); + auto const* tile = pinned != output_tiles_.cend() + ? &*pinned + : tile_cache_.object(requested); + if (tile) + { + cached = requested; + if (tile->empty) + { + result.push_back({ requested, cached, tile, {}, false, true }); + continue; + } + auto const* requested_matrix = matrix(requested.zoom); + source_rect = QRectF(1, 1, requested_matrix->tile_size.width(), + requested_matrix->tile_size.height()); + } + else if (allow_provisional) + { + tile = bestCachedTile(requested, &cached, &source_rect); + } + if (!tile) + { + if (has_missing) + *has_missing = true; + result.push_back({ requested, {}, nullptr, {}, false, false }); + continue; + } + if (has_pixels) + *has_pixels = true; + touched_pixels = true; + if (has_transparency && !tile->opaque) + *has_transparency = true; + result.push_back( + { requested, cached, tile, source_rect, cached.zoom != requested.zoom, false }); + } + } + if (touched_pixels) + retained_access_ = nextRetainedAccess(); + return result; +} + +bool OnlineRasterTemplate::appendOpaquePatches( + const VisualTile& visual, + double pixels_per_map_unit, + const std::shared_ptr& output_render_memory, + QVector& out) const +{ + if (!visual.tile) + return false; + auto const first = out.size(); + if (appendOpaquePatch( + visual, visual.source_rect, pixels_per_map_unit, 0, + output_render_memory, out)) + { + return true; + } + out.resize(first); + return false; +} + +bool OnlineRasterTemplate::appendOpaquePatch(const VisualTile& visual, QRectF source_rect, + double pixels_per_map_unit, int depth, + const std::shared_ptr& output_render_memory, + QVector& out) const +{ + double residual = 0; + auto transform = imageRectToMap(visual.cached, source_rect, nullptr, &residual); + if (!transform) + return false; + auto const exceeds_tolerance = residual * pixels_per_map_unit > 0.35; + auto const columns = source_rect.width() > 16 ? 2 : 1; + auto const rows = source_rect.height() > 16 ? 2 : 1; + if (depth < 6 && exceeds_tolerance && (columns > 1 || rows > 1)) + { + auto const patch_width = source_rect.width() / columns; + auto const patch_height = source_rect.height() / rows; + for (int y = 0; y < rows; ++y) + { + for (int x = 0; x < columns; ++x) + { + auto patch = QRectF(source_rect.x() + x * patch_width, + source_rect.y() + y * patch_height, patch_width, patch_height); + if (!appendOpaquePatch( + visual, patch, pixels_per_map_unit, depth + 1, + output_render_memory, out)) + { + return false; + } + } + } + return true; + } + if (exceeds_tolerance) + return false; + + auto padded_rect = source_rect.adjusted(-0.75, -0.75, 0.75, 0.75); + padded_rect = padded_rect.intersected(QRectF(QPointF(0, 0), QSizeF(visual.tile->image.size()))); + QRectF map_bounds; + transform = imageRectToMap(visual.cached, padded_rect, &map_bounds, nullptr); + if (!transform) + return false; + RasterMemoryReserver reserve_render_memory; + if (output_render_memory) + { + reserve_render_memory = + [memory = output_render_memory](qint64 bytes) + -> std::shared_ptr { + if (bytes <= 0 || bytes > memory->bytes) + return {}; + return memory; + }; + } + else + { + reserve_render_memory = + [this](qint64 bytes) + -> std::shared_ptr { + return reserveRetainedMemory(bytes); + }; + } + out.push_back({ + visual.tile->image, + map_bounds, + padded_rect, + quint64(visual.tile->image.cacheKey()), + false, + visual.provisional, + *transform, + true, + visual.tile->memory, + std::move(reserve_render_memory), + }); + return true; +} + +QVector OnlineRasterTemplate::atlasSignature(const TileWindow& window, + const QVector& visuals, + bool has_missing) const +{ + QVector signature; + signature.reserve(visuals.size() + 6); + signature.push_back(quint64(window.zoom)); + signature.push_back(quint64(window.min_column)); + signature.push_back(quint64(window.min_row)); + signature.push_back(quint64(window.max_column)); + signature.push_back(quint64(window.max_row)); + signature.push_back(has_missing ? 1 : 0); + for (auto const& visual : visuals) + { + auto value = visual.tile ? quint64(visual.tile->image.cacheKey()) : 0; + value = mixSignature(value, visual.complete_empty ? 1 : 0); + value = mixSignature(value, quint64(visual.cached.zoom + 1)); + value = mixSignature(value, quint64(visual.cached.column)); + value = mixSignature(value, quint64(visual.cached.row)); + signature.push_back(value); + } + return signature; +} + +QVector +OnlineRasterTemplate::atlasChunks( + const TileWindow& window, + double pixels_per_map_unit) const +{ + QVector result; + auto const* tile_matrix = matrix(window.zoom); + if (window.isEmpty() || !tile_matrix) + return result; + + auto const target_pixels = + std::max(1, max_atlas_pixels / 4); + auto const target_dimension = std::max( + 1, + std::min( + max_atlas_dimension - 2, + qint64(std::floor(std::sqrt( + double(target_pixels)))))); + auto projectedSize = [&](const TileWindow& chunk) + -> std::optional { + if (!(pixels_per_map_unit > 0) + || !std::isfinite(pixels_per_map_unit)) + return std::nullopt; + auto const top_left = tileBounds({ + chunk.zoom, + chunk.min_column, + chunk.min_row, + }); + if (!top_left.isValid()) + return std::nullopt; + auto const source_bounds = imagery::CrsBounds { + top_left.west, + top_left.north + - chunk.height() + * tile_matrix->tile_size.height() + * tile_matrix->cell_size, + top_left.west + + chunk.width() + * tile_matrix->tile_size.width() + * tile_matrix->cell_size, + top_left.north, + }; + auto const map_bounds = + mapBoundsForSourceBounds(source_bounds); + if (!map_bounds.isValid() || map_bounds.isEmpty()) + return std::nullopt; + return QSizeF( + std::ceil( + map_bounds.width() + * pixels_per_map_unit), + std::ceil( + map_bounds.height() + * pixels_per_map_unit)); + }; + + std::function appendChunk = + [&](const TileWindow& chunk) { + auto const source_width = + chunk.width() + * tile_matrix->tile_size.width(); + auto const source_height = + chunk.height() + * tile_matrix->tile_size.height(); + auto const projected = projectedSize(chunk); + auto const projected_width = + projected + ? projected->width() + : 0; + auto const projected_height = + projected + ? projected->height() + : 0; + auto const source_fits = + source_width <= target_dimension + && source_height <= target_dimension + && source_width + <= target_pixels + / std::max( + 1, source_height); + auto const projected_fits = + !projected + || (projected_width + <= target_dimension + && projected_height + <= target_dimension + && projected_width + <= double(target_pixels) + / std::max( + 1.0, + projected_height)); + if ((source_fits && projected_fits) + || (chunk.width() == 1 + && chunk.height() == 1)) + { + result.push_back(chunk); + return; + } + + auto const column_pressure = std::max( + double(source_width) + / target_dimension, + projected_width + / target_dimension); + auto const row_pressure = std::max( + double(source_height) + / target_dimension, + projected_height + / target_dimension); + if (chunk.width() > 1 + && (chunk.height() == 1 + || column_pressure >= row_pressure)) + { + auto const middle = + chunk.min_column + + (chunk.max_column + - chunk.min_column) + / 2; + appendChunk({ + chunk.zoom, + chunk.min_column, + middle, + chunk.min_row, + chunk.max_row, + }); + appendChunk({ + chunk.zoom, + middle + 1, + chunk.max_column, + chunk.min_row, + chunk.max_row, + }); + } + else + { + auto const middle = + chunk.min_row + + (chunk.max_row + - chunk.min_row) + / 2; + appendChunk({ + chunk.zoom, + chunk.min_column, + chunk.max_column, + chunk.min_row, + middle, + }); + appendChunk({ + chunk.zoom, + chunk.min_column, + chunk.max_column, + middle + 1, + chunk.max_row, + }); + } + }; + appendChunk(window); + return result; +} + +std::optional OnlineRasterTemplate::makeAtlasBuildRequest( + const TileWindow& window, const QVector& visuals, bool has_missing, + double pixels_per_map_unit, const QVector& signature) const +{ + auto const* tile_matrix = matrix(window.zoom); + if (!tile_matrix || !(pixels_per_map_unit > 0) || !std::isfinite(pixels_per_map_unit)) + { + return std::nullopt; + } + auto const tile_count = tileCount(window); + auto const tile_width = qint64(tile_matrix->tile_size.width()); + auto const tile_height = qint64(tile_matrix->tile_size.height()); + if (!tile_count || tile_width <= 0 || tile_height <= 0 + || window.width() > std::numeric_limits::max() / tile_width + || window.height() > std::numeric_limits::max() / tile_height) + { + return std::nullopt; + } + auto const width = window.width() * tile_width; + auto const height = window.height() * tile_height; + if (width <= 0 || height <= 0 + || width > max_atlas_dimension - 2 + || height > max_atlas_dimension - 2 + || width + 2 > max_atlas_pixels / (height + 2) + || width > std::numeric_limits::max() - 2 + || height > std::numeric_limits::max() - 2) + { + return std::nullopt; + } + + AtlasBuildRequest request; + request.window = window; + request.signature = signature; + request.core_size = { int(width), int(height) }; + request.pixels_per_map_unit = pixels_per_map_unit; + bool provisional = has_missing; + for (auto const& visual : visuals) + { + request.has_left_neighbor |= + visual.requested.column < window.min_column; + request.has_right_neighbor |= + visual.requested.column > window.max_column; + request.has_top_neighbor |= + visual.requested.row < window.min_row; + request.has_bottom_neighbor |= + visual.requested.row > window.max_row; + if (!visual.tile || visual.complete_empty) + continue; + request.visuals.push_back({ + QRectF(1 + (visual.requested.column - window.min_column) + * tile_matrix->tile_size.width(), + 1 + (visual.requested.row - window.min_row) + * tile_matrix->tile_size.height(), + tile_matrix->tile_size.width(), tile_matrix->tile_size.height()), + visual.tile->image, + visual.source_rect, + visual.tile->memory, + }); + provisional |= visual.provisional; + } + request.provisional = provisional; + + auto const top_left_key = OnlineRasterTileKey{ window.zoom, window.min_column, window.min_row }; + auto const bounds = tileBounds(top_left_key); + if (!bounds.isValid()) + return std::nullopt; + auto const core_west = bounds.west; + auto const core_north = bounds.north; + auto const core_east = + bounds.west + window.width() * tile_matrix->tile_size.width() * tile_matrix->cell_size; + auto const core_south = + bounds.north - window.height() * tile_matrix->tile_size.height() * tile_matrix->cell_size; + auto const source_west = core_west - tile_matrix->cell_size; + auto const source_north = core_north + tile_matrix->cell_size; + auto const source_east = core_east + tile_matrix->cell_size; + auto const source_south = core_south - tile_matrix->cell_size; + auto const map_top_left = nominalSourceToMap({ source_west, source_north }); + auto const map_top_right = nominalSourceToMap({ source_east, source_north }); + auto const map_bottom_left = nominalSourceToMap({ source_west, source_south }); + auto const map_bottom_right = nominalSourceToMap({ source_east, source_south }); + if (!map_top_left || !map_top_right || !map_bottom_left || !map_bottom_right) + return std::nullopt; + + auto const padded_size = QSize(int(width) + 2, int(height) + 2); + QTransform transform((map_top_right->x() - map_top_left->x()) / padded_size.width(), + (map_top_right->y() - map_top_left->y()) / padded_size.width(), + (map_bottom_left->x() - map_top_left->x()) / padded_size.height(), + (map_bottom_left->y() - map_top_left->y()) / padded_size.height(), + map_top_left->x(), map_top_left->y()); + if (!transform.isInvertible()) + return std::nullopt; + + double residual = 0; + for (int y = 0; y <= 4; ++y) + { + for (int x = 0; x <= 4; ++x) + { + auto const sample = + QPointF(padded_size.width() * x / 4.0, padded_size.height() * y / 4.0); + auto const actual = + nominalSourceToMap({ source_west + sample.x() * tile_matrix->cell_size, + source_north - sample.y() * tile_matrix->cell_size }); + if (!actual) + return std::nullopt; + residual = std::max(residual, pointDistance(transform.map(sample), *actual)); + } + } + + if (residual * pixels_per_map_unit <= 0.35) + { + auto const padded_bytes = rgbaImageBytes(width + 2, height + 2); + if (!padded_bytes + || *padded_bytes > max_atlas_peak_bytes) + return std::nullopt; + request.working_memory = + reserveRetainedMemory(*padded_bytes); + if (!request.working_memory) + return std::nullopt; + request.image_to_map = transform; + request.map_bounds = + boundsOf({ *map_top_left, *map_top_right, *map_bottom_left, *map_bottom_right }); + return request; + } + + auto map_bounds = mapBoundsForSourceBounds({ core_west, core_south, core_east, core_north }); + if (!map_bounds.isValid() || map_bounds.isEmpty()) + return std::nullopt; + auto const output_width = qint64(std::ceil(map_bounds.width() * pixels_per_map_unit)); + auto const output_height = qint64(std::ceil(map_bounds.height() * pixels_per_map_unit)); + if (output_width <= 0 || output_height <= 0 || output_width > max_atlas_dimension + || output_height > max_atlas_dimension || output_width > max_atlas_pixels / output_height + || output_width > std::numeric_limits::max() + || output_height > std::numeric_limits::max()) + { + return std::nullopt; + } + auto const source_bytes = rgbaImageBytes(width + 2, height + 2); + auto const output_bytes = rgbaImageBytes(output_width, output_height); + auto const padded_bytes = rgbaImageBytes(output_width + 2, output_height + 2); + if (!source_bytes || !output_bytes || !padded_bytes + || *source_bytes > max_atlas_peak_bytes - *output_bytes + || *source_bytes + *output_bytes + > max_atlas_peak_bytes - *padded_bytes) + return std::nullopt; + request.working_memory = reserveRetainedMemory( + *source_bytes + *output_bytes + *padded_bytes); + if (!request.working_memory) + return std::nullopt; + + AtlasWarpGrid accepted_grid; + for (int cells = 8; cells <= 64; cells *= 2) + { + AtlasWarpGrid grid; + grid.columns = cells; + grid.rows = cells; + grid.output_size = { int(output_width), int(output_height) }; + grid.source_points.reserve((cells + 1) * (cells + 1)); + bool valid = true; + for (int y = 0; y <= cells && valid; ++y) + { + for (int x = 0; x <= cells; ++x) + { + auto const map_point = QPointF(map_bounds.left() + map_bounds.width() * x / cells, + map_bounds.top() + map_bounds.height() * y / cells); + auto const source_point = mapToNominalSource(map_point); + if (!source_point) + { + valid = false; + break; + } + grid.source_points.push_back({ + 1 + (source_point->x() - core_west) + / tile_matrix->cell_size, + 1 + (core_north - source_point->y()) + / tile_matrix->cell_size, + }); + } + } + if (!valid) + continue; + + double maximum_error = 0; + for (int y = 0; y < cells && valid; ++y) + { + for (int x = 0; x < cells; ++x) + { + auto const map_point = + QPointF(map_bounds.left() + map_bounds.width() * (x + 0.5) / cells, + map_bounds.top() + map_bounds.height() * (y + 0.5) / cells); + auto const exact_source = mapToNominalSource(map_point); + if (!exact_source) + { + valid = false; + break; + } + auto const exact = + QPointF( + 1 + (exact_source->x() - core_west) + / tile_matrix->cell_size, + 1 + (core_north - exact_source->y()) + / tile_matrix->cell_size); + auto const row_stride = cells + 1; + auto const interpolated = 0.25 + * (grid.source_points.at(y * row_stride + x) + + grid.source_points.at(y * row_stride + x + 1) + + grid.source_points.at((y + 1) * row_stride + x) + + grid.source_points.at((y + 1) * row_stride + x + 1)); + maximum_error = std::max(maximum_error, pointDistance(exact, interpolated)); + } + } + if (valid && maximum_error <= 0.25) + { + grid.maximum_interpolation_error = maximum_error; + accepted_grid = std::move(grid); + break; + } + } + if (accepted_grid.columns == 0) + return std::nullopt; + + auto const x_scale = map_bounds.width() / output_width; + auto const y_scale = map_bounds.height() / output_height; + auto const& map_georeferencing = map->getGeoreferencing(); + accepted_grid.exact_ownership = true; + accepted_grid.map_crs = + map_georeferencing.getProjectedCRSSpec(); + accepted_grid.source_crs = source_->tile_matrix_set.crs; + accepted_grid.map_to_projected = + map_georeferencing.mapToProjected(); + accepted_grid.map_bounds = map_bounds; + accepted_grid.source_registration = source_->registration + ? QPointF( + source_->registration->dx, + source_->registration->dy) + : QPointF {}; + accepted_grid.core_west = core_west; + accepted_grid.core_north = core_north; + accepted_grid.cell_size = tile_matrix->cell_size; + request.warp = std::move(accepted_grid); + request.image_to_map = + QTransform(x_scale, 0, 0, y_scale, map_bounds.left() - x_scale, map_bounds.top() - y_scale); + request.map_bounds = map_bounds.adjusted(-x_scale, -y_scale, x_scale, y_scale); + return request; +} + +std::optional +OnlineRasterTemplate::buildAtlas(AtlasBuildRequest request, + const std::shared_ptr& cancelled, + const RasterResourceManager::CancellationToken& cancellation) +{ + auto is_cancelled = [&] { + return cancellation.isCancelled() || cancelled->load(std::memory_order_relaxed); + }; + if (is_cancelled()) + return std::nullopt; + + QImage source( + request.core_size + QSize(2, 2), + QImage::Format_RGBA8888_Premultiplied); + if (source.isNull()) + return std::nullopt; + source.fill(Qt::transparent); + { + QPainter painter(&source); + painter.setCompositionMode(QPainter::CompositionMode_Source); + painter.setRenderHint(QPainter::SmoothPixmapTransform, true); + for (auto const& visual : request.visuals) + { + if (is_cancelled()) + return std::nullopt; + painter.drawImage(visual.target_rect, visual.image, visual.source_rect); + } + } + auto copyPixel = [&source](int destination_x, + int destination_y, + int source_x, + int source_y) { + std::copy_n( + source.constScanLine(source_y) + + 4 * source_x, + 4, + source.scanLine(destination_y) + + 4 * destination_x); + }; + if (!request.has_left_neighbor) + { + for (int y = 0; y < source.height(); ++y) + copyPixel(0, y, 1, y); + } + if (!request.has_right_neighbor) + { + for (int y = 0; y < source.height(); ++y) + copyPixel( + source.width() - 1, + y, + source.width() - 2, + y); + } + if (!request.has_top_neighbor) + { + for (int x = 0; x < source.width(); ++x) + copyPixel(x, 0, x, 1); + } + if (!request.has_bottom_neighbor) + { + for (int x = 0; x < source.width(); ++x) + copyPixel( + x, + source.height() - 1, + x, + source.height() - 2); + } + + QImage output; + if (!request.warp) + { + output = std::move(source); + } + else + { + auto const& grid = *request.warp; + output = QImage(grid.output_size, QImage::Format_RGBA8888_Premultiplied); + if (output.isNull()) + return std::nullopt; + output.fill(Qt::transparent); + auto const row_stride = grid.columns + 1; + std::optional map_projection; + std::optional source_projection; + if (grid.exact_ownership) + { + map_projection.emplace(grid.map_crs); + source_projection.emplace(grid.source_crs); + if (!map_projection->isValid() + || !source_projection->isValid() + || !grid.map_to_projected.isInvertible() + || !grid.map_bounds.isValid() + || grid.map_bounds.isEmpty() + || !(grid.cell_size > 0) + || !std::isfinite(grid.cell_size)) + { + return std::nullopt; + } + } + auto sample = [&source](int x, int y, int channel) { + if (x < 0 || y < 0 || x >= source.width() || y >= source.height()) + { + return 0.0; + } + return double(source.constScanLine(y)[4 * x + channel]); + }; + for (int y = 0; y < output.height(); ++y) + { + if (is_cancelled()) + return std::nullopt; + auto* destination = output.scanLine(y); + auto const grid_y = (y + 0.5) * grid.rows / output.height(); + auto const cell_y = std::min(grid.rows - 1, int(std::floor(grid_y))); + auto const fraction_y = grid_y - cell_y; + for (int x = 0; x < output.width(); ++x) + { + auto const grid_x = (x + 0.5) * grid.columns / output.width(); + auto const cell_x = std::min(grid.columns - 1, int(std::floor(grid_x))); + auto const fraction_x = grid_x - cell_x; + auto const top_left = grid.source_points.at(cell_y * row_stride + cell_x); + auto const top_right = grid.source_points.at(cell_y * row_stride + cell_x + 1); + auto const bottom_left = grid.source_points.at((cell_y + 1) * row_stride + cell_x); + auto const bottom_right = + grid.source_points.at((cell_y + 1) * row_stride + cell_x + 1); + auto const top = top_left * (1 - fraction_x) + top_right * fraction_x; + auto const bottom = bottom_left * (1 - fraction_x) + bottom_right * fraction_x; + auto const source_point = + top * (1 - fraction_y) + + bottom * fraction_y; + auto ownership_point = source_point; + if (grid.exact_ownership) + { + auto const right = + request.core_size.width() + 1.0; + auto const bottom_edge = + request.core_size.height() + 1.0; + auto const edge_distance = std::min({ + std::abs(source_point.x() - 1), + std::abs(source_point.x() - right), + std::abs(source_point.y() - 1), + std::abs(source_point.y() - bottom_edge), + }); + auto const exact_margin = std::max( + 1.0, + grid.maximum_interpolation_error + 0.75); + if (edge_distance <= exact_margin) + { + auto const map_point = QPointF( + grid.map_bounds.left() + + grid.map_bounds.width() + * (x + 0.5) + / output.width(), + grid.map_bounds.top() + + grid.map_bounds.height() + * (y + 0.5) + / output.height()); + bool inverse_ok = false; + bool forward_ok = false; + auto const lat_lon = + map_projection->inverse( + grid.map_to_projected.map( + map_point), + &inverse_ok); + auto nominal_source = + source_projection->forward( + lat_lon, &forward_ok); + if (!inverse_ok || !forward_ok + || !std::isfinite( + nominal_source.x()) + || !std::isfinite( + nominal_source.y())) + { + continue; + } + nominal_source -= + grid.source_registration; + ownership_point = { + 1 + + (nominal_source.x() + - grid.core_west) + / grid.cell_size, + 1 + + (grid.core_north + - nominal_source.y()) + / grid.cell_size, + }; + } + } + // Neighbor pixels exist only as bilinear filter support. + // Exact inverse ownership near every edge makes adjacent + // chunks share one half-open nominal-source partition. + if (ownership_point.x() < 1 + || ownership_point.y() < 1 + || ownership_point.x() + >= request.core_size.width() + 1 + || ownership_point.y() + >= request.core_size.height() + 1) + { + continue; + } + auto const source_x = source_point.x() - 0.5; + auto const source_y = source_point.y() - 0.5; + auto const x0 = int(std::floor(source_x)); + auto const y0 = int(std::floor(source_y)); + auto const fx = source_x - x0; + auto const fy = source_y - y0; + for (int channel = 0; channel < 4; ++channel) + { + auto const top_sample = + sample(x0, y0, channel) * (1 - fx) + sample(x0 + 1, y0, channel) * fx; + auto const bottom_sample = sample(x0, y0 + 1, channel) * (1 - fx) + + sample(x0 + 1, y0 + 1, channel) * fx; + destination[4 * x + channel] = uchar(std::clamp( + int(std::lround(top_sample * (1 - fy) + bottom_sample * fy)), 0, 255)); + } + } + } + } + auto padded = request.warp + ? addGutter(std::move(output)) + : std::move(output); + if (padded.isNull() || is_cancelled()) + return std::nullopt; + return AtlasBuildResult{ + request.window, + std::move(request.signature), + std::move(padded), + request.image_to_map, + request.map_bounds, + request.pixels_per_map_unit, + request.provisional, + std::move(request.working_memory), + }; +} + +void OnlineRasterTemplate::cancelAtlasBuild(bool clear_failure) const +{ + atlas_owner_.invalidate(); + if (atlas_cancelled_) + atlas_cancelled_->store(true, std::memory_order_relaxed); + atlas_cancelled_.reset(); + atlas_pending_signature_.clear(); + atlas_pending_scale_ = 0; + atlas_pending_for_output_ = false; + atlas_queue_busy_ = false; + atlas_retry_timer_.stop(); + if (clear_failure) + { + atlas_failed_signature_.clear(); + atlas_failed_scale_ = 0; + } +} + +bool OnlineRasterTemplate::queueAtlasBuild(const TileWindow& window, + const QVector& visuals, bool has_missing, + double pixels_per_map_unit, + const QVector& signature, bool for_output) const +{ + auto* receiver = const_cast(this); + auto scale_matches = [](double first, double second) { + auto const scale = std::max({ 1.0, std::abs(first), std::abs(second) }); + return std::abs(first - second) <= scale * 1.0e-9; + }; + if (atlas_cancelled_ && atlas_pending_signature_ == signature + && scale_matches(atlas_pending_scale_, pixels_per_map_unit)) + { + atlas_queue_busy_ = false; + atlas_pending_for_output_ |= for_output; + return true; + } + if (atlas_failed_signature_ == signature + && scale_matches(atlas_failed_scale_, pixels_per_map_unit)) + { + return false; + } + + cancelAtlasBuild(false); + atlas_queue_busy_ = false; + auto request = + makeAtlasBuildRequest(window, visuals, has_missing, pixels_per_map_unit, signature); + if (!request) + { + atlas_failed_signature_ = signature; + atlas_failed_scale_ = pixels_per_map_unit; + receiver->updateResourceStatus(); + return false; + } + + auto cancelled = std::make_shared(false); + atlas_cancelled_ = cancelled; + atlas_pending_signature_ = signature; + atlas_pending_scale_ = pixels_per_map_unit; + atlas_pending_for_output_ = for_output; + auto const accepted = RasterResourceManager::instance().submit( + atlas_owner_, RasterResourceManager::Lane::Decode, RasterResourceManager::Priority::Visible, + receiver, + [request = std::move(*request), cancelled, + receiver](const RasterResourceManager::CancellationToken& cancellation) mutable { + auto result = buildAtlas(std::move(request), cancelled, cancellation); + return RasterResourceManager::Completion{ [receiver, result = std::move(result), + cancelled]() mutable { + receiver->finishAtlasBuild(std::move(result), cancelled); + } }; + }); + if (!accepted) + { + cancelAtlasBuild(false); + atlas_queue_busy_ = true; + atlas_retry_timer_.start(100); + receiver->updateResourceStatus(); + return false; + } + receiver->updateResourceStatus(); + return true; +} + +void OnlineRasterTemplate::finishAtlasBuild(std::optional result, + const std::shared_ptr& cancelled) +{ + if (atlas_cancelled_ != cancelled) + return; + auto const pending_signature = atlas_pending_signature_; + auto const pending_scale = atlas_pending_scale_; + auto const pending_for_output = atlas_pending_for_output_; + atlas_cancelled_.reset(); + atlas_pending_signature_.clear(); + atlas_pending_scale_ = 0; + atlas_pending_for_output_ = false; + atlas_queue_busy_ = false; + if (cancelled->load(std::memory_order_relaxed)) + { + updateResourceStatus(); + return; + } + if (!result) + { + atlas_failed_signature_ = pending_signature; + atlas_failed_scale_ = pending_scale; + updateResourceStatus(); + return; + } + auto const retained_bytes = + qint64(result->image.bytesPerLine()) * result->image.height(); + auto memory = std::move(result->working_memory); + if (!memory) + { + if (pending_for_output) + { + output_preparation_error_ = tr( + "Exact translucent imagery cannot fit within the " + "application-wide raster memory budget."); + updateResourceStatus(); + return; + } + atlas_queue_busy_ = true; + atlas_retry_timer_.start(250); + updateResourceStatus(); + return; + } + memory->shrinkTo(retained_bytes); + + AtlasCache completed; + completed.window = result->window; + completed.signature = std::move(result->signature); + completed.image = std::move(result->image); + completed.image_to_map = result->image_to_map; + completed.map_bounds = result->map_bounds; + completed.pixels_per_map_unit = + result->pixels_per_map_unit; + completed.provisional = result->provisional; + completed.output_owned = pending_for_output; + completed.memory = std::move(memory); + auto const dirty_bounds = completed.map_bounds; + if (pending_for_output) + { + auto found = std::ranges::find( + output_atlases_, completed.window, + &AtlasCache::window); + if (found == output_atlases_.end()) + output_atlases_.push_back(std::move(completed)); + else + *found = std::move(completed); + } + else + { + atlas_ = std::move(completed); + } + retained_access_ = nextRetainedAccess(); + atlas_failed_signature_.clear(); + atlas_failed_scale_ = 0; + if (dirty_bounds.isValid()) + map->setTemplateAreaDirty( + this, dirty_bounds, + getTemplateBoundingBoxPixelBorder()); + updateResourceStatus(); +} + +bool OnlineRasterTemplate::appendTransparentAtlas(const TileWindow& window, + const QVector& visuals, + bool has_missing, double pixels_per_map_unit, + bool on_screen, + QVector& out) const +{ + auto const signature = atlasSignature(window, visuals, has_missing); + auto const scale = + std::max({ 1.0, std::abs(atlas_.pixels_per_map_unit), std::abs(pixels_per_map_unit) }); + auto const scale_matches = + std::abs(atlas_.pixels_per_map_unit - pixels_per_map_unit) <= scale * 1.0e-9; + if (atlas_.window != window || atlas_.signature != signature || atlas_.image.isNull() + || !scale_matches) + { + if (on_screen && output_preparation_active_ + && (output_uses_atlases_ + || atlas_pending_for_output_)) + { + return false; + } + queueAtlasBuild(window, visuals, has_missing, pixels_per_map_unit, signature, + !on_screen && output_preparation_active_); + return false; + } + auto const source_rect = QRectF( + 1, 1, + std::max(0, atlas_.image.width() - 2), + std::max(0, atlas_.image.height() - 2)); + out.push_back({ + atlas_.image, + atlas_.image_to_map.mapRect(source_rect), + source_rect, + quint64(atlas_.image.cacheKey()), + false, + atlas_.provisional, + atlas_.image_to_map, + true, + atlas_.memory, + [this](qint64 bytes) + -> std::shared_ptr { + return reserveRetainedMemory(bytes); + }, + }); + return true; +} + +bool OnlineRasterTemplate::appendPreparedOutputAtlases( + const TileWindow& window, + double pixels_per_map_unit, + QVector& out) const +{ + if (!output_preparation_active_ + || !output_uses_atlases_ + || !output_window_.contains(window)) + return false; + + for (auto const& cached : std::as_const(output_atlases_)) + { + if (!cached.window.intersects(window)) + continue; + auto const scale = std::max({ + 1.0, + std::abs(cached.pixels_per_map_unit), + std::abs(pixels_per_map_unit), + }); + if (!cached.output_owned + || cached.image.isNull() + || cached.provisional + || !cached.render_memory + || std::abs( + cached.pixels_per_map_unit + - pixels_per_map_unit) + > scale * 1.0e-9) + return false; + auto const source_rect = QRectF( + 1, 1, + std::max(0, cached.image.width() - 2), + std::max(0, cached.image.height() - 2)); + out.push_back({ + cached.image, + cached.image_to_map.mapRect(source_rect), + source_rect, + quint64(cached.image.cacheKey()), + false, + false, + cached.image_to_map, + true, + cached.memory, + [render_memory = cached.render_memory]( + qint64 bytes) + -> std::shared_ptr { + if (!render_memory + || bytes <= 0 + || bytes > render_memory->bytes) + return {}; + return render_memory; + }, + }); + } + retained_access_ = nextRetainedAccess(); + return true; +} + +void OnlineRasterTemplate::collectRasterTiles(const QRectF& map_clip_rect, double scale, + bool on_screen, + QVector& out) const +{ + if (template_state != Loaded || !sourceReady()) + return; + auto zoom = on_screen && !wanted_window_.isEmpty() + ? wanted_window_.zoom + : (!on_screen && output_preparation_active_ && !output_window_.isEmpty() + ? output_window_.zoom + : chooseZoom(map_clip_rect, scale, !on_screen)); + if (zoom < 0) + { + out.push_back({ {}, map_clip_rect, {}, 0, true, false }); + return; + } + auto const window = tileWindowForMapRect(map_clip_rect, zoom, !on_screen); + if (window.isEmpty()) + return; + if (!on_screen + && appendPreparedOutputAtlases( + window, scale, out)) + { + last_render_bounds_ = map_clip_rect; + return; + } + + bool has_transparency = false; + bool has_missing = false; + bool has_pixels = false; + auto const visuals = + visualTiles(window, on_screen, &has_transparency, &has_missing, &has_pixels); + if (!has_pixels) + { + for (auto const& visual : visuals) + { + if (visual.complete_empty) + continue; + out.push_back({ + {}, + mapBoundsForSourceBounds(tileBounds(visual.requested)), + {}, + 0, + true, + false, + }); + } + return; + } + + if (has_transparency) + { + if (appendTransparentAtlas(window, visuals, has_missing, scale, on_screen, out)) + { + last_render_bounds_ = atlas_.map_bounds; + return; + } + // A pathological atlas size remains incomplete rather than exposing + // translucent per-tile antialias seams. + out.push_back({ + {}, + mapBoundsForSourceBounds( + tileBounds({ window.zoom, window.min_column, window.min_row })), + {}, + 0, + true, + false, + }); + return; + } + + QRectF render_bounds; + for (auto const& visual : visuals) + { + if (visual.complete_empty) + continue; + if (!visual.tile) + { + auto const bounds = mapBoundsForSourceBounds(tileBounds(visual.requested)); + out.push_back({ {}, bounds, {}, 0, true, false }); + rectIncludeSafe(render_bounds, bounds); + continue; + } + auto const before = out.size(); + std::shared_ptr output_render_memory; + if (!on_screen && output_preparation_active_) + { + auto const found = + output_render_memory_.constFind(visual.cached); + if (found == output_render_memory_.cend() || !*found) + { + auto const bounds = + mapBoundsForSourceBounds( + tileBounds(visual.requested)); + out.push_back( + { {}, bounds, {}, 0, true, false }); + rectIncludeSafe(render_bounds, bounds); + continue; + } + output_render_memory = *found; + } + if (!appendOpaquePatches( + visual, scale, output_render_memory, out)) + { + auto const bounds = mapBoundsForSourceBounds(tileBounds(visual.requested)); + out.push_back({ {}, bounds, {}, 0, true, false }); + rectIncludeSafe(render_bounds, bounds); + continue; + } + for (auto index = before; index < out.size(); ++index) + rectIncludeSafe(render_bounds, out.at(index).template_rect); + } + last_render_bounds_ = render_bounds; +} + +QRectF OnlineRasterTemplate::mapBoundsForSourceBounds(const imagery::CrsBounds& bounds) const +{ + if (!bounds.isValid()) + return {}; + QRectF result; + for (int index = 0; index <= 32; ++index) + { + auto const fraction = index / 32.0; + for (auto const point : { + QPointF(bounds.west + (bounds.east - bounds.west) * fraction, bounds.north), + QPointF(bounds.west + (bounds.east - bounds.west) * fraction, bounds.south), + QPointF(bounds.west, bounds.south + (bounds.north - bounds.south) * fraction), + QPointF(bounds.east, bounds.south + (bounds.north - bounds.south) * fraction), + }) + { + if (auto mapped = nominalSourceToMap(point)) + rectIncludeSafe(result, *mapped); + } + } + return result; +} + +QRectF OnlineRasterTemplate::calculateTemplateBoundingBox() const +{ + if (!sourceReady()) + return last_render_bounds_; + auto const* tile_matrix = matrix(source_->min_zoom); + if (!tile_matrix) + return last_render_bounds_; + qint64 min_column = 0; + qint64 max_column = tile_matrix->matrix_width - 1; + qint64 min_row = 0; + qint64 max_row = tile_matrix->matrix_height - 1; + if (auto const* tile_limits = limits(source_->min_zoom)) + { + min_column = tile_limits->min_column; + max_column = tile_limits->max_column; + min_row = tile_limits->min_row; + max_row = tile_limits->max_row; + } + auto const first = tile_matrix->tileBounds(min_column, min_row); + auto const last = tile_matrix->tileBounds(max_column, max_row); + return mapBoundsForSourceBounds({ first.west, last.south, last.east, first.north }); +} + +QRectF OnlineRasterTemplate::getTemplateExtent() const +{ + return calculateTemplateBoundingBox(); +} + +void OnlineRasterTemplate::markTileDirty(const OnlineRasterTileKey& key) +{ + auto const bounds = mapBoundsForSourceBounds(tileBounds(key)); + if (bounds.isValid()) + map->setTemplateAreaDirty(this, bounds, getTemplateBoundingBoxPixelBorder()); +} + +void OnlineRasterTemplate::onMapGeoreferencingChanged() +{ + if (template_state != Loaded) + return; + auto const old_bounds = last_render_bounds_; + cancelAtlasBuild(); + atlas_.clear(); + wanted_window_ = {}; + if (old_bounds.isValid()) + map->setTemplateAreaDirty(this, old_bounds, getTemplateBoundingBoxPixelBorder()); + auto const new_bounds = calculateTemplateBoundingBox(); + if (new_bounds.isValid()) + map->setTemplateAreaDirty(this, new_bounds, getTemplateBoundingBoxPixelBorder()); +} + +} // namespace OpenOrienteering diff --git a/src/templates/online_raster_template.h b/src/templates/online_raster_template.h new file mode 100644 index 000000000..cfa12e83a --- /dev/null +++ b/src/templates/online_raster_template.h @@ -0,0 +1,486 @@ +/* + * Copyright 2026 Ethan O'Connor + * + * This file is part of OpenOrienteering. + * + * OpenOrienteering is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +#ifndef OPENORIENTEERING_ONLINE_RASTER_TEMPLATE_H +#define OPENORIENTEERING_ONLINE_RASTER_TEMPLATE_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "imagery/imagery_source_snapshot.h" +#include "imagery/tile_network_manager.h" +#include "templates/raster_resource_manager.h" +#include "templates/template_image.h" + +class QXmlStreamReader; +class QXmlStreamWriter; + +namespace OpenOrienteering { + +class Map; +struct ProjTransform; +class OnlineRasterTemplateTest; + +struct OnlineRasterTileKey +{ + int zoom = -1; + qint64 column = 0; + qint64 row = 0; + + auto operator<=>(const OnlineRasterTileKey&) const = default; +}; + +inline size_t qHash(const OnlineRasterTileKey& key, size_t seed = 0) +{ + seed = ::qHash(key.zoom, seed); + seed = ::qHash(key.column, seed); + return ::qHash(key.row, seed); +} + +/** + * Native tiled imagery template backed by an embedded resolved source. + * + * Network I/O is shared and bounded by imagery::TileNetworkManager. Image + * decoding and gutter construction use RasterResourceManager's decode lane. + * Opaque tiles render as direct affine patches; windows containing alpha are + * first composed into one retained atlas to preserve alpha across tile seams. + */ +class OnlineRasterTemplate final : public TemplateImage +{ + Q_OBJECT + + public: + explicit OnlineRasterTemplate(imagery::ImagerySourceSnapshot snapshot, Map* map, + imagery::TileNetworkManager* network = nullptr); + ~OnlineRasterTemplate() override; + + static std::unique_ptr createForType(const QString& path, Map* map); + + const char* getTemplateType() const override; + bool fileExists() const override; + LookupResult tryToFindTemplateFile(const QString& map_path) override; + QSize getRasterPixelSize() const override; + QRectF calculateTemplateBoundingBox() const override; + + const imagery::ImagerySourceSnapshot* sourceSnapshot() const noexcept; + void setDisplayName(const QString& name); + bool sourceReady() const noexcept; + OutputRenderPreparation prepareForOutput(const QRectF& map_rect, + double pixels_per_map_unit) override; + void finishOutputPreparation(bool cancelled) override; + + protected: + OnlineRasterTemplate(const OnlineRasterTemplate& prototype); + OnlineRasterTemplate* duplicate() const override; + + bool loadTemplateFileImpl() override; + bool postLoadSetup(QWidget* dialog_parent, bool& out_center_in_view) override; + void unloadTemplateFileImpl() override; + void saveTypeSpecificTemplateConfiguration(QXmlStreamWriter& xml) const override; + bool loadTypeSpecificTemplateConfiguration(QXmlStreamReader& xml) override; + bool finishTypeSpecificTemplateConfiguration() override; + + void updateRenderContext(const ViewRenderContext& context) override; + QRectF getTemplateExtent() const override; + void collectRasterTiles(const QRectF& map_clip_rect, double scale, bool on_screen, + QVector& out) const override; + + private slots: + void onNetworkFinished(imagery::TileNetworkManager::Token token, + const imagery::TileNetworkResult& result); + void onMapGeoreferencingChanged(); + + private: + friend class OnlineRasterTemplateTest; + + struct TileWindow + { + int zoom = -1; + qint64 min_column = 0; + qint64 max_column = -1; + qint64 min_row = 0; + qint64 max_row = -1; + + bool isEmpty() const noexcept; + bool intersects(const TileWindow& other) const noexcept; + bool contains(const TileWindow& other) const noexcept; + qint64 width() const noexcept; + qint64 height() const noexcept; + auto operator<=>(const TileWindow&) const = default; + }; + + struct MemoryReservation final : public RasterMemoryLease + { + std::shared_ptr> counter; + qint64 bytes = 0; + + ~MemoryReservation(); + void shrinkTo(qint64 bytes) noexcept override; + }; + + struct CachedTile + { + CachedTile() = default; + CachedTile(QImage image, bool opaque, bool empty = false, + std::shared_ptr memory = {}) + : image(std::move(image)) + , opaque(opaque) + , empty(empty) + , memory(std::move(memory)) + {} + + QImage image; + bool opaque = false; + bool empty = false; + std::shared_ptr memory; + }; + + struct PendingFetch + { + OnlineRasterTileKey key; + quint64 generation = 0; + int endpoint = 0; + quint64 endpoint_offset = 0; + }; + + struct PendingDecode + { + std::shared_ptr cancelled; + int endpoint = 0; + quint64 endpoint_offset = 0; + }; + + struct EncodedTilePayload + { + QByteArray bytes; + std::shared_ptr> bytes_in_flight; + qint64 byte_count = 0; + + ~EncodedTilePayload(); + }; + + struct TileFailure + { + int attempts = 0; + quint64 next_endpoint_offset = 0; + QSet terminal_endpoints; + QHash policy_rejected_origins; + QDeadlineTimer retry; + bool permanent = false; + QString message; + }; + + struct StoredSnapshotPayload + { + QString encoding; + QString text; + }; + + struct VisualTile + { + OnlineRasterTileKey requested; + OnlineRasterTileKey cached; + const CachedTile* tile = nullptr; + QRectF source_rect; + bool provisional = false; + bool complete_empty = false; + }; + + struct AtlasCache + { + TileWindow window; + QVector signature; + QImage image; + QTransform image_to_map; + QRectF map_bounds; + double pixels_per_map_unit = 0; + bool provisional = false; + bool output_owned = false; + std::shared_ptr memory; + std::shared_ptr render_memory; + + void clear(); + }; + + struct AtlasBuildVisual + { + QRectF target_rect; + QImage image; + QRectF source_rect; + std::shared_ptr memory; + }; + + struct AtlasWarpGrid + { + int columns = 0; + int rows = 0; + QSize output_size; + QVector source_points; + // Sampling may use the bounded interpolation grid, but ownership of the + // half-open source chunk is resolved with this common exact inverse near + // its edges. Adjacent chunks therefore make the same boundary decision. + bool exact_ownership = false; + QString map_crs; + QString source_crs; + QTransform map_to_projected; + QRectF map_bounds; + QPointF source_registration; + double core_west = 0; + double core_north = 0; + double cell_size = 0; + double maximum_interpolation_error = 0; + }; + + struct AtlasBuildRequest + { + TileWindow window; + QVector signature; + QSize core_size; + QVector visuals; + std::optional warp; + QTransform image_to_map; + QRectF map_bounds; + double pixels_per_map_unit = 0; + bool provisional = false; + bool has_left_neighbor = false; + bool has_right_neighbor = false; + bool has_top_neighbor = false; + bool has_bottom_neighbor = false; + std::shared_ptr working_memory; + }; + + struct AtlasBuildResult + { + TileWindow window; + QVector signature; + QImage image; + QTransform image_to_map; + QRectF map_bounds; + double pixels_per_map_unit = 0; + bool provisional = false; + std::shared_ptr working_memory; + }; + + explicit OnlineRasterTemplate(const QString& path, Map* map, + imagery::TileNetworkManager* network); + + void initializeConnections(); + void resetRuntime(bool clear_cache); + void setSnapshot(imagery::ImagerySourceSnapshot snapshot); + bool decodeStoredSnapshot(); + + const imagery::ResolvedImagerySource* source() const noexcept; + const imagery::TileMatrix* matrix(int zoom) const noexcept; + const imagery::TileMatrixLimits* limits(int zoom) const noexcept; + bool tileAllowed(const OnlineRasterTileKey& key) const noexcept; + imagery::CrsBounds tileBounds(const OnlineRasterTileKey& key) const noexcept; + + std::optional mapToNominalSource(const QPointF& map_point) const; + std::optional nominalSourceToMap(const QPointF& source_point) const; + std::optional imagePointToMap(const OnlineRasterTileKey& image_key, + const QPointF& image_point) const; + std::optional imageRectToMap(const OnlineRasterTileKey& image_key, + const QRectF& source_rect, + QRectF* map_bounds = nullptr, + double* residual_map_units = nullptr) const; + + TileWindow tileWindowForMapRect(const QRectF& map_rect, int zoom, + bool exact_output = false, + bool* projection_complete = nullptr) const; + int chooseZoom(const QRectF& map_rect, double pixels_per_map_unit, + bool exact_output = false) const; + TileWindow withOverscan(TileWindow window, qint64 tiles) const; + std::optional tileCount(const TileWindow& window) const noexcept; + bool workingSetFits(const TileWindow& window) const noexcept; + bool keyNeededForWindow(const OnlineRasterTileKey& key, + const TileWindow& window) const noexcept; + void cancelUnwantedWork(const TileWindow& window); + + void queueWindow(const TileWindow& window, bool replace_pending); + void queueTile(const OnlineRasterTileKey& key, imagery::TileRequestPriority priority, + double distance_priority); + void recordFailure( + const OnlineRasterTileKey& key, + bool permanent, + QString message = {}); + void recordEndpointFailure( + const OnlineRasterTileKey& key, + int endpoint, + quint64 endpoint_offset, + bool terminal, + QString message, + const QUrl& policy_rejected_url = {}); + void clearFailure(const OnlineRasterTileKey& key); + bool retryAllowed(const OnlineRasterTileKey& key) const; + void scheduleNextRetry(); + void trimFailureHistory(); + void updateResourceStatus(); + qint64 encodedTileResponseLimit(const OnlineRasterTileKey& key) const noexcept; + std::shared_ptr reserveEncodedTilePayload(QByteArray bytes); + std::shared_ptr reserveRetainedMemory(qint64 bytes) const; + qint64 evictRetainedMemory(qint64 target_bytes); + + static std::optional + decodeTile(const QByteArray& bytes, const QSize& expected_size, + const RasterResourceManager::CancellationToken& cancellation, + const std::shared_ptr& source_cancelled); + void queueDecode( + const OnlineRasterTileKey& key, + QByteArray bytes, + quint64 generation, + int endpoint = 0, + quint64 endpoint_offset = 0); + void finishDecode(const OnlineRasterTileKey& key, std::optional tile, + quint64 generation, const std::shared_ptr& cancelled); + void insertTile(const OnlineRasterTileKey& key, CachedTile tile); + void insertEmptyTile(const OnlineRasterTileKey& key); + + const CachedTile* bestCachedTile(const OnlineRasterTileKey& requested, + OnlineRasterTileKey* cached_key, QRectF* source_rect) const; + QVector visualTiles(const TileWindow& window, bool allow_provisional, + bool* has_transparency, bool* has_missing, + bool* has_pixels) const; + bool appendOpaquePatches( + const VisualTile& visual, + double pixels_per_map_unit, + const std::shared_ptr& output_render_memory, + QVector& out) const; + bool appendOpaquePatch(const VisualTile& visual, QRectF source_rect, double pixels_per_map_unit, + int depth, + const std::shared_ptr& output_render_memory, + QVector& out) const; + bool appendTransparentAtlas(const TileWindow& window, const QVector& visuals, + bool has_missing, double pixels_per_map_unit, bool on_screen, + QVector& out) const; + bool appendPreparedOutputAtlases( + const TileWindow& window, + double pixels_per_map_unit, + QVector& out) const; + QVector atlasSignature(const TileWindow& window, const QVector& visuals, + bool has_missing) const; + QVector atlasChunks( + const TileWindow& window, + double pixels_per_map_unit) const; + std::optional makeAtlasBuildRequest(const TileWindow& window, + const QVector& visuals, + bool has_missing, + double pixels_per_map_unit, + const QVector& signature) const; + static std::optional + buildAtlas(AtlasBuildRequest request, const std::shared_ptr& cancelled, + const RasterResourceManager::CancellationToken& cancellation); + bool queueAtlasBuild(const TileWindow& window, const QVector& visuals, + bool has_missing, double pixels_per_map_unit, + const QVector& signature, bool for_output) const; + void finishAtlasBuild(std::optional result, + const std::shared_ptr& cancelled); + void cancelAtlasBuild(bool clear_failure = true) const; + + void markTileDirty(const OnlineRasterTileKey& key); + QRectF mapBoundsForSourceBounds(const imagery::CrsBounds& bounds) const; + + std::optional snapshot_; + QByteArray stored_snapshot_json_; + QByteArray stored_snapshot_sha256_; + QString stored_snapshot_version_; + QVector stored_snapshot_payloads_; + bool stored_version_attribute_ = true; + bool stored_checksum_attribute_ = true; + QString snapshot_error_; + std::shared_ptr source_; + std::unique_ptr source_projection_; + + imagery::TileNetworkManager* network_ = nullptr; + quint64 network_client_id_ = 0; + quint64 generation_ = 1; + TileWindow wanted_window_; + TileWindow output_window_; + bool output_preparation_active_ = false; + QString output_preparation_error_; + QSet output_keys_; + QHash output_tiles_; + QHash> + output_render_memory_; + bool output_source_tiles_released_ = false; + double output_preparation_scale_ = 0; + QHash pending_fetches_; + QSet queued_tiles_; + QHash pending_decodes_; + std::shared_ptr> decode_bytes_in_flight_; + QHash failed_tiles_; + QSet offline_tiles_; + QTimer retry_timer_; + RasterResourceManager::Owner decode_owner_ = RasterResourceManager::instance().createOwner(2); + mutable RasterResourceManager::Owner atlas_owner_ = + RasterResourceManager::instance().createOwner(1); + +#ifdef Q_OS_ANDROID + QCache tile_cache_{ 64 * 1024 }; +#else + QCache tile_cache_{ 256 * 1024 }; + #endif + mutable AtlasCache atlas_; + mutable QVector output_atlases_; + mutable bool output_uses_atlases_ = false; + mutable std::shared_ptr atlas_cancelled_; + mutable QVector atlas_pending_signature_; + mutable double atlas_pending_scale_ = 0; + mutable bool atlas_pending_for_output_ = false; + mutable QVector atlas_failed_signature_; + mutable double atlas_failed_scale_ = 0; + mutable bool atlas_queue_busy_ = false; + mutable QTimer atlas_retry_timer_; + mutable QRectF last_render_bounds_; + mutable bool exact_projection_failed_ = false; + mutable quint64 retained_access_ = 0; + + static constexpr qsizetype max_queued_tiles = 96; + #ifdef Q_OS_ANDROID + static constexpr qint64 max_window_tiles = 512; + static constexpr qint64 max_working_set_bytes = qint64(32) * 1024 * 1024; + static constexpr qint64 max_decode_encoded_bytes = qint64(24) * 1024 * 1024; + static constexpr qint64 max_retained_raster_bytes = qint64(128) * 1024 * 1024; + static constexpr qint64 max_encoded_tile_response_bytes = qint64(4) * 1024 * 1024; + static constexpr qint64 max_atlas_pixels = qint64(6) * 1024 * 1024; + static constexpr qint64 max_atlas_peak_bytes = qint64(48) * 1024 * 1024; + #else + static constexpr qint64 max_window_tiles = 1024; + static constexpr qint64 max_working_set_bytes = qint64(384) * 1024 * 1024; + static constexpr qint64 max_decode_encoded_bytes = qint64(64) * 1024 * 1024; + static constexpr qint64 max_retained_raster_bytes = qint64(512) * 1024 * 1024; + static constexpr qint64 max_encoded_tile_response_bytes = qint64(8) * 1024 * 1024; + static constexpr qint64 max_atlas_pixels = qint64(16) * 1024 * 1024; + static constexpr qint64 max_atlas_peak_bytes = qint64(128) * 1024 * 1024; +#endif + static constexpr qint64 max_tile_pixels = + imagery::maximum_runtime_tile_pixels; + static constexpr int max_tile_dimension = + imagery::maximum_runtime_tile_dimension; + static constexpr int max_atlas_dimension = 8192; + static constexpr qsizetype max_failure_records = 2048; +}; + +} // namespace OpenOrienteering + +#endif diff --git a/src/templates/raster_resource_manager.cpp b/src/templates/raster_resource_manager.cpp index 032fd4cec..93fb77e91 100644 --- a/src/templates/raster_resource_manager.cpp +++ b/src/templates/raster_resource_manager.cpp @@ -64,6 +64,7 @@ struct RasterResourceManager::Owner::SharedState std::vector> pending; std::uint64_t last_owner = 0; int active = 0; + std::size_t outstanding = 0; }; SharedState(RasterResourceManager* context, Limits limits) @@ -76,6 +77,9 @@ struct RasterResourceManager::Owner::SharedState , max_pending_per_lane(std::max( max_pending_per_owner, limits.max_pending_per_lane )) + , max_outstanding_per_lane(std::max( + 1, limits.max_outstanding_per_lane + )) {} LaneState& laneState(Lane lane) @@ -157,7 +161,8 @@ struct RasterResourceManager::Owner::SharedState { auto& lane = laneState(lane_id); while (!shutting_down - && lane.active < lane.pool.maxThreadCount()) + && lane.active < lane.pool.maxThreadCount() + && lane.outstanding < max_outstanding_per_lane) { auto selected = chooseNextLocked(lane_id, lane); if (selected == lane.pending.end()) @@ -166,6 +171,7 @@ struct RasterResourceManager::Owner::SharedState auto job = std::move(*selected); lane.pending.erase(selected); ++lane.active; + ++lane.outstanding; ++job->owner->active[laneIndex(lane_id)]; auto self = shared_from_this(); lane.pool.start([self = std::move(self), job = std::move(job)] { @@ -198,6 +204,8 @@ struct RasterResourceManager::Owner::SharedState --lane.active; --job->owner->active[laneIndex(job->lane)]; deliver = !shutting_down && completion && jobIsCurrent(*job); + if (!deliver) + --lane.outstanding; dispatchLocked(Lane::BlockingIo); dispatchLocked(Lane::Decode); } @@ -210,15 +218,20 @@ struct RasterResourceManager::Owner::SharedState context, [self = std::move(self), job = std::move(job), completion = std::move(completion)]() mutable { + bool run_completion = false; { std::lock_guard lock(self->mutex); - if (self->shutting_down || !self->jobIsCurrent(*job) - || !job->receiver) - { - return; - } + auto& lane = self->laneState(job->lane); + Q_ASSERT(lane.outstanding > 0); + --lane.outstanding; + run_completion = + !self->shutting_down + && self->jobIsCurrent(*job) + && job->receiver; + self->dispatchLocked(job->lane); } - completion(); + if (run_completion) + completion(); }, Qt::QueuedConnection ); @@ -272,6 +285,7 @@ struct RasterResourceManager::Owner::SharedState std::uint64_t next_sequence = 1; std::size_t max_pending_per_owner = 128; std::size_t max_pending_per_lane = 2048; + std::size_t max_outstanding_per_lane = 128; bool shutting_down = false; }; diff --git a/src/templates/raster_resource_manager.h b/src/templates/raster_resource_manager.h index c42b6365f..4b5a4a947 100644 --- a/src/templates/raster_resource_manager.h +++ b/src/templates/raster_resource_manager.h @@ -26,7 +26,10 @@ namespace OpenOrienteering { * * Sources continue to own demand, typed caches, and retry policy. The manager * owns only bounded execution, fair ordering, cancellation generations, and - * receiver-safe delivery back to the application thread. + * receiver-safe delivery back to the application thread. Worker admission + * reserves a bounded completion slot until the application thread consumes + * the result, so a blocked UI event loop cannot accumulate unbounded + * completion closures. */ class RasterResourceManager final : public QObject { @@ -50,6 +53,15 @@ class RasterResourceManager final : public QObject int decode_threads = 0; std::size_t max_pending_per_owner = 128; std::size_t max_pending_per_lane = 2048; + /** + * Maximum active jobs plus completions awaiting manager-thread + * delivery in each lane. + */ +#ifdef Q_OS_ANDROID + std::size_t max_outstanding_per_lane = 4; +#else + std::size_t max_outstanding_per_lane = 8; +#endif }; class CancellationToken diff --git a/src/templates/template.cpp b/src/templates/template.cpp index 5b9296446..63551d315 100644 --- a/src/templates/template.cpp +++ b/src/templates/template.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -40,10 +41,13 @@ #include #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -59,6 +63,7 @@ #include "gui/file_dialog.h" #include "templates/template_image.h" #include "templates/template_map.h" +#include "templates/online_raster_template.h" #include "templates/template_placeholder.h" #include "templates/template_track.h" #include "util/util.h" @@ -67,6 +72,64 @@ namespace OpenOrienteering { +TemplateResourceStatus::Condition TemplateResourceStatus::dominantCondition() const noexcept +{ + if (permanent_failures > 0) + return Condition::PermanentFailure; + if (offline_resources > 0) + return Condition::Offline; + if (transient_failures > 0) + return Condition::TransientFailure; + if (loading_resources > 0) + return Condition::Loading; + return Condition::Ready; +} + +bool TemplateResourceStatus::isReported() const noexcept +{ + return ready_resources > 0 + || loading_resources > 0 + || offline_resources > 0 + || transient_failures > 0 + || permanent_failures > 0 + || !message.isEmpty(); +} + +qsizetype TemplateResourceStatus::totalResources() const noexcept +{ + constexpr auto maximum = std::numeric_limits::max(); + qsizetype total = 0; + for (auto count : { + ready_resources, + loading_resources, + offline_resources, + transient_failures, + permanent_failures }) + { + if (count <= 0) + continue; + if (count > maximum - total) + return maximum; + total += count; + } + return total; +} + +bool operator==(const TemplateResourceStatus& lhs, const TemplateResourceStatus& rhs) noexcept +{ + return lhs.ready_resources == rhs.ready_resources + && lhs.loading_resources == rhs.loading_resources + && lhs.offline_resources == rhs.offline_resources + && lhs.transient_failures == rhs.transient_failures + && lhs.permanent_failures == rhs.permanent_failures + && lhs.message == rhs.message; +} + +bool operator!=(const TemplateResourceStatus& lhs, const TemplateResourceStatus& rhs) noexcept +{ + return !(lhs == rhs); +} + class Template::ScopedOffsetReversal { public: @@ -119,6 +182,75 @@ void Template::updateRenderContext(const ViewRenderContext& context) Q_UNUSED(context) } +OutputRenderPreparation Template::prepareForOutput( + const QRectF& map_rect, + double pixels_per_map_unit) +{ + Q_UNUSED(map_rect) + Q_UNUSED(pixels_per_map_unit) + return {}; +} + +void Template::finishOutputPreparation(bool cancelled) +{ + Q_UNUSED(cancelled) +} + +TemplateResourceStatus Template::resourceStatus() const +{ + return resource_status; +} + +void Template::setResourceStatus(TemplateResourceStatus status) +{ + if (QThread::currentThread() != thread()) + { + QMetaObject::invokeMethod( + this, + [this, status = std::move(status)]() mutable { + setResourceStatus(std::move(status)); + }, + Qt::QueuedConnection); + return; + } + + auto normalize = [](qsizetype value) { + return qMax(0, value); + }; + status.ready_resources = normalize(status.ready_resources); + status.loading_resources = normalize(status.loading_resources); + status.offline_resources = normalize(status.offline_resources); + status.transient_failures = normalize(status.transient_failures); + status.permanent_failures = normalize(status.permanent_failures); + + if (resource_status == status) + return; + + resource_status = std::move(status); + notifyResourceStatusChanged(); +} + +void Template::notifyResourceStatusChanged() +{ + if (QThread::currentThread() != thread()) + { + QMetaObject::invokeMethod( + this, + [this]() { notifyResourceStatusChanged(); }, + Qt::QueuedConnection); + return; + } + + if (resource_status_change_pending) + return; + + resource_status_change_pending = true; + QTimer::singleShot(0, this, [this]() { + resource_status_change_pending = false; + emit resourceStatusChanged(); + }); +} + // static TemplateTransform TemplateTransform::fromQTransform(const QTransform& qt) noexcept { @@ -1025,6 +1157,8 @@ std::unique_ptr