diff --git a/back-end/controllers/celery_controller/celery_config.py b/back-end/controllers/celery_controller/celery_config.py index ebdfd0a..1916101 100755 --- a/back-end/controllers/celery_controller/celery_config.py +++ b/back-end/controllers/celery_controller/celery_config.py @@ -28,7 +28,14 @@ def make_celery(): "sweep-stuck-tasks": { "task": "controllers.celery_controller.celery_tasks.sweep_stuck_tasks", "schedule": 300.0, - } + }, + # Edit retiles splice only the dirty region (z9-16); this settles + # the deliberately stale z0-8 overview tiles once a folder has + # been quiet for a while. + "settle-stale-tiles": { + "task": "controllers.celery_controller.celery_tasks.settle_stale_tiles", + "schedule": 300.0, + }, }, ) return celery diff --git a/back-end/controllers/celery_controller/celery_tasks.py b/back-end/controllers/celery_controller/celery_tasks.py index 5575813..64d578f 100755 --- a/back-end/controllers/celery_controller/celery_tasks.py +++ b/back-end/controllers/celery_controller/celery_tasks.py @@ -153,6 +153,10 @@ def process_data(self, folderid, operation): # fabric intake dispatches the recompute when the new fabric lands. has_fabric = bool(file_ops.get_files_by_type(folderid, "fabric", session)) + # The fabric is the same for every coverage file — parse it once for + # the whole run, not once per file (it's an ~80 MB CSV). + fabric_gdf = kml_ops.load_fabric_gdf(folderid) if has_fabric else None + for file in coverage_files: if not has_fabric: file.computed = False @@ -183,6 +187,7 @@ def process_data(self, folderid, operation): latency, category, session, + fabric=fabric_gdf, ) file.computed = True @@ -250,9 +255,12 @@ def async_delete_files(self, file_ids, editfile_ids): # with a dirty flag so rapid edits coalesce. TILES_DIRTY_KEY = "bdk:tiles-dirty:{folderid}" +TILES_DIRTY_BBOX_KEY = "bdk:tiles-dirty-bbox:{folderid}" +TILES_SETTLE_KEY = "bdk:tiles-settle:{folderid}" TILES_LOCK_KEY = "bdk:tiles-lock:{folderid}" TILES_LOCK_TTL = 3600 # safety expiry on the lock; no rebuild should take this long TILES_LOCK_WAIT = 1800 # max time a regenerate waits behind another rebuild +TILES_SETTLE_QUIET = 600 # settle the stale z0-8 only after edits go quiet this long def _tiles_redis(): @@ -269,10 +277,59 @@ def _tiles_redis(): return None -def _mark_tiles_dirty(folderid): +def _mark_tiles_dirty(folderid, bbox=None): + """Record that the folder's tiles no longer match DB truth. With a bbox + ([minx, miny, maxx, maxy] lon/lat) the dirt is scoped to that region and + the rebuild may splice just its tiles; without one the whole tileset is + stale. A bbox never downgrades already-recorded whole-tileset dirt.""" r = _tiles_redis() - if r is not None: - r.set(TILES_DIRTY_KEY.format(folderid=folderid), "1") + if r is None: + return + dirty_key = TILES_DIRTY_KEY.format(folderid=folderid) + if bbox is None: + r.set(dirty_key, "full") + else: + r.rpush(TILES_DIRTY_BBOX_KEY.format(folderid=folderid), json.dumps(bbox)) + r.set(dirty_key, "bbox", nx=True) + + +def _pop_tiles_dirt(r, folderid): + """Atomically consume the folder's recorded dirt -> (flag, [bbox, ...]). + Atomic so a mark racing this pop is either fully consumed now or fully + left for the next rebuild — never half-eaten.""" + pipe = r.pipeline(transaction=True) + pipe.lrange(TILES_DIRTY_BBOX_KEY.format(folderid=folderid), 0, -1) + pipe.delete(TILES_DIRTY_BBOX_KEY.format(folderid=folderid)) + pipe.get(TILES_DIRTY_KEY.format(folderid=folderid)) + pipe.delete(TILES_DIRTY_KEY.format(folderid=folderid)) + raw_bboxes, _, dirty, _ = pipe.execute() + return dirty, [json.loads(b) for b in raw_bboxes] + + +def _features_bbox(features): + """[minx, miny, maxx, maxy] over every coordinate of the given GeoJSON + features, or None when there are no coordinates.""" + coords = [] + + def walk(node): + if isinstance(node, (list, tuple)): + if ( + len(node) >= 2 + and isinstance(node[0], (int, float)) + and isinstance(node[1], (int, float)) + ): + coords.append((node[0], node[1])) + else: + for item in node: + walk(item) + + for feature in features or []: + walk(((feature or {}).get("geometry") or {}).get("coordinates")) + if not coords: + return None + xs = [c[0] for c in coords] + ys = [c[1] for c in coords] + return [min(xs), min(ys), max(xs), max(ys)] @celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) @@ -367,9 +424,14 @@ def apply_edit_changes(self, markers, folderid, polygonfeatures): + file_ops.get_files_with_postfix(user_folder.id, ".geojson", session) ] + from controllers.database_controller.setting_ops import export_max_service_only + results = session.query(kml_data).filter(kml_data.file_id.in_(all_file_ids)).all() availability_csv = kml_ops.generate_csv_data( - results, user_folder.organization.provider_id, user_folder.organization.brand_name + results, + user_folder.organization.provider_id, + user_folder.organization.brand_name, + max_service_only=export_max_service_only(session), ) csv_name = f"availability-{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}.csv" @@ -391,24 +453,16 @@ def apply_edit_changes(self, markers, folderid, polygonfeatures): session.commit() session.close() - _mark_tiles_dirty(folderid) + # An edit only changes points inside its drawn polygons, so the dirt is + # scoped: the rebuild can splice just that region's z9-16 tiles. + _mark_tiles_dirty(folderid, bbox=_features_bbox(polygonfeatures)) def _rebuild_folder_tiles(folderid): """Rebuild a folder's vector tiles from current DB truth (the slow part).""" session = Session() try: - geojson_data = [] - all_kmls = file_ops.get_files_with_postfix(folderid, ".kml", session) - for kml_f in all_kmls: - geojson_data.append(vt_ops.read_kml(kml_f.id, session)) - - all_geojsons = file_ops.get_files_with_postfix( - folderid=folderid, postfix=".geojson", session=session - ) - for geojson_f in all_geojsons: - geojson_data.append(vt_ops.read_geojson(geojson_f.id, session)) - + geojson_data = vt_ops.folder_coverage_features(folderid, session) mbtiles_ops.delete_mbtiles(folderid, session) vt_ops.create_tiles(geojson_data, folderid, session) finally: @@ -416,20 +470,38 @@ def _rebuild_folder_tiles(folderid): session.close() +def _splice_folder_tiles(folderid, bboxes): + """Splice the dirty regions' z9-16 tiles into the live tileset. True on + success; False (fall back to a full rebuild) on any failure.""" + try: + session = Session() + try: + return vt_ops.splice_tiles(folderid, bboxes, session) + finally: + session.close() + except Exception: + logger.exception(f"tile splice failed for folder {folderid}; doing a full rebuild") + return False + + def _coalesced_tile_rebuild(folderid, owner_id): - """Rebuild a folder's tiles, single-flight + coalescing. Callers mark the - folder dirty after changing its data; the rebuild that runs after that - clears the flag. A caller that finds the folder clean no-ops (an earlier - rebuild already covered its change), so N rapid changes cost ~1 rebuild. - Returns only once the tiles cover the caller's change. Without redis, the - debounce degrades to always rebuilding (correct, less efficient).""" + """Refresh a folder's tiles, single-flight + coalescing. Callers mark the + folder dirty after changing its data; the refresh that runs after that + consumes the dirt. A caller that finds the folder clean no-ops (an + earlier refresh already covered its change), so N rapid changes cost ~1 + refresh. Returns only once the tiles cover the caller's change. + + Dirt scoped to bboxes (edits) is SPLICED — only the dirty regions' z9-16 + tiles are regenerated, into the current tileset — leaving the z0-8 + overview tiles slightly stale (sub-pixel at those zooms); the settle + task full-rebuilds once the folder goes quiet. Whole-tileset dirt, a + splice failure, or no redis means a full rebuild.""" r = _tiles_redis() if r is None: _rebuild_folder_tiles(folderid) return "tiles rebuilt" lock_key = TILES_LOCK_KEY.format(folderid=folderid) - dirty_key = TILES_DIRTY_KEY.format(folderid=folderid) waited = 0 while not r.set(lock_key, str(owner_id), nx=True, ex=TILES_LOCK_TTL): if waited >= TILES_LOCK_WAIT: @@ -437,10 +509,14 @@ def _coalesced_tile_rebuild(folderid, owner_id): time.sleep(5) waited += 5 try: - if not r.get(dirty_key): + dirty, bboxes = _pop_tiles_dirt(r, folderid) + if not dirty: return "tiles fresh (an earlier rebuild covered this edit)" - r.delete(dirty_key) + if dirty == b"bbox" and bboxes and _splice_folder_tiles(folderid, bboxes): + r.set(TILES_SETTLE_KEY.format(folderid=folderid), str(time.time())) + return "tiles spliced" _rebuild_folder_tiles(folderid) + r.delete(TILES_SETTLE_KEY.format(folderid=folderid)) return "tiles rebuilt" finally: r.delete(lock_key) @@ -448,12 +524,42 @@ def _coalesced_tile_rebuild(folderid, owner_id): @celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) def regenerate_tiles(self, folderid): - """The slow half of an edit chain: rebuild the folder's tiles via the - shared coalescing path. 'Task SUCCESS' always means 'the tiles include - this chain's edit'.""" + """The slow half of an edit chain: refresh the folder's tiles via the + shared coalescing path (a splice when the dirt is edit-scoped, a full + rebuild otherwise). 'Task SUCCESS' always means 'the tiles include this + chain's edit'.""" return _coalesced_tile_rebuild(folderid, self.request.id) +@celery.task +def settle_stale_tiles(): + """Beat housekeeping: splices leave a folder's z0-8 overview tiles + slightly stale (an edited dot is sub-pixel at those zooms). Once a + spliced folder has been quiet for TILES_SETTLE_QUIET, run one full + rebuild to reconcile. Background work — no job row, so no pill.""" + r = _tiles_redis() + if r is None: + return "no redis; nothing to settle" + settled = [] + for key in r.scan_iter(match=TILES_SETTLE_KEY.format(folderid="*")): + key = key.decode() if isinstance(key, bytes) else key + try: + folderid = int(key.rsplit(":", 1)[1]) + last_splice = float(r.get(key) or 0) + except (ValueError, AttributeError): + r.delete(key) + continue + if time.time() - last_splice < TILES_SETTLE_QUIET: + continue + # Consume the flag first: if the rebuild fails the next edit's splice + # re-flags, and the dirty mark below survives for the retry anyway. + r.delete(key) + _mark_tiles_dirty(folderid) + regenerate_tiles.apply_async(args=[folderid]) + settled.append(folderid) + return f"settling folders {settled}" if settled else "nothing to settle" + + @celery.task(bind=True, autoretry_for=(Exception,), retry_backoff=True) def async_folder_copy_for_export(self, folderid, serialized_csv, brandname, deadline): from services import export_service @@ -678,10 +784,10 @@ def raster2vector(self, data, userid, outfile_name): geojson_array = [] all_kmls = file_ops.get_files_with_postfix(fileVal.folder_id, ".kml", session) for kml_f in all_kmls: - geojson_array.append(vt_ops.read_kml(kml_f.id, session)) + geojson_array.extend(vt_ops.read_kml(kml_f.id, session)) all_geojsons = file_ops.get_files_with_postfix(fileVal.folder_id, ".geojson", session) for geojson_f in all_geojsons: - geojson_array.append(vt_ops.read_geojson(geojson_f.id, session)) + geojson_array.extend(vt_ops.read_geojson(geojson_f.id, session)) logger.info("Creating Vector Tiles") mbtiles_ops.delete_mbtiles(fileVal.folder_id, session) diff --git a/back-end/controllers/database_controller/file_ops.py b/back-end/controllers/database_controller/file_ops.py index 1ba986e..9460526 100755 --- a/back-end/controllers/database_controller/file_ops.py +++ b/back-end/controllers/database_controller/file_ops.py @@ -38,9 +38,13 @@ def get_files_with_postfix(folderid, postfix, session=None): owns_session = True try: + # Deterministic order: downstream merges (get_kml_data's per-file + # property stamping, the tile pipeline's feature stream) must produce + # identical output for identical inputs. files_with_ending = ( session.query(file) .filter(file.folder_id == folderid, file.name.endswith(postfix)) + .order_by(file.id) .all() ) return files_with_ending @@ -72,7 +76,10 @@ def get_files_by_type(folderid, filetype, session=None): try: files_with_type = ( - session.query(file).filter(file.folder_id == folderid, file.type == filetype).all() + session.query(file) + .filter(file.folder_id == folderid, file.type == filetype) + .order_by(file.id) + .all() ) return files_with_type except NoResultFound: diff --git a/back-end/controllers/database_controller/kml_ops.py b/back-end/controllers/database_controller/kml_ops.py index cd45e58..f998aa6 100755 --- a/back-end/controllers/database_controller/kml_ops.py +++ b/back-end/controllers/database_controller/kml_ops.py @@ -2,25 +2,20 @@ import json import logging from io import StringIO -from multiprocessing import Lock import geopandas import pandas -import shapely from shapely.geometry import shape -from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from sqlalchemy.exc import SQLAlchemyError from database.models import fabric_data, kml_data from database.sessions import Session from utils.logger_config import logger -from utils.settings import BATCH_SIZE from .file_editfile_link_ops import get_editfiles_for_file from .file_ops import get_file_with_id, get_files_by_type, get_files_with_postfix from .geo_io import read_geo_bytes, suffix_for -db_lock = Lock() - def get_kml_data(folderid, session=None): owns_session = False @@ -39,6 +34,8 @@ def get_kml_data(folderid, session=None): all_data = {} if len(fabric_files) > 0: for fabric_file in fabric_files: + # Ordered so the tile pipeline's feature stream (and therefore + # tile bytes) is a deterministic function of the data. all_locations = ( session.query( fabric_data.location_id, @@ -48,8 +45,9 @@ def get_kml_data(folderid, session=None): fabric_data.bsl_flag, ) .filter(fabric_data.file_id == fabric_file.id) + .order_by(fabric_data.location_id) .all() - ) # Change to fabric_file.id + ) # Initialize a dictionary to hold location_id as key and its data as value, including location_id itself all_data.update( @@ -195,63 +193,93 @@ def get_kml_data_by_file(fileid, session=None): def add_to_db(pandaDF, kmlid, download, upload, tech, wireless, latency, category, session): - batch = [] - + """Write one coverage file's computed rows via COPY (one statement on the + session's own connection, so it commits with the session) — orders of + magnitude faster than per-row ORM objects for the big result sets.""" fileVal = get_file_with_id(kmlid) - for _, row in pandaDF.iterrows(): - try: - if row.location_id == "": - continue + try: + rows = pandaDF[pandaDF.location_id != ""] + if download == "": + download = 0 - if download == "": - download = 0 - - newData = kml_data( - location_id=int(row.location_id), - served=True, - wireless=wireless, - lte=False, - coveredLocations=fileVal.name, - maxDownloadNetwork=fileVal.name, - maxDownloadSpeed=int(download), - maxUploadSpeed=int(upload), - techType=tech, - file_id=fileVal.id, - address_primary=row.address_primary, - longitude=row.longitude, - latitude=row.latitude, - latency=latency, - category=category, + columns = [ + "location_id", + "served", + "wireless", + "lte", + "coveredLocations", + "maxDownloadNetwork", + "maxDownloadSpeed", + "maxUploadSpeed", + "techType", + "file_id", + "address_primary", + "longitude", + "latitude", + "latency", + "category", + ] + out = pandas.DataFrame( + { + "location_id": rows.location_id.astype(int).values, + "served": True, + "wireless": bool(wireless), + "lte": False, + "coveredLocations": fileVal.name, + "maxDownloadNetwork": fileVal.name, + "maxDownloadSpeed": int(download), + "maxUploadSpeed": int(upload), + "techType": tech, + "file_id": fileVal.id, + "address_primary": rows.address_primary.values, + "longitude": rows.longitude.values, + "latitude": rows.latitude.values, + "latency": latency, + "category": category, + }, + columns=columns, + ) + + if len(out): + buf = StringIO() + out.to_csv(buf, index=False, header=False, na_rep="\\N") + buf.seek(0) + cols_sql = ", ".join(f'"{c}"' for c in columns) + cursor = session.connection().connection.cursor() + cursor.copy_expert( + f"COPY kml_data ({cols_sql}) FROM STDIN WITH (FORMAT csv, NULL '\\N')", buf ) - batch.append(newData) - - if len(batch) >= BATCH_SIZE: - with db_lock: - try: - session.bulk_save_objects(batch) - session.commit() - except IntegrityError: - session.rollback() - - batch = [] - except Exception as e: - logging.error(f"Error occurred while inserting data: {e}") - return False - - if batch: - with db_lock: - try: - session.bulk_save_objects(batch) - session.commit() - except IntegrityError: - session.rollback() - session.commit() + session.commit() + except Exception as e: + session.rollback() + logging.error(f"Error occurred while inserting data: {e}") + return False return True -def generate_csv_data(results, provider_id, brand_name): +def filter_max_service(availability_csv): + """One BDC row per location: the fastest claim wins (download desc, then + upload desc, then low-latency first, then lowest technology code as a + deterministic tiebreak). The surviving row keeps its own values. The BDC + accepts multiple technology claims per location, so this is OPTIONAL + behavior behind the export_max_service_only site setting — default is to + report every claim.""" + ordered = availability_csv.sort_values( + by=[ + "max_advertised_download_speed", + "max_advertised_upload_speed", + "low_latency", + "technology", + ], + ascending=[False, False, False, True], + kind="mergesort", + ) + return ordered.drop_duplicates(subset=["location_id"], keep="first") + + +def generate_csv_data(results, provider_id, brand_name, max_service_only=False): availability_csv = pandas.DataFrame() availability_csv["location_id"] = [row.location_id for row in results] @@ -266,6 +294,8 @@ def generate_csv_data(results, provider_id, brand_name): availability_csv.drop_duplicates( subset=["location_id", "technology"], keep="first", inplace=True ) + if max_service_only: + availability_csv = filter_max_service(availability_csv) availability_csv = availability_csv[ [ "provider_id", @@ -294,7 +324,11 @@ def export(folderid, providerid, brandname, deadline, session, dispatch_copy=Tru all_file_ids = [file.id for file in all_files] results = session.query(kml_data).filter(kml_data.file_id.in_(all_file_ids)).all() - availability_csv = generate_csv_data(results, providerid, brandname) + from .setting_ops import export_max_service_only + + availability_csv = generate_csv_data( + results, providerid, brandname, max_service_only=export_max_service_only(session) + ) output = io.BytesIO() availability_csv.to_csv(output, index=False, encoding="utf-8") @@ -404,28 +438,35 @@ def reapply_plan_markers(coverage_file, session): session.commit() -def compute_wireless_locations(folderid, kmlid, download, upload, tech, latency, category, session): +def load_fabric_gdf(folderid): + """Parse a folder's active-fabric CSVs into ONE point GeoDataFrame. - # Only the active (BSL) fabric drives coverage — non_bsl and supplemental - # CSVs live in the folder too but never enter computation. + This is the heavy, per-folder-constant part of every coverage compute + (an ~80 MB CSV parse + point construction), so callers recomputing + several coverage files should load it once and pass it to + add_network_data instead of paying it per file. Only the active (BSL) + fabric drives coverage — non_bsl and supplemental CSVs live in the + folder too but never enter computation. Returns None with no fabric.""" fabric_files = get_files_by_type(folderid, "fabric") - coverage_file = get_file_with_id(kmlid) - - if fabric_files is None or coverage_file is None: - raise FileNotFoundError("Fabric or coverage file not found in the database") - - fabric_arr = [] - for fabric_file in fabric_files: - tempdf = pandas.read_csv(StringIO(fabric_file.data.decode())) - fabric_arr.append(tempdf) - df = pandas.concat(fabric_arr) - - fabric = geopandas.GeoDataFrame( + if not fabric_files: + return None + df = pandas.concat([pandas.read_csv(StringIO(ff.data.decode())) for ff in fabric_files]) + return geopandas.GeoDataFrame( df, crs="EPSG:4326", - geometry=[shapely.geometry.Point(xy) for xy in zip(df.longitude, df.latitude)], + geometry=geopandas.points_from_xy(df.longitude, df.latitude), ) + +def compute_wireless_locations( + folderid, kmlid, download, upload, tech, latency, category, session, fabric=None +): + coverage_file = get_file_with_id(kmlid) + if fabric is None: + fabric = load_fabric_gdf(folderid) + if fabric is None or coverage_file is None: + raise FileNotFoundError("Fabric or coverage file not found in the database") + wireless_coverage = read_geo_bytes(coverage_file.data, suffix_for(coverage_file.name)) wireless_coverage = wireless_coverage.to_crs("EPSG:4326") @@ -453,26 +494,13 @@ def compute_wireless_locations(folderid, kmlid, download, upload, tech, latency, def preview_wireless_locations(folderid, kml_filename): - - fabric_files = get_files_by_type(folderid, "fabric") with open(kml_filename, "rb") as file: # Open the file in binary mode coverage_data = file.read() # Read the entire content of the file into memory - if fabric_files is None: + fabric = load_fabric_gdf(folderid) + if fabric is None: raise FileNotFoundError("Fabric or coverage file not found in the database") - fabric_arr = [] - for fabric_file in fabric_files: - tempdf = pandas.read_csv(StringIO(fabric_file.data.decode())) - fabric_arr.append(tempdf) - df = pandas.concat(fabric_arr) - - fabric = geopandas.GeoDataFrame( - df, - crs="EPSG:4326", - geometry=[shapely.geometry.Point(xy) for xy in zip(df.longitude, df.latitude)], - ) - wireless_coverage = read_geo_bytes(coverage_data, ".kml") wireless_coverage = wireless_coverage.to_crs("EPSG:4326") @@ -488,20 +516,14 @@ def preview_wireless_locations(folderid, kml_filename): return bsl_fabric_in_wireless -def compute_wired_locations(folderid, kmlid, download, upload, tech, latency, category, session): - - # Fetch the active (BSL) fabric from the database — never non_bsl / - # supplemental CSVs (see compute_wireless_locations). - fabric_files = get_files_by_type(folderid, "fabric") - if not fabric_files: +def compute_wired_locations( + folderid, kmlid, download, upload, tech, latency, category, session, fabric=None +): + if fabric is None: + fabric = load_fabric_gdf(folderid) + if fabric is None: raise ValueError("No fabric file found") - fabric_arr = [] - for fabric_file in fabric_files: - tempdf = pandas.read_csv(StringIO(fabric_file.data.decode())) - fabric_arr.append(tempdf) - df = pandas.concat(fabric_arr) - # Fetch Fiber file from database fiber_file_record = get_file_with_id(kmlid) if not fiber_file_record: @@ -509,12 +531,6 @@ def compute_wired_locations(folderid, kmlid, download, upload, tech, latency, ca f"No file found with name {fiber_file_record.name} and id {fiber_file_record.id}" ) - fabric = geopandas.GeoDataFrame( - df, - crs="EPSG:4326", - geometry=[shapely.geometry.Point(xy) for xy in zip(df.longitude, df.latitude)], - ) - # Per-file override (files & plans "advanced" setting); NULL keeps the # pipeline's longstanding 100 m default. buffer_meters = fiber_file_record.coverage_buffer_m or 100 @@ -549,14 +565,19 @@ def compute_wired_locations(folderid, kmlid, download, upload, tech, latency, ca return res -def add_network_data(folderid, kmlid, download, upload, tech, type, latency, category, session): +def add_network_data( + folderid, kmlid, download, upload, tech, type, latency, category, session, fabric=None +): + """Compute one coverage file's served locations. `fabric` is the optional + preloaded load_fabric_gdf() frame — pass it when computing several files + so the fabric is parsed once, not per file.""" res = False if type == 0: res = compute_wired_locations( - folderid, kmlid, download, upload, tech, latency, category, session + folderid, kmlid, download, upload, tech, latency, category, session, fabric=fabric ) elif type == 1: res = compute_wireless_locations( - folderid, kmlid, download, upload, tech, latency, category, session + folderid, kmlid, download, upload, tech, latency, category, session, fabric=fabric ) return res diff --git a/back-end/controllers/database_controller/setting_ops.py b/back-end/controllers/database_controller/setting_ops.py index dba2ff4..57eefd9 100644 --- a/back-end/controllers/database_controller/setting_ops.py +++ b/back-end/controllers/database_controller/setting_ops.py @@ -21,3 +21,10 @@ def set_setting(key, value, session): def get_site_theme(session): theme = get_setting("site_theme", DEFAULT_THEME, session) return theme if theme in SITE_THEMES else DEFAULT_THEME + + +def export_max_service_only(session): + """When on, the BDC export reports only the fastest claim per location + instead of every (location, technology) claim. Default OFF — the BDC + accepts multiple technology claims per location.""" + return get_setting("export_max_service_only", "0", session) == "1" diff --git a/back-end/controllers/database_controller/vt_ops.py b/back-end/controllers/database_controller/vt_ops.py index fcb6414..d96e6c7 100755 --- a/back-end/controllers/database_controller/vt_ops.py +++ b/back-end/controllers/database_controller/vt_ops.py @@ -1,4 +1,4 @@ -import json +import math import os import sqlite3 import subprocess @@ -6,6 +6,7 @@ from datetime import datetime from multiprocessing import Lock +import orjson import psycopg2 from psycopg2 import Binary from psycopg2.extras import execute_values @@ -24,6 +25,25 @@ db_lock = Lock() +# The full pyramid is built in TWO tippecanoe runs merged into one tileset: +# +# z0-8 (overview) — density dropping allowed: a z0 tile contains every +# fabric point, so thinning is mandatory there. These zooms are only +# ever produced by full rebuilds. +# z9-16 (detail) — built with NO dropping of any kind (-pk -pf, no +# --drop-densest-as-needed, no byte cap) so a tile's bytes are a pure +# function of the features that intersect it. tippecanoe's +# --drop-densest-as-needed shares the min-gap it discovers on an +# oversized tile across the whole zoom level, making tile content +# depend on OTHER tiles' density — which both silently thinned dense +# areas and would make regenerating just an edited region produce +# different bytes than a full rebuild. Purity at z9-16 is what lets +# an edit retile regenerate only its dirty region and splice the rows +# into the live tileset. +TIPPECANOE_SHARED = "--base-zoom=7 -P --force --use-attribute-for-id=location_id --layer=data" +TIPPECANOE_LOW = f"-z 8 --maximum-tile-bytes=3000000 --drop-densest-as-needed {TIPPECANOE_SHARED}" +TIPPECANOE_HIGH = f"-Z 9 -z 16 -pk -pf {TIPPECANOE_SHARED}" + def _features_from_gdf(gdf, name, skip_points=False, keep_types=None): """Build GeoJSON Feature dicts from a GeoDataFrame, mirroring the legacy @@ -73,66 +93,61 @@ def read_geojson(fileid, session): ) -def add_values_to_VT(geojson_file_path, mbtiles_file_path, folderid): - with sqlite3.connect(mbtiles_file_path) as mb_conn: - mb_c = mb_conn.cursor() - mb_c.execute( - """ - SELECT zoom_level, tile_column, tile_row, tile_data - FROM tiles - """ - ) - - # Create a new connection to Postgres - conn = psycopg2.connect(DATABASE_URL) - cur = conn.cursor() - - try: - with open(mbtiles_file_path, "rb") as file: - mbtiles_data = Binary(file.read()) +def add_values_to_VT(geojson_file_path, mbtiles_file_paths, folderid): + """Store the tiles from one or more freshly built .mbtiles files as ONE + tileset: a single mbtiles anchor row (no file blob — vector_tiles rows + are the only thing ever served) plus all the tile rows. The zoom ranges + of the input files must be disjoint (z0-8 + z9-16).""" + conn = psycopg2.connect(DATABASE_URL) + cur = conn.cursor() - cur.execute("SELECT COUNT(*) FROM mbtiles WHERE folder_id = %s", (folderid,)) - count = cur.fetchone()[0] - cur.execute('SELECT "name" FROM "folder" WHERE id = %s', (folderid,)) - foldername = cur.fetchone()[0] - new_filename = f"{foldername}-{count + 1}.mbtiles" - - cur.execute( - """ - INSERT INTO mbtiles (tile_data, filename, timestamp, folder_id) - VALUES (%s, %s, %s, %s) RETURNING id - """, - (mbtiles_data, new_filename, datetime.now(), folderid), - ) + try: + cur.execute("SELECT COUNT(*) FROM mbtiles WHERE folder_id = %s", (folderid,)) + count = cur.fetchone()[0] + cur.execute('SELECT "name" FROM "folder" WHERE id = %s', (folderid,)) + foldername = cur.fetchone()[0] + new_filename = f"{foldername}-{count + 1}.mbtiles" - mbt_id = cur.fetchone()[0] + cur.execute( + """ + INSERT INTO mbtiles (tile_data, filename, timestamp, folder_id) + VALUES (%s, %s, %s, %s) RETURNING id + """, + (None, new_filename, datetime.now(), folderid), + ) - data = [(row[0], row[1], row[2], Binary(row[3]), mbt_id) for row in mb_c] + mbt_id = cur.fetchone()[0] + for mbtiles_file_path in mbtiles_file_paths: + with sqlite3.connect(mbtiles_file_path) as mb_conn: + mb_c = mb_conn.cursor() + mb_c.execute("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles") + data = [(row[0], row[1], row[2], Binary(row[3]), mbt_id) for row in mb_c] execute_values( cur, """ - INSERT INTO vector_tiles (zoom_level, tile_column, tile_row, tile_data, mbtiles_id) + INSERT INTO vector_tiles (zoom_level, tile_column, tile_row, tile_data, mbtiles_id) VALUES %s """, data, ) - # Commit the transaction - conn.commit() - except psycopg2.Error as e: - print(f"Database error occurred: {e}") - conn.rollback() - return -1 - except Exception as e: - print(f"Unexpected error occurred: {e}") - conn.rollback() - return -1 - finally: - cur.close() - conn.close() - os.remove(mbtiles_file_path) - os.remove(geojson_file_path) + conn.commit() + except psycopg2.Error as e: + print(f"Database error occurred: {e}") + conn.rollback() + return -1 + except Exception as e: + print(f"Unexpected error occurred: {e}") + conn.rollback() + return -1 + finally: + cur.close() + conn.close() + for mbtiles_file_path in mbtiles_file_paths: + if os.path.exists(mbtiles_file_path): + os.remove(mbtiles_file_path) + os.remove(geojson_file_path) return 1 @@ -187,56 +202,243 @@ def add_values_to_VT(geojson_file_path, mbtiles_file_path, folderid): # return None -def run_tippecanoe(command, folderid, geojsonpath, mbtilepath): +def run_tippecanoe(command): result = subprocess.run(command, shell=True, check=True, stderr=subprocess.PIPE) if result.stderr: print("Tippecanoe stderr:", result.stderr.decode()) - add_values_to_VT(geojsonpath, mbtilepath, folderid) return result.returncode +def _point_features(network_data): + """The point half of the tile feature stream, in get_kml_data order. Both + the full build and the splice MUST build features this way (same shape, + same relative order) for spliced tiles to byte-match a full rebuild.""" + return [ + { + "type": "Feature", + "properties": { + "location_id": point["location_id"], + "served": point["served"], + "address": point["address"], + "wireless": point["wireless"], + "lte": point["lte"], + "network_coverages": point["coveredLocations"], + "maxDownloadNetwork": point["maxDownloadNetwork"], + "maxDownloadSpeed": point["maxDownloadSpeed"], + "bsl": point["bsl"], + "feature_type": "Point", + }, + "geometry": { + "type": "Point", + "coordinates": [point["longitude"], point["latitude"]], + }, + } + for point in network_data + ] + + +def _write_ldjson(path, features): + # Newline-delimited features (not one giant FeatureCollection): orjson is + # much faster than json for the big point sets, and tippecanoe's -P can + # only parallelize input parsing on line-delimited input. + with open(path, "wb") as f: + for feat in features: + f.write(orjson.dumps(feat)) + f.write(b"\n") + + def create_tiles(geojson_array, folderid, session): network_data = get_kml_data(folderid, session) if network_data: - point_geojson = { - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": { - "location_id": point["location_id"], - "served": point["served"], - "address": point["address"], - "wireless": point["wireless"], - "lte": point["lte"], - "network_coverages": point["coveredLocations"], - "maxDownloadNetwork": point["maxDownloadNetwork"], - "maxDownloadSpeed": point["maxDownloadSpeed"], - "bsl": point["bsl"], - "feature_type": "Point", - }, - "geometry": { - "type": "Point", - "coordinates": [point["longitude"], point["latitude"]], - }, - } - for point in network_data - ], - } - - # print(geojson_array) - point_geojson["features"].extend(geojson for geojson in geojson_array) + features = _point_features(network_data) + features.extend(geojson_array) uuid_str = str(uuid.uuid4()) unique_geojson_filename = f"data{uuid_str}.geojson" + _write_ldjson(unique_geojson_filename, features) + + low_file = f"output{uuid_str}-low.mbtiles" + high_file = f"output{uuid_str}-high.mbtiles" + run_tippecanoe(f"tippecanoe -o {low_file} {TIPPECANOE_LOW} {unique_geojson_filename}") + run_tippecanoe(f"tippecanoe -o {high_file} {TIPPECANOE_HIGH} {unique_geojson_filename}") + add_values_to_VT(unique_geojson_filename, [low_file, high_file], folderid) + + +# ---- splice: regenerate only the z9-16 tiles a change touched -------------- + +SPLICE_MIN_Z = 9 +MAX_Z = 16 +# tippecanoe renders features up to its --buffer (default 5/256 of a tile) +# beyond a tile's edge; 16/256 of a z9 tile safely covers that overhang at +# every spliced zoom. +TILE_BUFFER_MARGIN = 16 / 256 + + +def _lonlat_to_z9_tile_f(lon, lat): + n = 2**SPLICE_MIN_Z + x = (lon + 180) / 360 * n + r = math.radians(lat) + y = (1 - math.log(math.tan(r) + 1 / math.cos(r)) / math.pi) / 2 * n + return x, y + + +def _z9_tile_f_to_lonlat(xf, yf): + n = 2**SPLICE_MIN_Z + lon = xf / n * 360 - 180 + lat = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * yf / n)))) + return lon, lat + + +def dirty_z9_ranges(bboxes): + """Snap lon/lat bboxes ([minx, miny, maxx, maxy]) to inclusive z9 XYZ + tile ranges (x0, x1, y0, y1). Each bbox is expanded by the tile-buffer + overhang BEFORE snapping: a changed point within buffer distance of a + region edge is also rendered by the neighboring tile, so that neighbor + must be regenerated too.""" + n = 2**SPLICE_MIN_Z + ranges = [] + for minx, miny, maxx, maxy in bboxes: + x0f, y0f = _lonlat_to_z9_tile_f(minx, maxy) # NW corner + x1f, y1f = _lonlat_to_z9_tile_f(maxx, miny) # SE corner + ranges.append( + ( + max(0, math.floor(x0f - TILE_BUFFER_MARGIN)), + min(n - 1, math.floor(x1f + TILE_BUFFER_MARGIN)), + max(0, math.floor(y0f - TILE_BUFFER_MARGIN)), + min(n - 1, math.floor(y1f + TILE_BUFFER_MARGIN)), + ) + ) + return ranges - with open(unique_geojson_filename, "w") as f: - json.dump(point_geojson, f) - outputFile = f"output{uuid_str}.mbtiles" - command = f"tippecanoe -o {outputFile} --base-zoom=7 -P --maximum-tile-bytes=3000000 -z 16 --drop-densest-as-needed {unique_geojson_filename} --force --use-attribute-for-id=location_id --layer=data" - run_tippecanoe(command, folderid, unique_geojson_filename, outputFile) +def _selection_box(rng): + """The lon/lat box of features a splice run needs for this region: the + region itself plus the buffer margin, so region-edge tiles get their full + buffer content.""" + x0, x1, y0, y1 = rng + minx, maxy = _z9_tile_f_to_lonlat(x0 - TILE_BUFFER_MARGIN, y0 - TILE_BUFFER_MARGIN) + maxx, miny = _z9_tile_f_to_lonlat(x1 + 1 + TILE_BUFFER_MARGIN, y1 + 1 + TILE_BUFFER_MARGIN) + return minx, miny, maxx, maxy + + +def _in_ranges(z, x, y_xyz, ranges): + """Is the (XYZ) tile inside any dirty z9 range (by z9 ancestor)?""" + k = z - SPLICE_MIN_Z + ax, ay = x >> k, y_xyz >> k + return any(x0 <= ax <= x1 and y0 <= ay <= y1 for x0, x1, y0, y1 in ranges) + + +def folder_coverage_features(folderid, session): + """Every coverage file's line/polygon features, in deterministic file + order — the geometry half of the tile feature stream.""" + from .file_ops import get_files_with_postfix + + features = [] + for kml_f in get_files_with_postfix(folderid, ".kml", session): + features.extend(read_kml(kml_f.id, session)) + for geojson_f in get_files_with_postfix(folderid, ".geojson", session): + features.extend(read_geojson(geojson_f.id, session)) + return features + + +def splice_tiles(folderid, bboxes, session): + """Regenerate the z9-16 tiles in the regions the given lon/lat bboxes + touch and replace those rows INSIDE the folder's current tileset (the + serving path never changes; z0-8 overview tiles go deliberately stale + until a settle rebuild). Because z9-16 is built drop-free, the spliced + tiles are byte-identical to what a full rebuild would produce. + + The splice input is the points inside the region + margin and every + coverage geometry passed WHOLE — clipping geometries (even with + tippecanoe's own --clip-bounding-box) perturbs polygon simplification + inside the region; out-of-region output tiles are simply discarded. + + Returns True on success, False when there's nothing to splice into or + nothing to draw (callers fall back to a full rebuild).""" + current = ( + session.query(mbtiles) + .filter(mbtiles.folder_id == folderid) + .order_by(desc(mbtiles.timestamp)) + .first() + ) + if current is None or not bboxes: + return False + network_data = get_kml_data(folderid, session) + if not network_data: + return False + + ranges = dirty_z9_ranges(bboxes) + boxes = [_selection_box(r) for r in ranges] + points = [ + f + for f in _point_features(network_data) + if any( + minx <= f["geometry"]["coordinates"][0] <= maxx + and miny <= f["geometry"]["coordinates"][1] <= maxy + for minx, miny, maxx, maxy in boxes + ) + ] + features = points + folder_coverage_features(folderid, session) + + uuid_str = str(uuid.uuid4()) + src = f"splice{uuid_str}.geojson" + out = f"splice{uuid_str}.mbtiles" + _write_ldjson(src, features) + try: + run_tippecanoe(f"tippecanoe -o {out} {TIPPECANOE_HIGH} {src}") + with sqlite3.connect(out) as mb_conn: + rows = mb_conn.execute( + "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles" + ).fetchall() + finally: + for path in (src, out): + if os.path.exists(path): + os.remove(path) + + new_tiles = [ + (z, x, y_tms, Binary(bytes(data)), current.id) + for z, x, y_tms, data in rows + if _in_ranges(z, x, (2**z - 1) - y_tms, ranges) + ] + + # Delete-then-insert the region's rows in ONE transaction, so a viewer + # never sees the region empty mid-splice. + conn = psycopg2.connect(DATABASE_URL) + cur = conn.cursor() + try: + for z in range(SPLICE_MIN_Z, MAX_Z + 1): + k = z - SPLICE_MIN_Z + n = 2**z + for x0, x1, y0, y1 in ranges: + ya, yb = y0 << k, ((y1 + 1) << k) - 1 # XYZ rows + cur.execute( + """ + DELETE FROM vector_tiles + WHERE mbtiles_id = %s AND zoom_level = %s + AND tile_column BETWEEN %s AND %s + AND tile_row BETWEEN %s AND %s + """, + (current.id, z, x0 << k, ((x1 + 1) << k) - 1, n - 1 - yb, n - 1 - ya), + ) + if new_tiles: + execute_values( + cur, + """ + INSERT INTO vector_tiles (zoom_level, tile_column, tile_row, tile_data, mbtiles_id) + VALUES %s + """, + new_tiles, + ) + conn.commit() + except Exception as e: + print(f"Splice failed, tiles unchanged: {e}") + conn.rollback() + return False + finally: + cur.close() + conn.close() + return True def retrieve_tiles(zoom, x, y, folderid): diff --git a/back-end/database/models.py b/back-end/database/models.py index 8fae15f..912e612 100755 --- a/back-end/database/models.py +++ b/back-end/database/models.py @@ -492,9 +492,11 @@ class mbtiles(Base): vector_tiles = relationship("vector_tiles", back_populates="mbtiles", cascade="all, delete") def copy(self, session, new_folder_id): + # The vector_tiles rows are the tileset; the mbtiles file blob is no + # longer stored (nothing ever read it back). new_mbtile = mbtiles( filename=self.filename, - tile_data=self.tile_data, + tile_data=None, timestamp=datetime.now(), folder_id=new_folder_id, ) diff --git a/back-end/pyproject.toml b/back-end/pyproject.toml index edb725d..1bcea8c 100644 --- a/back-end/pyproject.toml +++ b/back-end/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "Pillow>=10.3", "shortuuid>=1.0.11", "flask-limiter>=3.8", + "orjson>=3.11.9", ] [dependency-groups] diff --git a/back-end/routes/admin_ui.py b/back-end/routes/admin_ui.py index cd549a4..ec864da 100644 --- a/back-end/routes/admin_ui.py +++ b/back-end/routes/admin_ui.py @@ -175,8 +175,8 @@ def audit(): @bp.route("/admin/settings", methods=["GET", "POST"]) @require_platform_admin def settings(): - """Site-wide settings (currently just the theme every provider-facing page - renders with). Theme classes are defined in static/app/bdk.css.""" + """Site-wide settings: the theme every provider-facing page renders with + (classes in static/app/bdk.css) and the export max-service-only toggle.""" from controllers.database_controller import setting_ops session = get_session() @@ -188,11 +188,18 @@ def settings(): error = "Unknown theme." else: setting_ops.set_setting("site_theme", theme, session) - log_action("set_site_theme", user_id=g.admin_user.id, details={"theme": theme}) + max_service = "1" if request.form.get("export_max_service_only") else "0" + setting_ops.set_setting("export_max_service_only", max_service, session) + log_action( + "set_site_settings", + user_id=g.admin_user.id, + details={"theme": theme, "export_max_service_only": max_service}, + ) saved = True ctx = { "themes": setting_ops.SITE_THEMES, "current": setting_ops.get_site_theme(session), + "max_service_only": setting_ops.export_max_service_only(session), "saved": saved, "error": error, } diff --git a/back-end/templates/admin/settings.html b/back-end/templates/admin/settings.html index f35ca1e..79fa635 100644 --- a/back-end/templates/admin/settings.html +++ b/back-end/templates/admin/settings.html @@ -14,6 +14,13 @@

Site settings

{% endfor %}

Applied to every provider-facing page site-wide. Providers don't choose themes.

+ +

When on, generated BDC reports include just the fastest claim per location + (by download, then upload) instead of every technology claim. The BDC accepts multiple + claims per location, so this is off by default.

{% endblock %} diff --git a/back-end/tests/test_admin_ui.py b/back-end/tests/test_admin_ui.py index a6ca8ca..aaa998b 100644 --- a/back-end/tests/test_admin_ui.py +++ b/back-end/tests/test_admin_ui.py @@ -190,3 +190,27 @@ def test_settings_rejects_unknown_theme(client): resp = client.post("/admin/settings", data={"csrf_token": csrf, "site_theme": "hotdog-stand"}) assert resp.status_code == 400 assert b"Unknown theme." in resp.data + + +def test_settings_max_service_only_roundtrip(client, db_session): + """The export max-service toggle persists (checkbox on -> '1', absent -> + '0') and is read back by the export-path helper. Default is OFF.""" + from controllers.database_controller.setting_ops import export_max_service_only + + assert export_max_service_only(db_session) is False + + _, csrf = _admin_session(client) + resp = client.post( + "/admin/settings", + data={"csrf_token": csrf, "site_theme": "civic-light", "export_max_service_only": "1"}, + ) + assert resp.status_code == 200 and b"Saved." in resp.data + db_session.expire_all() + assert export_max_service_only(db_session) is True + assert b"checked" in client.get("/admin/settings").data + + # Unchecking (the field absent from the POST) turns it back off. + resp = client.post("/admin/settings", data={"csrf_token": csrf, "site_theme": "civic-light"}) + assert resp.status_code == 200 + db_session.expire_all() + assert export_max_service_only(db_session) is False diff --git a/back-end/tests/test_csv_export.py b/back-end/tests/test_csv_export.py index 8289d8c..54010b4 100644 --- a/back-end/tests/test_csv_export.py +++ b/back-end/tests/test_csv_export.py @@ -47,3 +47,32 @@ def test_provider_and_brand_are_filled(): df = generate_csv_data([_row(1), _row(2)], provider_id=330054, brand_name="Acme") assert (df.provider_id == 330054).all() assert (df.brand_name == "Acme").all() + + +def test_default_reports_every_technology_claim(): + """The BDC accepts multiple technology claims per location; by default a + location under several coverages files a row per technology.""" + rows = [_row(1, tech=50), _row(1, tech=70), _row(1, tech=71)] + df = generate_csv_data(rows, provider_id=330054, brand_name="Acme") + assert set(zip(df.location_id, df.technology)) == {(1, 50), (1, 70), (1, 71)} + + +def test_max_service_only_picks_the_fastest_claim_per_location(): + """With max_service_only, a location files exactly ONE row — the fastest + claim: download desc, then upload desc, then low-latency first, then the + lowest technology code as a deterministic tiebreak. The surviving row + keeps its own values.""" + rows = [ + _row(1, tech=50, dl=1000, ul=1000), + _row(1, tech=70, dl=25, ul=3), # slower -> dropped + _row(2, tech=70, dl=100, ul=50), + _row(2, tech=71, dl=100, ul=75), # same download, faster upload -> wins + _row(3, tech=70, dl=100, ul=20, latency=0), + _row(3, tech=71, dl=100, ul=20, latency=1), # low latency wins the tie + _row(4, tech=71, dl=100, ul=20), + _row(4, tech=70, dl=100, ul=20), # full tie -> lowest tech code + ] + df = generate_csv_data(rows, provider_id=330054, brand_name="Acme", max_service_only=True) + assert set(zip(df.location_id, df.technology)) == {(1, 50), (2, 71), (3, 71), (4, 70)} + kept = df[df.location_id == 1].iloc[0] + assert kept.max_advertised_download_speed == 1000 # the winner's own values diff --git a/back-end/tests/test_prod_golden_csv.py b/back-end/tests/test_prod_golden_csv.py new file mode 100644 index 0000000..a80da32 --- /dev/null +++ b/back-end/tests/test_prod_golden_csv.py @@ -0,0 +1,84 @@ +"""Layer 4b — PROD golden for the EXPORT CSV: replay real production filings +and assert the availability CSV our export path generates matches the bytes +prod actually stored (the BDC report the provider filed). + +This pins the export-layer behavior that kml_data equality cannot see. The +shipped default reports every (location, technology) claim — the BDC accepts +multiple claims per location — which matches the reference era pinned here. +(Some historical snapshots were generated with a since-retired per-location +wireless dedup; they stay in the fixture set as reference only — see +csv-golden.json. The optional export_max_service_only setting, pinned by +unit tests in test_csv_export.py, is the supported variant of that idea.) + +References: prod-extract//output/prod-stored-export.csv and +prod-extract/csv-golden.json, which maps each upload folder to the stored +export representing its filed CSV (fixtures are local-only, never committed). + +Rows are compared as parsed CSV records, order-insensitively — physical row +order and quoting style are not part of the contract; field values are. +""" + +import csv +import io +import json +import os + +import pytest + +from controllers.database_controller import kml_ops +from database.models import folder as folder_model +from database.models import organization as organization_model +from tests.conftest import REAL_DIR +from tests.test_prod_golden import _replay + +PROD = os.path.join(REAL_DIR, "prod-extract") +MAPPING = os.path.join(PROD, "csv-golden.json") + +pytestmark = [ + pytest.mark.golden, + pytest.mark.realdata, + pytest.mark.skipif(not os.path.isfile(MAPPING), reason="csv golden fixtures not present"), +] + + +def _cases(): + if not os.path.isfile(MAPPING): + return [] + mapping = json.load(open(MAPPING)) + return [(int(fid), exp) for fid, exp in mapping.items() if not fid.startswith("_")] + + +def _rows(text): + if isinstance(text, bytes): + text = text.decode("utf-8") + reader = csv.reader(io.StringIO(text)) + rows = list(reader) + return rows[0], sorted(tuple(r) for r in rows[1:]) + + +@pytest.mark.parametrize("fid,export_dir", _cases()) +def test_export_csv_matches_prod_filed_bytes(db_session, fid, export_dir): + s = db_session + _replay(s, fid) + org = s.query(organization_model).filter_by(name=f"org{fid}").one() + folder = s.query(folder_model).filter_by(organization_id=org.id).one() + + out = kml_ops.export( + folder.id, + org.provider_id, + org.brand_name, + folder.deadline.strftime("%Y-%m-%d"), + s, + dispatch_copy=False, + ) + ours_header, ours = _rows(out.getvalue()) + + with open(os.path.join(PROD, export_dir, "output", "prod-stored-export.csv")) as fh: + ref_header, ref = _rows(fh.read()) + + assert ours_header == ref_header + assert len(ours) == len(ref), ( + f"folder {fid}: {len(ours)} rows generated vs {len(ref)} rows prod filed " + f"(export folder {export_dir})" + ) + assert ours == ref, f"folder {fid}: CSV content diverges from prod's filed export" diff --git a/back-end/tests/test_tiles_pipeline.py b/back-end/tests/test_tiles_pipeline.py new file mode 100644 index 0000000..e2fb062 --- /dev/null +++ b/back-end/tests/test_tiles_pipeline.py @@ -0,0 +1,269 @@ +"""Tile-pipeline tests running the REAL create_tiles (tippecanoe, in Docker). + +Tiles previously had no correctness gate at all (the goldens assert kml_data +and the export CSV; the map tests assert routing against pre-seeded rows). +These tests pin the storage contract of the build pipeline itself: + +- a full build produces ONE tileset: a single mbtiles row whose vector_tiles + rows span the whole z0-16 pyramid (internally it is two tippecanoe runs - + overview zooms z0-8 with density dropping, detail zooms z9-16 drop-free - + merged into that one row set); +- the mbtiles file blob is NOT stored (vector_tiles rows are the only + serving truth, and nothing ever read the blob back); +- folder.copy snapshots carry the tile rows, not a blob; +- the rebuild path (_rebuild_folder_tiles) replaces the tileset instead of + accumulating, and serves through retrieve_tiles afterwards. +""" + +import gzip +import json +import math + +import pytest + +from tests import conftest_helpers as H + +pytestmark = pytest.mark.tippecanoe + + +# A small grid of fabric locations around (-80.0, 37.25) with a coverage +# polygon over its middle: enough features for tippecanoe to emit tiles at +# every zoom, small enough to tile in well under a second. +GRID_N = 20 +LON0, LAT0, STEP = -80.10, 37.20, 0.01 +COV_POLY = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-80.06, 37.24], + [-79.96, 37.24], + [-79.96, 37.30], + [-80.06, 37.30], + [-80.06, 37.24], + ] + ], + }, + } + ], +} + + +def lonlat_to_tile(lon, lat, z): + n = 2**z + x = int((lon + 180) / 360 * n) + r = math.radians(lat) + y = int((1 - math.log(math.tan(r) + 1 / math.cos(r)) / math.pi) / 2 * n) + return x, y + + +def seed_tile_folder(s): + """Org + folder + fabric grid + one coverage polygon with served points.""" + from database.models import fabric_data, kml_data + + org = H.make_org(s) + folder = H.make_folder(s, org.id) + fab = H.seed_coverage(s, folder.id, b"stub", "fab.csv", "fabric", None) + cov = H.seed_coverage(s, folder.id, json.dumps(COV_POLY).encode(), "cov.geojson", "wired", 50) + + loc = 1000 + for i in range(GRID_N): + for j in range(GRID_N): + lon, lat = LON0 + i * STEP, LAT0 + j * STEP + s.add( + fabric_data( + file_id=fab.id, + location_id=loc, + latitude=lat, + longitude=lon, + address_primary=f"{loc} MAIN ST", + bsl_flag="True", + ) + ) + if -80.06 <= lon <= -79.96 and 37.24 <= lat <= 37.30: + s.add( + kml_data( + location_id=loc, + served=True, + wireless=False, + lte=False, + coveredLocations="cov.geojson", + maxDownloadNetwork="cov.geojson", + maxDownloadSpeed=100, + maxUploadSpeed=20, + techType=50, + file_id=cov.id, + address_primary=f"{loc} MAIN ST", + longitude=lon, + latitude=lat, + ) + ) + loc += 1 + s.commit() + return folder, fab, cov + + +def build_tiles(s, folder, cov): + from controllers.database_controller import vt_ops + + geojson_array = vt_ops.read_geojson(cov.id, s) + vt_ops.create_tiles(geojson_array, folder.id, s) + + +def folder_tilesets(s, folderid): + from database.models import mbtiles + + return s.query(mbtiles).filter(mbtiles.folder_id == folderid).all() + + +def tile_rows(s, mbtiles_id): + from database.models import vector_tiles + + return s.query(vector_tiles).filter(vector_tiles.mbtiles_id == mbtiles_id).all() + + +def test_full_build_one_tileset_full_pyramid_no_blob(db_session): + from controllers.database_controller import vt_ops + + folder, fab, cov = seed_tile_folder(db_session) + build_tiles(db_session, folder, cov) + db_session.expire_all() + + sets = folder_tilesets(db_session, folder.id) + assert len(sets) == 1 + assert sets[0].tile_data is None # blob is dead weight; rows are the truth + + rows = tile_rows(db_session, sets[0].id) + zooms = {r.zoom_level for r in rows} + assert zooms == set(range(17)) # both runs landed in the one tileset + + # The grid center must serve at an overview zoom and a detail zoom. + center_lon, center_lat = LON0 + GRID_N // 2 * STEP, LAT0 + GRID_N // 2 * STEP + for z in (6, 9, 16): + x, y = lonlat_to_tile(center_lon, center_lat, z) + tms_y = (2**z - 1) - y + tile = vt_ops.retrieve_tiles(z, x, tms_y, folder.id) + assert tile is not None, f"no tile at z{z}" + assert tile.tile_data[:2] == b"\x1f\x8b" # gzipped PBF + assert gzip.decompress(tile.tile_data) + + +def test_rebuild_replaces_tileset(db_session): + from controllers.celery_controller import celery_tasks as ct + + folder, fab, cov = seed_tile_folder(db_session) + build_tiles(db_session, folder, cov) + first = folder_tilesets(db_session, folder.id)[0].id + + # The rebuild path (edit retiles, regenerate) must swap, not accumulate. + ct._rebuild_folder_tiles(folder.id) + db_session.expire_all() + sets = folder_tilesets(db_session, folder.id) + assert len(sets) == 1 + assert sets[0].id != first + assert sets[0].tile_data is None + assert {r.zoom_level for r in tile_rows(db_session, sets[0].id)} == set(range(17)) + + +def snapshot_rows(s, mbtiles_id): + return { + (r.zoom_level, r.tile_column, r.tile_row): bytes(r.tile_data) + for r in tile_rows(s, mbtiles_id) + } + + +def test_splice_matches_full_rebuild_byte_for_byte(db_session): + """THE splice gate: after an edit-shaped change, regenerating only the + dirty region and splicing the rows into the live tileset must produce + byte-identical z9-16 tiles to a full rebuild — inside the region (same + content) and outside it (untouched). z0-8 stays deliberately stale.""" + from controllers.celery_controller import celery_tasks as ct + from controllers.database_controller import vt_ops + from database.models import kml_data + + folder, fab, cov = seed_tile_folder(db_session) + build_tiles(db_session, folder, cov) + db_session.expire_all() + tileset = folder_tilesets(db_session, folder.id)[0] + before = snapshot_rows(db_session, tileset.id) + + # The "edit": exclude every served location inside a small box (exactly + # what an exclusion edit does - it deletes the kml rows). + EDIT = (-80.03, 37.25, -80.00, 37.27) # minx, miny, maxx, maxy + doomed = ( + db_session.query(kml_data) + .filter( + kml_data.file_id == cov.id, + kml_data.longitude >= EDIT[0], + kml_data.longitude <= EDIT[2], + kml_data.latitude >= EDIT[1], + kml_data.latitude <= EDIT[3], + ) + .all() + ) + assert doomed, "the edit box must catch served points" + for row in doomed: + db_session.delete(row) + db_session.commit() + + assert vt_ops.splice_tiles(folder.id, [list(EDIT)], db_session) is True + db_session.expire_all() + sets = folder_tilesets(db_session, folder.id) + assert len(sets) == 1 and sets[0].id == tileset.id # spliced IN PLACE + spliced = snapshot_rows(db_session, tileset.id) + + # Reference: what a full rebuild produces from the same DB truth. + ct._rebuild_folder_tiles(folder.id) + db_session.expire_all() + rebuilt = snapshot_rows(db_session, folder_tilesets(db_session, folder.id)[0].id) + + spliced_hi = {k: v for k, v in spliced.items() if k[0] >= 9} + rebuilt_hi = {k: v for k, v in rebuilt.items() if k[0] >= 9} + assert spliced_hi.keys() == rebuilt_hi.keys() + diff = [k for k in rebuilt_hi if spliced_hi[k] != rebuilt_hi[k]] + assert diff == [], f"{len(diff)} z9-16 tiles differ from a full rebuild" + + # The splice must have actually changed something (the edit is visible)... + ranges = vt_ops.dirty_z9_ranges([list(EDIT)]) + changed = [k for k in spliced_hi if spliced_hi[k] != before.get(k)] + assert changed, "the spliced region should differ from the pre-edit tiles" + # ...while every changed tile lies inside the dirty region (locality). + for z, x, y_tms in changed: + y_xyz = (2**z - 1) - y_tms + assert vt_ops._in_ranges(z, x, y_xyz, ranges), ( + f"tile {(z, x, y_tms)} outside the region changed" + ) + + # z0-8 untouched (stale by design until the settle rebuild). + for k, v in spliced.items(): + if k[0] <= 8: + assert before[k] == v + + +def test_splice_without_tileset_falls_back(db_session): + from controllers.database_controller import vt_ops + + folder, fab, cov = seed_tile_folder(db_session) + # No tiles built yet -> nothing to splice into. + assert vt_ops.splice_tiles(folder.id, [[-80.03, 37.25, -80.00, 37.27]], db_session) is False + + +def test_folder_copy_carries_rows_not_blob(db_session): + folder, fab, cov = seed_tile_folder(db_session) + build_tiles(db_session, folder, cov) + db_session.expire_all() + src = folder_tilesets(db_session, folder.id)[0] + n_rows = len(tile_rows(db_session, src.id)) + assert n_rows > 0 + + new_folder = folder.copy(session=db_session, export=True) + db_session.commit() + copies = folder_tilesets(db_session, new_folder.id) + assert len(copies) == 1 + assert copies[0].tile_data is None + assert len(tile_rows(db_session, copies[0].id)) == n_rows diff --git a/back-end/tests/test_tiles_split.py b/back-end/tests/test_tiles_split.py index 7d51017..ae75f84 100644 --- a/back-end/tests/test_tiles_split.py +++ b/back-end/tests/test_tiles_split.py @@ -32,21 +32,52 @@ def tasks(monkeypatch): """celery_tasks with the tile pipeline stubbed; records rebuild calls.""" from controllers.celery_controller import celery_tasks as ct - calls = {"create_tiles": [], "delete_mbtiles": []} + calls = {"create_tiles": [], "delete_mbtiles": [], "splice": [], "splice_result": True} monkeypatch.setattr( ct.vt_ops, "create_tiles", lambda gj, fid, s: calls["create_tiles"].append(fid) ) monkeypatch.setattr( ct.mbtiles_ops, "delete_mbtiles", lambda fid, s: calls["delete_mbtiles"].append(fid) ) - empty = {"type": "FeatureCollection", "features": []} - monkeypatch.setattr(ct.vt_ops, "read_kml", lambda fid, s: empty) - monkeypatch.setattr(ct.vt_ops, "read_geojson", lambda fid, s: empty) + + def fake_splice(fid, bboxes, s): + calls["splice"].append((fid, bboxes)) + return calls["splice_result"] + + monkeypatch.setattr(ct.vt_ops, "splice_tiles", fake_splice) + # read_kml/read_geojson return flat feature LISTS (one ldjson line each). + monkeypatch.setattr(ct.vt_ops, "read_kml", lambda fid, s: []) + monkeypatch.setattr(ct.vt_ops, "read_geojson", lambda fid, s: []) return ct, calls +def _b(value): + return value.encode() if isinstance(value, str) else value + + +class _StubPipeline: + """Queues commands like redis-py's transactional pipeline.""" + + def __init__(self, stub): + self.stub = stub + self.ops = [] + + def lrange(self, key, start, end): + self.ops.append(lambda: self.stub.lrange(key, start, end)) + + def get(self, key): + self.ops.append(lambda: self.stub.get(key)) + + def delete(self, key): + self.ops.append(lambda: self.stub.delete(key)) + + def execute(self): + return [op() for op in self.ops] + + class _StubRedis: - """Just enough of the redis interface for the dirty/lock flags.""" + """Just enough of the redis interface for the dirty/lock/settle flags. + Returns bytes like redis-py does.""" def __init__(self): self.store = {} @@ -60,11 +91,27 @@ def set(self, key, value, nx=False, ex=None): return True def get(self, key): - return self.store.get(key) + return _b(self.store.get(key)) def delete(self, key): self.store.pop(key, None) + def rpush(self, key, value): + self.store.setdefault(key, []).append(value) + self.sets.append(key) + + def lrange(self, key, start, end): + assert (start, end) == (0, -1) + return [_b(v) for v in self.store.get(key, [])] + + def scan_iter(self, match): + import fnmatch + + return [_b(k) for k in list(self.store) if fnmatch.fnmatch(k, match)] + + def pipeline(self, transaction=True): + return _StubPipeline(self) + def _seed_edit_fixture(s): """Org + folder + one coverage file with one served kml_data location.""" @@ -113,31 +160,96 @@ def test_regenerate_tiles_rebuilds(db_session, tasks): assert calls["create_tiles"] == [folder.id] -def test_regenerate_tiles_coalesces_when_fresh(db_session, tasks, monkeypatch): - """With redis flags: a rebuild only runs if the folder is dirty, so a - queued regenerate whose edits were covered by an earlier rebuild no-ops.""" +def test_regenerate_tiles_coalesces_and_splices(db_session, tasks, monkeypatch): + """With redis flags: a refresh only runs if the folder is dirty, and + edit-scoped (bbox) dirt is SPLICED — only the edited region's tiles are + regenerated, no full rebuild — leaving a settle flag for the stale + overview zooms.""" ct, calls = tasks s = db_session _, _, folder, _ = _seed_edit_fixture(s) stub = _StubRedis() monkeypatch.setattr(ct, "_tiles_redis", lambda: stub) - # not dirty -> tiles already cover current truth -> no rebuild + # not dirty -> tiles already cover current truth -> no refresh res = ct.regenerate_tiles.apply_async(args=[folder.id]).get() assert calls["create_tiles"] == [] assert "fresh" in res - # the apply phase marks the folder dirty -> next regenerate rebuilds + clears + # the apply phase records the edit's bbox -> next regenerate splices markers = [[{"id": 123, "editedFile": ["cov.kml"]}]] ct.apply_edit_changes.apply_async(args=[markers, folder.id, [POLYGON]]).get() - assert stub.get(f"bdk:tiles-dirty:{folder.id}") is not None + assert stub.get(f"bdk:tiles-dirty:{folder.id}") == b"bbox" res = ct.regenerate_tiles.apply_async(args=[folder.id]).get() - assert calls["create_tiles"] == [folder.id] - assert "rebuilt" in res + assert "spliced" in res + assert calls["create_tiles"] == [] # no full rebuild + assert calls["splice"] == [(folder.id, [[-80.01, 37.27, -80.0, 37.28]])] assert stub.get(f"bdk:tiles-dirty:{folder.id}") is None + assert stub.get(f"bdk:tiles-dirty-bbox:{folder.id}") is None + assert stub.get(f"bdk:tiles-settle:{folder.id}") is not None # z0-8 stale assert stub.get(f"bdk:tiles-lock:{folder.id}") is None # lock released +def test_splice_failure_falls_back_to_full_rebuild(db_session, tasks, monkeypatch): + ct, calls = tasks + s = db_session + _, _, folder, _ = _seed_edit_fixture(s) + stub = _StubRedis() + monkeypatch.setattr(ct, "_tiles_redis", lambda: stub) + calls["splice_result"] = False # e.g. no tileset to splice into + + markers = [[{"id": 123, "editedFile": ["cov.kml"]}]] + ct.apply_edit_changes.apply_async(args=[markers, folder.id, [POLYGON]]).get() + res = ct.regenerate_tiles.apply_async(args=[folder.id]).get() + assert "rebuilt" in res + assert calls["splice"] != [] # tried + assert calls["create_tiles"] == [folder.id] # fell back + assert stub.get(f"bdk:tiles-settle:{folder.id}") is None # full = settled + + +def test_full_dirt_wins_over_bboxes(db_session, tasks, monkeypatch): + """A whole-tileset change (upload, recompute) coalescing with an edit must + full-rebuild — splicing only the edit's region would miss the rest.""" + ct, calls = tasks + s = db_session + _, _, folder, _ = _seed_edit_fixture(s) + stub = _StubRedis() + monkeypatch.setattr(ct, "_tiles_redis", lambda: stub) + + ct._mark_tiles_dirty(folder.id) # process_data style: everything stale + ct._mark_tiles_dirty(folder.id, bbox=[-80.01, 37.27, -80.0, 37.28]) + assert stub.get(f"bdk:tiles-dirty:{folder.id}") == b"full" # bbox didn't downgrade + + res = ct.regenerate_tiles.apply_async(args=[folder.id]).get() + assert "rebuilt" in res + assert calls["splice"] == [] + assert calls["create_tiles"] == [folder.id] + + +def test_settle_task_rebuilds_quiet_folders(db_session, tasks, monkeypatch): + import time as time_mod + + ct, calls = tasks + s = db_session + _, _, folder, _ = _seed_edit_fixture(s) + stub = _StubRedis() + monkeypatch.setattr(ct, "_tiles_redis", lambda: stub) + dispatched = [] + monkeypatch.setattr(ct.regenerate_tiles, "apply_async", lambda args: dispatched.append(args[0])) + + # A folder spliced moments ago: still in its quiet window -> left alone. + stub.set(f"bdk:tiles-settle:{folder.id}", str(time_mod.time())) + ct.settle_stale_tiles.run() + assert dispatched == [] + + # Quiet long enough -> one full rebuild dispatched, flag consumed. + stub.set(f"bdk:tiles-settle:{folder.id}", str(time_mod.time() - ct.TILES_SETTLE_QUIET - 1)) + ct.settle_stale_tiles.run() + assert dispatched == [folder.id] + assert stub.get(f"bdk:tiles-settle:{folder.id}") is None + assert stub.get(f"bdk:tiles-dirty:{folder.id}") == b"full" + + def test_process_data_uses_coalesced_rebuild(db_session, tasks, monkeypatch): """process_data's tile step goes through the same per-folder lock + dirty-flag path as edit retiles, so upload/delete/regenerate rebuilds diff --git a/back-end/uv.lock b/back-end/uv.lock index 97a9646..41852fb 100644 --- a/back-end/uv.lock +++ b/back-end/uv.lock @@ -75,6 +75,7 @@ dependencies = [ { name = "geopandas" }, { name = "gunicorn" }, { name = "numpy" }, + { name = "orjson" }, { name = "pandas" }, { name = "pillow" }, { name = "psycopg2-binary" }, @@ -108,6 +109,7 @@ requires-dist = [ { name = "geopandas", specifier = ">=1.0" }, { name = "gunicorn", specifier = ">=22.0" }, { name = "numpy", specifier = ">=1.26,<3.0" }, + { name = "orjson", specifier = ">=3.11.9" }, { name = "pandas", specifier = ">=2.2,<3.0" }, { name = "pillow", specifier = ">=10.3" }, { name = "psycopg2-binary", specifier = ">=2.9.9" }, @@ -747,6 +749,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/55/af02708f230eb77084a299d7b08175cff006dea4f2721074b92cdb0296c0/ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562", size = 7634, upload-time = "2022-01-26T14:38:48.677Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2"