DINO export & import bug

backend/annotation_server.py

@@ -55,6 +55,7 @@
 from cache import TTLCache
 from coco_export import (
     build_export_plan,
+    fold_lightly_splits,
     lightly_classes_map,
     shape_to_mask,
     write_coco_split,
@@ -66,6 +67,7 @@
     ExportSourceItem,
     GuidePayload,
     ImageMeta,
+    IngestPreflightRequest,
     MeasureRequest,
     SaveVersionRequest,
 )
@@ -982,6 +984,7 @@ def _cb(message: str, _jid: str = jid) -> None:
                 sample_global_stats_fn=images_mod._sample_global_stats,
                 progress_cb=_cb,
                 include_polygons=payload.include_polygons,
+                lightly=lightly,
             )
             skipped_total += plan["skipped_zero_area"]
             if not merged_categories:
@@ -1011,6 +1014,10 @@ def _cb(message: str, _jid: str = jid) -> None:
                 (c.model_dump() if hasattr(c, "model_dump") else dict(c)) for c in payload.classes
             ],
         }
+        if lightly:
+            # Document the DINOv3/Lightly convention: masks are 0-indexed class ids
+            # with unannotated pixels set to this ignore index.
+            manifest["ignore_index"] = 255
 
         # Write files AND build the download .zip in one pass. ZIP_STORED: the
         # PNGs are already compressed, so re-deflating them is wasted CPU.
@@ -1028,14 +1035,14 @@ def _cb(message: str, _jid: str = jid) -> None:
             zf.writestr("manifest.json", manifest_json)
 
             if lightly:
-                # DINOv3 / Lightly: classes.json (index→name, 0=bg) at the dataset
-                # root; each split as images/ + masks/ with matching stems.
+                # DINOv3 / Lightly: classes.json (index→name, 0-indexed, no background)
+                # at the dataset root; each split as images/ + masks/ with matching
+                # stems. Unannotated pixels are 255 (the ignore index).
                 classes_json = json.dumps(lightly_classes_map(merged_categories), indent=2)
                 (out_root / "classes.json").write_text(classes_json)
                 zf.writestr("classes.json", classes_json.encode("utf-8"))
-                for split_name, split_data in merged_splits.items():
-                    # Lightly convention: 'valid' → 'val'; 'train'/'test' unchanged.
-                    dir_name = "val" if split_name == "valid" else split_name
+                # Lightly uses train/val only: 'valid' AND 'test' fold into 'val'.
+                for dir_name, split_data in fold_lightly_splits(merged_splits).items():
                     export_jobs.log(jid, f"Writing split '{dir_name}' ({len(split_data['images'])} images + masks)…")
                     written[dir_name] = write_lightly_split(
                         out_root / dir_name,
@@ -1155,18 +1162,45 @@ def _run() -> dict:
         raise HTTPException(500, f"Import failed: {exc}") from exc
 
 
+@app.post("/api/ingest/preflight")
+async def ingest_preflight(req: IngestPreflightRequest) -> dict:
+    """Report which of ``req.names`` already exist in ``req.container_path``.
+
+    Called before the upload so the user can resolve collisions (replace / skip /
+    new dataset / browse the existing one) instead of uploading files that would
+    each fail with a 409. POST (not GET) because a dropped folder easily carries
+    hundreds of filenames — see :class:`schemas.IngestPreflightRequest`.
+
+    Raises:
+        HTTPException: 502 if the Tiled server could not be read.
+    """
+    try:
+        return await asyncio.to_thread(
+            ingest_mod.preflight, req.server_uri, req.container_path, req.names
+        )
+    except Exception as exc:
+        logger.warning("ingest preflight failed for %s: %s", req.container_path, exc)
+        raise HTTPException(502, "Could not check the destination on the Tiled server") from exc
+
+
 @app.post("/api/ingest/upload")
 async def ingest_upload(
     server_uri: Optional[str] = Query(None, description="Target Tiled server URI"),
     container_path: str = Form(..., description="Target container, e.g. 'browse/myset'"),
     description: str = Form("", description="Optional keyword(s) stored on every ingested node"),
+    on_conflict: str = Form("fail", description="'fail', 'replace' or 'skip' for existing keys"),
     files: list[UploadFile] = File(..., description="Image files to copy into Tiled"),
 ) -> dict:
     """Stream uploaded files to temp storage and start a background ingest job.
 
     Each supported image becomes its own browsable node in *container_path* on
     the connected Tiled server. Returns a ``job_id`` to poll for progress.
     """
+    if on_conflict not in ingest_mod.ON_CONFLICT_MODES:
+        raise HTTPException(
+            400, f"on_conflict must be one of {sorted(ingest_mod.ON_CONFLICT_MODES)}"
+        )
+
     tmp_dir = Path(tempfile.mkdtemp(prefix="ingest_"))
     saved: list[tuple[str, Path]] = []
     for index, upload in enumerate(files):
@@ -1189,7 +1223,7 @@ async def ingest_upload(
     jid = ingest_mod.new_job(len(saved), server_uri, container_path)
     threading.Thread(
         target=ingest_mod.run_ingest_job,
-        args=(jid, server_uri, container_path, saved, description),
+        args=(jid, server_uri, container_path, saved, description, on_conflict),
         daemon=True,
     ).start()
     return {"job_id": jid, "total": len(saved), "container_path": container_path}

backend/coco_export.py

@@ -433,13 +433,27 @@ def _emit(rel: str, data: bytes) -> None:
     return {"n_images": n, "n_annotations": 0, "path": str(img_dir)}
 
 
+# Lightly/DINOv3 uses train/val only: the app's 'valid' AND 'test' both fold into 'val'.
+LIGHTLY_SPLIT_DIRS = {"train": "train", "valid": "val", "test": "val"}
+
+
+def fold_lightly_splits(merged_splits: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """Map the app's train/valid/test splits onto Lightly's train/val directories,
+    merging the image lists of any splits that map to the same directory (valid+test
+    → val). Returns ``{dir_name: {"images": [...]}}``."""
+    folded: dict[str, dict[str, Any]] = {}
+    for split_name, split_data in merged_splits.items():
+        dir_name = LIGHTLY_SPLIT_DIRS.get(split_name, split_name)
+        folded.setdefault(dir_name, {"images": []})["images"].extend(split_data.get("images", []))
+    return folded
+
+
 def lightly_classes_map(categories: list[dict[str, Any]]) -> dict[str, str]:
-    """Build the Lightly ``classes`` mapping (index → name), 0 = background.
-    Category ids start at 1 and are contiguous (see build_export_plan)."""
-    out: dict[str, str] = {"0": "background"}
-    for c in categories:
-        out[str(int(c["id"]))] = str(c["name"])
-    return out
+    """Build the Lightly/DINOv3 ``classes`` mapping (index → name), 0-indexed and
+    contiguous with NO background class. Internal category ids are 1-based (see
+    build_export_plan), so shift by -1; unannotated pixels use 255 (ignore) instead
+    of a background class."""
+    return {str(int(c["id"]) - 1): str(c["name"]) for c in categories}
 
 
 def build_export_plan(
@@ -451,6 +465,7 @@ def build_export_plan(
     sample_global_stats_fn: Any,
     progress_cb: Any = None,
     include_polygons: bool = False,
+    lightly: bool = False,
 ) -> dict[str, Any]:
     """Build the full export plan (rasterize all shapes, render PNGs).
 
@@ -516,9 +531,10 @@ def _process_slice(slice_key: str) -> dict[str, Any]:
         file_name = f"{source_stem}_{slice_idx:04d}.png"
         anns: list[dict[str, Any]] = []
         skipped = 0
-        # Semantic label map (class index per pixel, 0 = bg) + per-class binary
-        # masks, built from the same rasterization used for the COCO annotations.
-        label = np.zeros((h, w), dtype=np.uint8)
+        # Semantic label map + per-class binary masks, from the same rasterization
+        # used for the COCO annotations. COCO: 0 = background, classes 1..N. Lightly/
+        # DINOv3: unannotated = 255 (ignore), classes 0-indexed (cat_id - 1).
+        label = np.full((h, w), 255, dtype=np.uint8) if lightly else np.zeros((h, w), dtype=np.uint8)
         class_acc: dict[str, np.ndarray] = {}
         for shape in payload.slices.get(slice_key, []):
             shape_dict = shape if isinstance(shape, dict) else shape.model_dump()
@@ -541,7 +557,8 @@ def _process_slice(slice_key: str) -> dict[str, Any]:
             ann["_image_file_name"] = file_name
             anns.append(ann)
             # Paint label map (last shape wins on overlap) + accumulate per class.
-            label[mask] = cat_id
+            # Lightly is 0-indexed (cat_id - 1); COCO keeps the 1-based id.
+            label[mask] = (cat_id - 1) if lightly else cat_id
             cname = cat_id_to_name.get(cat_id, str(cat_id))
             if cname not in class_acc:
                 class_acc[cname] = np.zeros((h, w), dtype=bool)