diff --git a/docker/README.md b/docker/README.md index b4d170e63e..65f209711a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -181,6 +181,9 @@ and batch latency/throughput, checks the expected dimensions, and reports fixture retrieval metrics. Replace docker/benchmarks/qwen3_smoke_dataset.json with a representative corpus and labeled queries before using quality scores to select a production model. +When `--mode both` is used, the JSON report also contains one comparison per +model with GPU speedups and GPU-minus-CPU retrieval-quality deltas. This makes +the combined profile suitable for choosing a deployment target from one run. This change intentionally retains all six variants; unselected services can be removed in a follow-up after the benchmark review. diff --git a/docker/benchmarks/embedding_benchmark.py b/docker/benchmarks/embedding_benchmark.py index ac0a9d228d..b6efd0b89d 100644 --- a/docker/benchmarks/embedding_benchmark.py +++ b/docker/benchmarks/embedding_benchmark.py @@ -217,6 +217,84 @@ def validate_vectors(vectors: list[list[float]], expected_dimension: int) -> Non ) +def nested_float(result: dict[str, Any], *keys: str) -> float | None: + """Read a numeric value from a nested benchmark result.""" + value: Any = result + for key in keys: + if not isinstance(value, dict) or key not in value: + return None + value = value[key] + return float(value) if isinstance(value, (int, float)) else None + + +def rounded_ratio(numerator: float | None, denominator: float | None) -> float | None: + """Return a rounded ratio, or ``None`` when the denominator is unavailable.""" + if numerator is None or denominator is None or denominator == 0: + return None + return round(numerator / denominator, 3) + + +def compare_modes(results: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Compare CPU and GPU results for each model in a completed matrix.""" + by_model: dict[str, dict[str, dict[str, Any]]] = {} + for result in results: + if result.get("status", "ok") != "ok": + continue + model = str(result["model"]) + mode = str(result["mode"]) + if mode in MODES: + by_model.setdefault(model, {})[mode] = result + + comparisons: list[dict[str, Any]] = [] + for model in sorted(by_model): + pair = by_model[model] + if not all(mode in pair for mode in MODES): + continue + cpu = pair["cpu"] + gpu = pair["gpu"] + comparison: dict[str, Any] = { + "model": model, + "cpu_service": cpu["service"], + "gpu_service": gpu["service"], + "single_latency_speedup": rounded_ratio( + nested_float(cpu, "single_latency_ms", "p50"), + nested_float(gpu, "single_latency_ms", "p50"), + ), + "batch_throughput_speedup": rounded_ratio( + nested_float(gpu, "batch_throughput_items_per_second"), + nested_float(cpu, "batch_throughput_items_per_second"), + ), + } + cpu_mrr = nested_float(cpu, "quality", "mrr") + gpu_mrr = nested_float(gpu, "quality", "mrr") + cpu_ndcg = nested_float(cpu, "quality", "ndcg_at_5") + gpu_ndcg = nested_float(gpu, "quality", "ndcg_at_5") + if cpu_mrr is not None and gpu_mrr is not None: + comparison["mrr_delta"] = round(gpu_mrr - cpu_mrr, 6) + if cpu_ndcg is not None and gpu_ndcg is not None: + comparison["ndcg_at_5_delta"] = round(gpu_ndcg - cpu_ndcg, 6) + comparisons.append(comparison) + return comparisons + + +def print_comparisons(comparisons: list[dict[str, Any]]) -> None: + """Print CPU/GPU comparison rows for a completed matrix.""" + if not comparisons: + return + print("\nCPU/GPU comparison") + print( + "model single_latency_speedup batch_throughput_speedup mrr_delta ndcg_at_5_delta" + ) + for comparison in comparisons: + print( + f"{comparison['model']} " + f"{comparison['single_latency_speedup']} " + f"{comparison['batch_throughput_speedup']} " + f"{comparison.get('mrr_delta', '-')} " + f"{comparison.get('ndcg_at_5_delta', '-')}" + ) + + def benchmark_endpoint( model: dict[str, Any], mode: str, @@ -408,6 +486,8 @@ def main() -> int: }, "results": results, } + comparisons = compare_modes(results) if args.mode == "both" else [] + report["comparisons"] = comparisons if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") @@ -425,6 +505,7 @@ def main() -> int: f"{result.get('batch_throughput_items_per_second', '-')} " f"{quality.get('mrr', '-')}" ) + print_comparisons(comparisons) has_errors = any(result.get("status") == "error" for result in results) return 1 if args.strict and has_errors else 0 diff --git a/docker/benchmarks/test_embedding_benchmark.py b/docker/benchmarks/test_embedding_benchmark.py new file mode 100644 index 0000000000..da830b3c41 --- /dev/null +++ b/docker/benchmarks/test_embedding_benchmark.py @@ -0,0 +1,64 @@ +"""Tests for CPU/GPU comparison reporting.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from embedding_benchmark import compare_modes # noqa: E402 + + +def result( + model: str, + mode: str, + single_p50: float, + throughput: float, + mrr: float, + ndcg_at_5: float, + status: str = "ok", +) -> dict[str, object]: + """Build the subset of a benchmark result used by comparisons.""" + return { + "model": model, + "mode": mode, + "service": f"{model}-{mode}", + "status": status, + "single_latency_ms": {"p50": single_p50}, + "batch_throughput_items_per_second": throughput, + "quality": {"mrr": mrr, "ndcg_at_5": ndcg_at_5}, + } + + +def test_compare_modes_reports_speedup_and_quality_delta() -> None: + """GPU speedups and quality deltas are calculated in the expected direction.""" + comparisons = compare_modes( + [ + result("qwen3-embedding-06b", "cpu", 300, 2, 0.70, 0.75), + result("qwen3-embedding-06b", "gpu", 30, 20, 0.80, 0.85), + ] + ) + + assert comparisons == [ + { + "model": "qwen3-embedding-06b", + "cpu_service": "qwen3-embedding-06b-cpu", + "gpu_service": "qwen3-embedding-06b-gpu", + "single_latency_speedup": 10.0, + "batch_throughput_speedup": 10.0, + "mrr_delta": 0.1, + "ndcg_at_5_delta": 0.1, + } + ] + + +def test_compare_modes_skips_incomplete_or_failed_pairs() -> None: + """A failed or one-sided pair is not presented as a comparison.""" + comparisons = compare_modes( + [ + result("failed", "cpu", 300, 2, 0.70, 0.75, status="error"), + result("failed", "gpu", 30, 20, 0.80, 0.85), + result("incomplete", "cpu", 300, 2, 0.70, 0.75), + ] + ) + + assert comparisons == []