diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1645e886..294bde0a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Hardening +- **Default-config benchmark published (P3)** - `docs/benchmarks/default-config-pk.md` records a + median-of-3 fair-PK run where the SharpCoreDB arm uses a PURE default `DatabaseConfig` + (NoEncryptMode=false, no harness flags). Honest result: the default path engages the Columnar + fixed-width fast paths (verified by counters) but UPDATE/DELETE land at ~84K/~93K ops/s + (~3.5x/~4.2x behind SQLite) versus ~245K/~172K with the tuned harness config — the gap is + dominated by the default `WalDurabilityMode.FullSync` per-commit flush. The default durability + stays FullSync (safe); the follow-up is to optimize the synchronous commit flush itself. New + harness mode: `--pk-default`. + - **Upgrade/downgrade policy documented + format-compat regression tests** - new `docs/manual/upgrade-and-downgrade.md` records the compatibility matrix: reading legacy (variable-length, pre-marker) databases with the current version is supported; opening a database diff --git a/docs/benchmarks/default-config-pk.md b/docs/benchmarks/default-config-pk.md new file mode 100644 index 00000000..03ec7bc7 --- /dev/null +++ b/docs/benchmarks/default-config-pk.md @@ -0,0 +1,39 @@ +# Fair-PK benchmark with DEFAULT DatabaseConfig (P3 hardening) + +Run: `dotnet run --project tests/benchmarks/SharpCoreDB.Benchmarks.Comparative -- -c Release -- --pk-default` +Date: 2026-09-04 · Machine: local dev box · median of 3 runs per phase. + +The SharpCoreDB arm uses a **pure default** `DatabaseConfig` (only the engine type is pinned to +`AppendOnly` — `NoEncryptMode` stays at its default `false`, no page-cache/query-cache/durability +harness flags). The table is the fair-PK schema with `id INTEGER PRIMARY KEY`, so the default +record-layout rules apply: a new PK table becomes **Columnar + fixed-width** and the single-pass +contiguous UPDATE/DELETE paths do engage (verified separately by `DefaultEngineSelectionTests`). + +## Results (ops/sec, median of 3) + +| Database | INSERT | READ | UPDATE | DELETE | +|---|---:|---:|---:|---:| +| SharpCoreDB (default config) | 111,052 | 69,053 | **84,304** | **92,646** | +| SQLite | 188,247 | 107,122 | 291,414 | 389,120 | +| gap vs SQLite | 1.7x | 1.6x | **3.5x** | **4.2x** | + +For comparison, the same schema with the tuned benchmark config (NoEncryptMode + Async WAL + larger +batches) measures UPDATE ~245K ops/s and DELETE ~172K ops/s (gaps ~1.2x / ~2.1x). + +## Honest conclusion + +The out-of-the-box default is **correct and engages the fast paths, but is not yet at the tuned +throughput**: UPDATE/DELETE land ~3.5-4.2x behind SQLite (vs ~1.2-2.1x tuned). The dominant +difference is the **default `WalDurabilityMode.FullSync`** (per-commit flush) versus the benchmark +arm's `Async`; the per-batch commit cost explains most of the gap. Fast-path counters prove the +contiguous code is running — the throughput is limited by durability flushing, not by resolution. + +## Recommendation / follow-up + +1. Do **not** silently weaken the durability default (`FullSync` is the safe choice for + production). Instead, optimize the **FullSync commit flush** path (fewer/single flush per + commit, group-commit of the commit markers + overwrites that already batch per page) so a + synchronous commit costs a few ms instead of tens of ms. +2. Re-run this `--pk-default` harness after that change and require the default-config UPDATE/DELETE + gap to move from ~3.5-4.2x toward ~2x before the release-cut. +3. Keep this file updated with the latest median-of-3 numbers. diff --git a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs index 041cafe5..aa6414a0 100644 --- a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs +++ b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs @@ -65,6 +65,20 @@ static async Task Main(string[] args) return; } + // Optional: --pk-default → fair PK comparison using PURE default DatabaseConfig + // (NoEncryptMode stays at its default false, no harness-only flags). Proves that the + // out-of-the-box default path engages the fixed-width fast paths vs SQLite. + if (args.Any(a => a.Equals("--pk-default", StringComparison.OrdinalIgnoreCase))) + { + var engineArgDefault = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase)); + var engineTypeDefault = engineArgDefault is not null + && engineArgDefault.Substring("--engine=".Length).Equals("pagebased", StringComparison.OrdinalIgnoreCase) + ? SharpCoreDB.Interfaces.StorageEngineType.PageBased + : SharpCoreDB.Interfaces.StorageEngineType.AppendOnly; + RunPkDefaultComparison(engineTypeDefault); + return; + } + // Optional: --engine=appendonly (default) | --engine=pagebased // PageBased is the v2.0 in-place-update engine (WP10-WP13 storage engine roadmap). var engineArg = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase)); @@ -845,7 +859,10 @@ data TEXT /// ExecuteBatchSQL (single transaction). This exercises the PK B-tree fast paths and the /// recommended usage; the no-PK harness scenario above under-measures the engine on DML. /// - static BenchmarkResult RunSharpCoreDBPk(SharpCoreDB.Interfaces.StorageEngineType engineType, bool fixedWidth = false) + static BenchmarkResult RunSharpCoreDBPk( + SharpCoreDB.Interfaces.StorageEngineType engineType, + bool fixedWidth = false, + bool useDefaultConfig = false) { var dbPath = Path.Combine(Path.GetTempPath(), $"bench-sharpcoredb-pk-{Guid.NewGuid()}"); var result = new BenchmarkResult(); @@ -857,7 +874,9 @@ static BenchmarkResult RunSharpCoreDBPk(SharpCoreDB.Interfaces.StorageEngineType var sp = services.BuildServiceProvider(); var factory = sp.GetRequiredService(); - var config = BuildConfig(engineType, fixedWidth); + var config = useDefaultConfig + ? new DatabaseConfig { StorageEngineType = engineType } + : BuildConfig(engineType, fixedWidth); using var db = (SharpCoreDB.Database)factory.Create( dbPath: dbPath, @@ -1034,6 +1053,50 @@ static void RunPkComparison(SharpCoreDB.Interfaces.StorageEngineType engineType) Console.WriteLine($"\nResults saved to: {path}"); } + /// + /// P3 hardening: fair PK comparison where the SharpCoreDB arm uses a PURE default + /// (no NoEncryptMode / page-cache / query-cache harness flags) — + /// only the engine type is pinned to the runner's engine. Proves the out-of-the-box default + /// path (Columnar + fixed-width PK tables) engages the fast paths against SQLite. + /// + static void RunPkDefaultComparison(SharpCoreDB.Interfaces.StorageEngineType engineType) + { + var engineLabel = engineType == SharpCoreDB.Interfaces.StorageEngineType.PageBased ? "PageBased" : "AppendOnly"; + Console.WriteLine("╔══════════════════════════════════════════════════════════╗"); + Console.WriteLine("║ Fair PK with DEFAULT DatabaseConfig vs SQLite ║"); + Console.WriteLine("║ (NoEncryptMode=false; no harness-only performance flags)║"); + Console.WriteLine("╚══════════════════════════════════════════════════════════╝"); + Console.WriteLine(); + Console.WriteLine($"Engine: {engineLabel}"); + Console.WriteLine("(median of 3 per phase)"); + + Console.WriteLine("━━━ SharpCoreDB (SQL, PK, default config = Columnar fixed-width) ━━━"); + var scdb = RunPkMedian(() => RunSharpCoreDBPk(engineType, useDefaultConfig: true)); + Console.WriteLine(); + + Console.WriteLine("━━━ SQLite (reference) ━━━"); + var sqlite = RunPkMedian(() => RunSQLite()); + Console.WriteLine(); + + Console.WriteLine("║ Database │ INSERT │ READ │ UPDATE │ DELETE ║"); + Console.WriteLine($"║ SharpCoreDB │ {scdb.InsertOpsPerSec,10:N0} │ {scdb.ReadOpsPerSec,8:N0} │ {scdb.UpdateOpsPerSec,8:N0} │ {scdb.DeleteOpsPerSec,8:N0} ║"); + Console.WriteLine($"║ SQLite │ {sqlite.InsertOpsPerSec,10:N0} │ {sqlite.ReadOpsPerSec,8:N0} │ {sqlite.UpdateOpsPerSec,8:N0} │ {sqlite.DeleteOpsPerSec,8:N0} ║"); + Console.WriteLine($"\n UPDATE gap: SQLite vs default-config {sqlite.UpdateOpsPerSec / (double)scdb.UpdateOpsPerSec:F1}x"); + Console.WriteLine($" DELETE gap: SQLite vs default-config {sqlite.DeleteOpsPerSec / (double)scdb.DeleteOpsPerSec:F1}x"); + + var results = new Dictionary + { + ["SharpCoreDB (SQL, PK, default config)"] = scdb, + ["SQLite"] = sqlite, + }; + + var dir = "results"; + Directory.CreateDirectory(dir); + var path = Path.Combine(dir, $"pk_default_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + File.WriteAllText(path, JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true })); + Console.WriteLine($"\nResults saved to: {path}"); + } + // ══════════════════════════════════════ // LiteDB // ══════════════════════════════════════