From a63336a8f5c9318e3cf7084199b3c55ebebcdcce Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Thu, 3 Sep 2026 15:29:12 +0200 Subject: [PATCH 1/2] perf: batch the contiguous-UPDATE hash re-points (one lock per index); lock scan/count-after-DELETE regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UpdateMultiple's contiguous fast path previously did a per-row hashIndex.Remove + Add for each fixed-size indexed SET column (2 lock acquisitions per row per index). It now collects old/new keys for every touched index and applies them with HashIndex.RemoveBatchKeys + new AddBatchKeys (one lock per index per batch). Adds count/scan regression tests for delete paths: BulkDelete_ScanAndCountStayConsistent (contiguous bulk delete: SELECT returns exactly the 1000 live rows > id 1000 and COUNT(*) matches) and GappedDeletes now also asserts the live scan is exactly the 400 even rows — locking in the resolution of the earlier COUNT-after-DELETE inconsistency (root cause was the BTree.Delete separator corruption fixed in #361). --- src/SharpCoreDB/DataStructures/HashIndex.cs | 96 +++++++++++++++++++ src/SharpCoreDB/DataStructures/Table.CRUD.cs | 63 ++++++++---- .../FixedWidthBulkDeleteTests.cs | 37 +++++++ 3 files changed, 175 insertions(+), 21 deletions(-) diff --git a/src/SharpCoreDB/DataStructures/HashIndex.cs b/src/SharpCoreDB/DataStructures/HashIndex.cs index fd240397..2827bd14 100644 --- a/src/SharpCoreDB/DataStructures/HashIndex.cs +++ b/src/SharpCoreDB/DataStructures/HashIndex.cs @@ -326,6 +326,102 @@ internal void RemoveBatchKeys(object?[] keys, long[] positions) } } + /// + /// Key-based overload of : callers that already know each indexed key + /// (e.g. an in-place UPDATE re-point) add all rows with one lock acquisition per index. + /// + /// The indexed key values (null entries are skipped). + /// Corresponding storage positions. + internal void AddBatchKeys(object?[] keys, long[] positions) + { + if (keys.Length == 0) + { + return; + } + + // PERF: Non-unique unsafe path — batch keys, then a single UnsafeEqualityIndex acquisition. + if (_useUnsafeEqualityIndex && !_isUnique) + { + var keyArrays = ArrayPool.Shared.Rent(keys.Length); + var rowIdBuf = ArrayPool.Shared.Rent(keys.Length); + int validCount = 0; + + try + { + for (int i = 0; i < keys.Length; i++) + { + if (keys[i] is null) + { + continue; + } + + keyArrays[validCount] = BuildUnsafeKey(NormalizeKey(keys[i])); + rowIdBuf[validCount] = positions[i]; + validCount++; + } + + if (validCount > 0) + { + _unsafeIndex.AddBatch(keyArrays, rowIdBuf, validCount); + Interlocked.Add(ref _unsafeTotalRows, validCount); + } + } + finally + { + ArrayPool.Shared.Return(keyArrays, clearArray: true); + ArrayPool.Shared.Return(rowIdBuf); + } + + return; + } + + _lock.EnterWriteLock(); + try + { + for (int i = 0; i < keys.Length; i++) + { + if (keys[i] is null) + { + continue; + } + + var normalizedKey = NormalizeKey(keys[i]); + + if (_useUnsafeEqualityIndex) + { + // Unique path: atomic check + add under outer lock. + var keyBytes = BuildUnsafeKey(normalizedKey); + if (HasUnsafeRowsForKey(keyBytes)) + { + throw new InvalidOperationException( + $"Duplicate key value '{keys[i]}' violates unique constraint on index '{_columnName}'"); + } + + _unsafeIndex.Add(keyBytes, positions[i]); + _unsafeTotalRows++; + continue; + } + + if (!_index.TryGetValue(normalizedKey, out var list)) + { + list = []; + _index[normalizedKey] = list; + } + else if (_isUnique && list.Count > 0) + { + throw new InvalidOperationException( + $"Duplicate key value '{keys[i]}' violates unique constraint on index '{_columnName}'"); + } + + list.Add(positions[i]); + } + } + finally + { + _lock.ExitWriteLock(); + } + } + /// /// Adds multiple rows to the index in a single lock acquisition. /// ✅ PERF: Acquires write lock once for entire batch instead of per-row. diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index 71b02178..9a8f37d0 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -2470,35 +2470,56 @@ this.storage is null || { return false; } + } - // Re-point hash-index entries for every fixed-size indexed SET column (mirrors the - // generic fastPatch path: old value decoded from the pre-write record, new value at the - // same position). + // Re-point hash-index entries for every fixed-size indexed SET column with ONE lock + // acquisition per index (mirrors the generic fastPatch path: old value decoded from the + // pre-write record, new value at the same position). Per-row Remove/Add would take 2 locks + // per row per index on the batch-DML hot path. + var indexedColumns = new List(); + for (int i = 0; i < count; i++) + { var repoints = repointColumns[i]; - if (repoints is { Count: > 0 }) + if (repoints is null) + { + continue; + } + + foreach (var colIdx in repoints) { - foreach (var colIdx in repoints) + if (!indexedColumns.Contains(colIdx)) { - var colName = this.Columns[colIdx]; - if (!this.hashIndexes.TryGetValue(colName, out var hashIdx)) - { - continue; - } + indexedColumns.Add(colIdx); + } + } + } - var slot = payload.AsSpan(layout.Offsets[colIdx], layout.SlotSizes[colIdx]); - var oldVal = ReadTypedValueFromSpan(slot, this.ColumnTypes[colIdx], out _); - if (oldVal is not null) - { - hashIdx.Remove(oldVal, positions[i]); - } + foreach (var colIdx in indexedColumns) + { + var colName = this.Columns[colIdx]; + if (!this.hashIndexes.TryGetValue(colName, out var hashIdx)) + { + continue; + } - var newVal = operations[i].updates[colName]; - if (newVal is not null) - { - hashIdx.Add(newVal, positions[i]); - } + var type = this.ColumnTypes[colIdx]; + var oldKeys = new object?[count]; + var newKeys = new object?[count]; + for (int i = 0; i < count; i++) + { + var repoints = repointColumns[i]; + if (repoints is null || !repoints.Contains(colIdx)) + { + continue; } + + var slot = raw.AsSpan((int)(i * stride) + 4 + layout.Offsets[colIdx], layout.SlotSizes[colIdx]); + oldKeys[i] = ReadTypedValueFromSpan(slot, type, out _); + newKeys[i] = operations[i].updates[colName]; } + + hashIdx.RemoveBatchKeys(oldKeys, positions); + hashIdx.AddBatchKeys(newKeys, positions); } Interlocked.Increment(ref _bulkContiguousUpdateBatches); diff --git a/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs b/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs index 9bdff64b..89550df3 100644 --- a/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs +++ b/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs @@ -137,6 +137,38 @@ public void DescendingUpdateBatch_FallsBackToGenericLoop_AndAppliesEveryRow() finally { (db as IDisposable)?.Dispose(); } } + [Fact] + public void BulkDelete_ScanAndCountStayConsistent() + { + var db = CreateDb(); + try + { + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + InsertDocs(db, 1, 2000); + db.Flush(); + + var table = TableOf(db, "docs"); + var stmts = new List(1000); + for (int i = 1; i <= 1000; i++) + { + stmts.Add(string.Format(CultureInfo.InvariantCulture, "DELETE FROM docs WHERE id = {0}", i)); + } + + db.ExecuteBatchSQL(stmts); + db.Flush(); + Assert.Equal(1, table.BulkContiguousDeleteBatches); + + // After logical deletes the full scan and COUNT(*) must reflect exactly the live rows + // (regression for the earlier inconsistency, resolved by the BTree separator-delete fix). + var scan = db.ExecuteQuery("SELECT id FROM docs"); + Assert.Equal(1000, scan.Count); + Assert.All(scan, r => Assert.True(Convert.ToInt64(r["id"]) > 1000)); + var c = db.ExecuteQuery("SELECT COUNT(*) AS n FROM docs"); + Assert.Equal(1000L, Convert.ToInt64(c[0].Values.First())); + } + finally { (db as IDisposable)?.Dispose(); } + } + [Fact] public void GappedDeletes_FallBackToGenericLoop_AndStayCorrect() { @@ -162,6 +194,11 @@ public void GappedDeletes_FallBackToGenericLoop_AndStayCorrect() Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 2")); Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE name = 'user3'")); Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE name = 'user2'")); + + // Generic-path deletes keep the scan/count live too. + var scan = db.ExecuteQuery("SELECT id FROM docs"); + Assert.Equal(400, scan.Count); + Assert.All(scan, r => Assert.Equal(0L, Convert.ToInt64(r["id"]) % 2)); } finally { (db as IDisposable)?.Dispose(); } } From ea4d6f23487c56d56cde35fcb8c1556e489f29a8 Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Thu, 3 Sep 2026 15:30:40 +0200 Subject: [PATCH 2/2] docs: record the fixed-width/contiguous-DML benchmark results in CHANGELOG + docs/benchmarks Unreleased performance entries for AutoFixedWidthRecords, single-pass contiguous UPDATE/DELETE (B8/B9), the BTree separator-delete fix and the batched hash re-points, plus a fair-PK benchmark sheet with the machine-noise caveat. --- docs/CHANGELOG.md | 24 ++++++++++++++ .../PK_FAIR_FIXEDWIDTH_2026-09-03.md | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 docs/benchmarks/PK_FAIR_FIXEDWIDTH_2026-09-03.md diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cbf31f6e..2159e681 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- **Fixed-width record layout is now the default for new columnar PK tables (B7)** ÔÇö + `DatabaseConfig.AutoFixedWidthRecords` (default `true`) creates new directory-mode Columnar tables + with an explicitly declared PRIMARY KEY in the fixed-width layout (constant record size, + out-of-line overflow arena, in-place UPDATE/DELETE). Tables without a declared PK, PageBased and + single-file (.scdb) layouts are unchanged; existing tables are never rewritten (the persisted + per-table flag stays authoritative). Fair-PK harness (`--pk`, AppendOnly): INSERT ~128K, + UPDATE ~51K, DELETE ~75K vs 87K/41K/65K on the legacy layout. +- **Single-pass contiguous UPDATE / DELETE (B8/B9)** ÔÇö batch `UPDATE/DELETE ... WHERE pk = literal` + statements with strictly ascending keys on a **plaintext** fixed-width columnar table now read the + target records as **one contiguous byte range** (storage cached handle) and patch/remove them in + memory: no per-row pread. UPDATE ~45K ÔåÆ **~84K ops/s** (gap vs SQLite 5.1├ù ÔåÆ **3.2├ù**); DELETE + ~55-77K ÔåÆ **~135K ops/s** (gap ~4-6├ù ÔåÆ **2.1├ù**). Any shape deviation (gaps, descending keys, + PK writes, CHECK constraints, variable-length indexed columns, per-record encryption) falls back + to the generic per-row loop before any row is touched. New primitives: + `IStorage.HasBufferedOverwrite`, `IStorage.ReadBytesRange` (shared-handle range read) and the + runtime `AreRecordsEncrypted` gate (so default-config plaintext databases benefit without + `NoEncryptMode`). Fixed-size hash-indexed SET columns are re-pointed with one lock per index + (`HashIndex.RemoveBatchKeys`/`AddBatchKeys`). +- **BTree separator-delete corruption fixed (correctness)** ÔÇö `BTree.Delete` removed separator keys + from internal nodes without repairing the child-pointer mapping, so sizable delete batches could + leave whole key ranges unreachable (and scans/COUNT(*) under-counted). Internal separators are + now replaced by their in-order successor from the right subtree's leftmost leaf (leaf underflow is + harmless), with empty-neighbour fallbacks keeping child counts consistent. Full suite + **1655/1655**. - **Dedicated SQL batch-INSERT fast path (WP14)** ÔÇö `ExecuteBatchSQL` INSERTs no longer build a per-row `Dictionary`; VALUES clauses are parsed directly into column-ordered `object[]` rows (`PreparedInsertStatement.ParseValuesToArray`) and inserted via the new diff --git a/docs/benchmarks/PK_FAIR_FIXEDWIDTH_2026-09-03.md b/docs/benchmarks/PK_FAIR_FIXEDWIDTH_2026-09-03.md new file mode 100644 index 00000000..2825df49 --- /dev/null +++ b/docs/benchmarks/PK_FAIR_FIXEDWIDTH_2026-09-03.md @@ -0,0 +1,32 @@ +# Fair PK harness — fixed-width default & contiguous UPDATE/DELETE (2026-09-03) + +Machine: Windows, SDK 11 preview (builds net10.0). AppendOnly engine, `--pk` scenario: +100k inserts / 10k point reads / 10k PK updates / 10k PK deletes on +`docs(id INTEGER PRIMARY KEY, name TEXT, email TEXT, age INTEGER, score REAL, data TEXT)` +with `CREATE INDEX idx_docs_name ON docs(name)`; SQLite run on the identical schema. +Values are single-run samples from this machine and vary with load (one run under load dropped +SQLite itself from ~230-270K UPDATE to ~103K); use them as ranges, not exact deltas. + +## Legacy vs fixed-width vs SQLite (AppendOnly) + +| DB | INSERT | READ | UPDATE | DELETE | +|---|---:|---:|---:|---:| +| SharpCoreDB legacy variable-length (opt-out) | 75-96K | 55-63K | 39-63K | 54-94K | +| SharpCoreDB default (fixed-width) before contiguous paths | 128K | 74K | 51K | 75K | +| SharpCoreDB default (fixed-width) after B8/B9 contiguous UPDATE/DELETE | 134-140K | 70-83K | 79-84K | 135K | +| SQLite | 119-124K | 79-93K | 230-270K | 279-353K | + +Gap vs SQLite (representative, default fixed-width): INSERT ~1.0x, READ ~1.2x, UPDATE ~2.9-3.2x, +DELETE ~2.1x. + +## What changed to get there + +- #359 fixed-width default for new columnar PK tables (`AutoFixedWidthRecords`), incl. fixing the + SQL batch-INSERT fast path to serialize fixed-width records. +- #360 single-pass contiguous UPDATE (one contiguous range read + in-memory patch). +- #361 single-pass contiguous DELETE + BTree separator-delete corruption fix. +- #363 gate widened to default-config plaintext databases (runtime `AreRecordsEncrypted`). +- #364 batched hash-index re-points (one lock per index per batch). + +Raw per-run JSON results land in `results/pk_comparative_*.json` when running +`SharpCoreDB.Benchmarks.Comparative --pk`.