From d234e1ad51b4ca7ba0c3d959f3245b711086849f Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Thu, 3 Sep 2026 16:56:52 +0200 Subject: [PATCH 1/2] perf: make durable-delete flush compaction proportional (data-only + single-pass index rebuild) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flush-time compaction added for durable Columnar deletes (#365) cost ~46s on a 100K-row table: CompactStorage compacted the overflow arena (46s) and rebuilt PK + each hash index with a separate full-file decode pass. CompactPendingDeletes now rewrites only the data file (CompactTable over the live PK positions) and rebuilds all indexes in ONE decode pass (RebuildAllIndexesFromFile) — the arena is left for a later explicit VACUUM/compact. Measured flush for delete-10k-of-100k: ~46s -> ~0.4s. CompactStorage (explicit compaction) also reuses the traversal + single-pass rebuild. --- .../DataStructures/Table.Compaction.cs | 132 ++++++++++++++---- 1 file changed, 104 insertions(+), 28 deletions(-) diff --git a/src/SharpCoreDB/DataStructures/Table.Compaction.cs b/src/SharpCoreDB/DataStructures/Table.Compaction.cs index c77eddb8..e19f4ae2 100644 --- a/src/SharpCoreDB/DataStructures/Table.Compaction.cs +++ b/src/SharpCoreDB/DataStructures/Table.Compaction.cs @@ -55,7 +55,9 @@ public void TryAutoCompact() /// Physically removes rows that were logically deleted since the last flush (Columnar tables /// with a primary key) so DELETE survives a reopen — the on-load PK-index rebuild would otherwise /// resurrect them from the untouched .dat. Runs synchronously at flush/dispose, outside a - /// transaction, when any logical deletes are pending. + /// transaction, when any logical deletes are pending. Data-file only: the overflow arena is left + /// untouched (its space is reclaimed by a later explicit compaction/VACUUM) so the flush stays + /// proportional to the rewritten live rows. /// public void CompactPendingDeletes() { @@ -71,13 +73,35 @@ public void CompactPendingDeletes() return; // defer until the transaction commits (the next flush will run again) } + rwLock.EnterWriteLock(); try { - CompactStorage(); + var engine = GetOrCreateStorageEngine(); + if (engine is not AppendOnlyEngine appendEngine) + { + return; + } + + var activePositions = new List(); + if (this.Index is BTree pkTree) + { + foreach (var (_, position) in pkTree.InOrderTraversal()) + { + activePositions.Add(position); + } + } + else + { + return; // no enumerable PK tree — cannot rewrite safely; keep logical deletes + } + + appendEngine.CompactTable(Name, activePositions); + RebuildAllIndexesFromFile(); } finally { Interlocked.Exchange(ref _pendingLogicalDeletes, 0); + rwLock.ExitWriteLock(); } } @@ -108,24 +132,30 @@ public CompactionStats CompactStorage() if (PrimaryKeyIndex >= 0) { - // Collect all positions from primary key index. - // ✅ FIX (1.9.5): Include the hidden _rowid column when present — otherwise rows in - // tables with an internal ULID primary key cannot be resolved to storage positions - // and compaction would drop every row. - var pkColumn = Columns[PrimaryKeyIndex]; - var allRows = HasInternalRowId - ? SelectIncludingRowId(where: null, orderBy: null, asc: true, noEncrypt: false) - : Select(); - - foreach (var row in allRows) + if (this.Index is BTree pkTree) + { + // Collect the live (key → position) pairs straight from the PK B-tree — no row + // materialization and no per-row re-search. Covers the hidden _rowid PK too, + // because every live row has an entry in the tree. + foreach (var (_, position) in pkTree.InOrderTraversal()) + { + activePositions.Add(position); + } + } + else { - if (row.TryGetValue(pkColumn, out var pkValue) && pkValue != null) + // Non-BTree index fallback: resolve positions through the current rows. + var pkColumn = Columns[PrimaryKeyIndex]; + var allRows = SelectIncludingRowId(where: null, orderBy: null, asc: true, noEncrypt: false); + foreach (var row in allRows) { - var pkStr = pkValue.ToString() ?? string.Empty; - var searchResult = Index.Search(pkStr); - if (searchResult.Found) + if (row.TryGetValue(pkColumn, out var pkValue) && pkValue != null) { - activePositions.Add(searchResult.Value); + var searchResult = Index.Search(pkValue.ToString() ?? string.Empty); + if (searchResult.Found) + { + activePositions.Add(searchResult.Value); + } } } } @@ -150,17 +180,13 @@ public CompactionStats CompactStorage() // Reset counters Interlocked.Exchange(ref _deletedRowCount, 0); Interlocked.Exchange(ref _updatedRowCount, 0); - - // Rebuild primary key index with new positions - // Note: After compaction, positions change! We need to rebuild the index. - RebuildPrimaryKeyIndex(); - - // Rebuild hash indexes - foreach (var col in loadedIndexes.ToList()) - { - RebuildHashIndex(col); - } - + Interlocked.Exchange(ref _pendingLogicalDeletes, 0); + + // Rebuild the PK B-tree and every loaded hash index in ONE file pass (positions change + // after compaction; a per-index rescan would re-read + re-decode the whole file once + // per index, which is pathological on large tables). + RebuildAllIndexesFromFile(); + return new CompactionStats { BytesReclaimed = bytesReclaimed, @@ -235,6 +261,56 @@ private static void CollectVariableOffsets(byte[] record, FixedWidthRecordLayout private static byte[]? RepointVariableSlots(byte[] record, FixedWidthRecordLayout layout, Dictionary mapping) => FixedWidthCodec.RepointVariableSlots(record, layout, mapping); + /// + /// Rebuilds the PK B-tree and every loaded hash index from the rewritten data file in ONE pass: + /// each record is read and decoded once and feeds all indexes, instead of rescanning the whole + /// file (with full deserialization, including overflow-arena reads) once per index. + /// + private void RebuildAllIndexesFromFile() + { + var engine = GetOrCreateStorageEngine(); + + if (PrimaryKeyIndex >= 0) + { + Index = new BTree(); + } + + foreach (var hashIndex in hashIndexes.Values) + { + hashIndex.Clear(); + } + + var loadedHashIndexes = new List(); + foreach (var kvp in hashIndexes) + { + if (loadedIndexes.Contains(kvp.Key)) + { + loadedHashIndexes.Add(kvp.Value); + } + } + + foreach (var (position, data) in engine.GetAllRecords(Name)) + { + var row = DeserializeRow(data); + if (row is null) + { + continue; + } + + if (PrimaryKeyIndex >= 0 && + row.TryGetValue(Columns[PrimaryKeyIndex], out var pkValue) && + pkValue != null) + { + Index.Insert(pkValue.ToString() ?? string.Empty, position); + } + + foreach (var hashIndex in loadedHashIndexes) + { + hashIndex.Add(row, position); + } + } + } + /// /// Rebuilds the primary key index after compaction. /// Positions change after compaction, so we need to rescan the file. From 8e0110a821bc85e8c05c236eead7795115f4bc4a Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Thu, 3 Sep 2026 17:02:18 +0200 Subject: [PATCH 2/2] docs: note the data-only durable-delete flush compaction in CHANGELOG --- docs/CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 257c1318..6b3ff8e9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -36,8 +36,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **DELETE now survives a reopen (durability)** ÔÇö Columnar deletes were logical only (index removal), so the on-load PK-index rebuild resurrected deleted rows from the untouched `.dat`. Logically deleted rows are now counted (`_pendingLogicalDeletes`) and physically compacted at flush/dispose - (`Table.CompactPendingDeletes`, outside a transaction, Columnar tables with a PK). Regression: - delete half the rows, `Flush`, reopen ÔÇö exactly the remaining rows come back. + (`Table.CompactPendingDeletes`, outside a transaction, Columnar tables with a PK). Flush + compaction rewrites only the data file (live PK positions via B-tree traversal, single-pass index + rebuild) so the cost is proportional to the remaining rows (~0.4s for a 90K-live table); the + overflow arena is reclaimed on the next explicit VACUUM/compaction. Regression: delete half the + rows, `Flush`, reopen ÔÇö exactly the remaining rows come back. Measured DELETE in the `--pk` + harness now includes this durability rewrite (~18.6K ops/s when deleting 10K of 100K rows). - **Dedicated SQL batch-INSERT fast path (WP14)** ÔÇö `ExecuteBatchSQL` INSERTs no longer build a per-row `Dictionary`; VALUES clauses are parsed directly into column-ordered `object[]` rows (`PreparedInsertStatement.ParseValuesToArray`) and inserted via the new