From 0d85ac91b2c83bb5a7efe9ff95e01346a0a08600 Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Thu, 3 Sep 2026 18:38:11 +0200 Subject: [PATCH] perf: durable Columnar DELETE in-place via tombstone markers (no flush rewrite) Non-transactional Columnar deletes now write a tombstone marker over the record's 4-byte length prefix (negative slot size) instead of queueing a flush-time full-file rewrite, so DELETE survives a reopen in O(delete). --pk harness DELETE: ~18.6K ops/s -> ~64K (legacy) / ~81K ops/s (fixed-width), back on par with UPDATE. - IStorage.TombstoneRecord (default false) + Storage implementation (read slot size, write -slotSize marker, evict page cache). - Every raw record enumerator/compactor skips negative prefixes: Storage.ReadAllRecords, AppendOnlyEngine.GetAllRecords + CompactTable, Table.Scanning/CRUD/ParallelScan. - CompactTable broke on a negative prefix, which made the ULID-migration compaction (delete+insert then CompactStorage) write an empty file; it now skips tombstones so migration stays correct. - Transactional deletes are NOT tombstoned at delete time (a marker would survive a rollback that restores the row in the PK index); they are physically removed by the post-commit flush/dispose compaction against the current PK, keeping the rollback-safe semantics of #366. - Tombstone space is reclaimed by the tombstone-aware CompactStorage/CompactTable (explicit, auto-compact threshold, ULID migration). CHANGELOG updated. --- docs/CHANGELOG.md | 11 ++++ src/SharpCoreDB/DataStructures/Table.CRUD.cs | 47 ++++++++++++-- .../DataStructures/Table.Compaction.cs | 39 ++++++++++-- .../DataStructures/Table.ParallelScan.cs | 24 +++++++ .../DataStructures/Table.Scanning.cs | 17 ++++- src/SharpCoreDB/DataStructures/Table.cs | 10 +-- src/SharpCoreDB/Interfaces/IStorage.cs | 9 +++ src/SharpCoreDB/Services/Storage.Append.cs | 63 +++++++++++++++++++ .../Storage/Engines/AppendOnlyEngine.cs | 36 +++++++++-- 9 files changed, 235 insertions(+), 21 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6b3ff8e9..8f0003ce 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -42,6 +42,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 overflow arena is reclaimed on the next explicit VACUUM/compaction. Regression: delete half the rows, `Flush`, reopen ÔÇö exactly the remaining rows come back. Measured DELETE in the `--pk` harness now includes this durability rewrite (~18.6K ops/s when deleting 10K of 100K rows). +- **Durable DELETE is now in-place via tombstones (no rewrite)** ÔÇö non-transactional Columnar + deletes write a tombstone marker (the record's 4-byte length prefix is replaced by the NEGATIVE + slot size) instead of queueing a flush-time rewrite, and every raw record enumerator/compactor + skips the slot, so DELETE survives a reopen in O(delete). Flush/dispose compaction now only + covers transactional deletes (a marker written before commit would survive a rollback that + restores the row in the PK index; those deletes are compacted against the current PK after commit + ÔÇö rollback-safe). Tombstoned space is reclaimed by the tombstone-aware `CompactTable`, including + the ULID-migration compaction (which previously produced an empty file when tombstones were + present because `CompactTable` broke on a negative prefix). Measured `--pk` DELETE (10K of 100K + rows): ~0.54s/18.6K ops/s (flush rewrite) ÔåÆ **~0.16s/~64K ops/s (legacy)** and + **~0.12s/~81K ops/s (fixed-width)** ÔÇö DELETE is back on par with UPDATE. - **Dedicated SQL batch-INSERT fast path (WP14)** ÔÇö `ExecuteBatchSQL` INSERTs no longer build a per-row `Dictionary`; VALUES clauses are parsed directly into column-ordered `object[]` rows (`PreparedInsertStatement.ParseValuesToArray`) and inserted via the new diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index 7b7325b1..24c44b62 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -1364,7 +1364,20 @@ private List> ScanRowsWithSimdAndFilterStale(byte[] d dataSpan.Slice(filePosition, 4)); const int MaxRecordSize = 1_000_000_000; - if (recordLength < 0 || recordLength > MaxRecordSize) + if (recordLength < 0) + { + // Tombstoned (deleted) record: the prefix stores the negative slot size to skip. + int slotSize = -recordLength; + if (slotSize < 4) + { + break; + } + + filePosition += slotSize; + continue; + } + + if (recordLength > MaxRecordSize) { break; } @@ -2586,9 +2599,10 @@ private bool HasExplicitNamedIndex(string column) foreach (var (storagePosition, _) in recordsToDelete) engine.Delete(Name, storagePosition); } - else + else if (StorageMode != StorageMode.Columnar) { - // Track Columnar logical deletes so flush-time compaction makes them durable. + // Legacy logical-delete accounting (Columnar deletes are tracked in the branch below, + // which decides between an immediate durable tombstone and deferred flush compaction). Interlocked.Add(ref _pendingLogicalDeletes, recordsToDelete.Count); } @@ -2639,6 +2653,20 @@ private bool HasExplicitNamedIndex(string column) } } + if (this.storage is { IsInTransaction: true }) + { + // Transactional delete: writing the tombstone now would survive a rollback that + // restores the row in the PK index, so defer the physical removal to the post-commit + // flush compaction (rollback-safe: that rewrite is driven by the CURRENT PK, which + // still contains the row after a rollback). + Interlocked.Add(ref _pendingLogicalDeletes, recordsToDelete.Count); + } + else + { + // Durable DELETE: physically mark the removed records so a reopen skips them. + TombstoneDeletedPositions(positions); + } + TryAutoCompact(); } @@ -3184,8 +3212,19 @@ this.storage is null || hashIdx.RemoveBatchKeys(decoded, positions); } + if (this.storage is { IsInTransaction: true }) + { + // Transactional delete: defer physical removal to the post-commit flush compaction + // (see DeleteRecordsCore — a tombstone would survive a rollback that restores the row). + Interlocked.Add(ref _pendingLogicalDeletes, count); + } + else + { + // Durable DELETE: physically mark the removed records so a reopen skips them. + TombstoneDeletedPositions(positions); + } + Interlocked.Add(ref _cachedRowCount, -count); - Interlocked.Add(ref _pendingLogicalDeletes, count); Interlocked.Increment(ref _bulkContiguousDeleteBatches); return true; } diff --git a/src/SharpCoreDB/DataStructures/Table.Compaction.cs b/src/SharpCoreDB/DataStructures/Table.Compaction.cs index e19f4ae2..e3a49518 100644 --- a/src/SharpCoreDB/DataStructures/Table.Compaction.cs +++ b/src/SharpCoreDB/DataStructures/Table.Compaction.cs @@ -8,6 +8,7 @@ namespace SharpCoreDB.DataStructures; using SharpCoreDB.Storage.Engines; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -52,12 +53,14 @@ public void TryAutoCompact() } /// - /// Physically removes rows that were logically deleted since the last flush (Columnar tables - /// with a primary key) so DELETE survives a reopen — the on-load PK-index rebuild would otherwise - /// resurrect them from the untouched .dat. Runs synchronously at flush/dispose, outside a - /// transaction, when any logical deletes are pending. Data-file only: the overflow arena is left - /// untouched (its space is reclaimed by a later explicit compaction/VACUUM) so the flush stays - /// proportional to the rewritten live rows. + /// Makes TRANSACTIONAL deletes durable (Columnar tables with a primary key). Deletes issued + /// inside a transaction are intentionally not tombstoned at delete time (a marker would survive a + /// rollback that restores the row in the PK index), so they are physically removed here by + /// compacting against the CURRENT PK once the owning transaction has committed. Runs at + /// flush/dispose, outside a transaction, when any transactional deletes are pending. No-op when + /// none are pending — non-transactional DELETE is already durable via the per-row tombstone and + /// must not pay a rewrite. Data-file only: the overflow arena is left untouched (its space is + /// reclaimed by a later explicit compaction/VACUUM). /// public void CompactPendingDeletes() { @@ -105,6 +108,30 @@ public void CompactPendingDeletes() } } + /// + /// Writes the deleted-record marker over the length prefix of each position that is physically + /// present in the data file, making the Columnar logical delete durable across reopen (readers + /// skip the marker) without rewriting the remaining rows. Rows appended inside an uncommitted + /// transaction (positions beyond the current file length) are skipped — their delete commits + /// with the rest of the transaction. + /// + private void TombstoneDeletedPositions(long[] positions) + { + if (this.storage is null || positions.Length == 0) + { + return; + } + + long fileLength = File.Exists(DataFile) ? new FileInfo(DataFile).Length : 0; + foreach (var position in positions) + { + if (position >= 0 && position + 4 <= fileLength) + { + this.storage.TombstoneRecord(DataFile, position); + } + } + } + /// /// Compacts the table storage by removing deleted and stale rows. /// Only applicable for columnar (append-only) storage mode. diff --git a/src/SharpCoreDB/DataStructures/Table.ParallelScan.cs b/src/SharpCoreDB/DataStructures/Table.ParallelScan.cs index cbbade82..76777bfe 100644 --- a/src/SharpCoreDB/DataStructures/Table.ParallelScan.cs +++ b/src/SharpCoreDB/DataStructures/Table.ParallelScan.cs @@ -173,6 +173,18 @@ private List> ScanRowsParallel(byte[] data, string? w int recordLength = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian( data.AsSpan(filePosition, 4)); + if (recordLength < 0) + { + int slotSize = -recordLength; // tombstoned slot: skip the full slot + if (slotSize < 4) + { + break; + } + + filePosition += slotSize; + continue; + } + if (recordLength <= 0 || recordLength > 1_000_000_000) break; currentRecordCount++; @@ -207,6 +219,18 @@ private List> ScanRowsParallel(byte[] data, string? w int recordLength = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian( partitionData.Slice(localFilePosition, 4)); + if (recordLength < 0) + { + int slotSize = -recordLength; // tombstoned slot: skip the full slot + if (slotSize < 4) + { + break; + } + + localFilePosition += slotSize; + continue; + } + if (recordLength <= 0 || recordLength > 1_000_000_000) break; if (localFilePosition + 4 + recordLength > partitionData.Length) break; diff --git a/src/SharpCoreDB/DataStructures/Table.Scanning.cs b/src/SharpCoreDB/DataStructures/Table.Scanning.cs index 820138b7..fc013348 100644 --- a/src/SharpCoreDB/DataStructures/Table.Scanning.cs +++ b/src/SharpCoreDB/DataStructures/Table.Scanning.cs @@ -65,10 +65,23 @@ private List> ScanRowsWithSimd(byte[] data, string? w // ✅ C# 14: Range operator - extract length prefix span first var lengthSpan = dataSpan[filePosition..(filePosition + 4)]; int recordLength = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(lengthSpan); - + + if (recordLength < 0) + { + // Tombstoned (deleted) record: the prefix stores the negative slot size to skip. + int slotSize = -recordLength; + if (slotSize < 4) + { + break; + } + + filePosition += slotSize; + continue; + } + // Sanity check: record length must be reasonable const int MaxRecordSize = 1_000_000_000; // 1 GB max per record - if (recordLength < 0 || recordLength > MaxRecordSize) + if (recordLength > MaxRecordSize) { break; } diff --git a/src/SharpCoreDB/DataStructures/Table.cs b/src/SharpCoreDB/DataStructures/Table.cs index b18462f6..2e93e975 100644 --- a/src/SharpCoreDB/DataStructures/Table.cs +++ b/src/SharpCoreDB/DataStructures/Table.cs @@ -733,9 +733,10 @@ public void Flush() } } - // Durable DELETE across reopen: physically compact rows that were logically deleted - // since the last flush (Columnar + PK, outside a transaction). Runs after the engine - // and any transaction buffer have been flushed. + // Transactional deletes (which are intentionally NOT tombstoned at delete time so a + // rollback stays correct) become durable by compacting against the current PK once the + // owning transaction has committed. No-op when there are no deferred transactional + // deletes, so non-transactional DELETE keeps the O(delete) tombstone path. CompactPendingDeletes(); // Flush indexes @@ -783,7 +784,8 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - // Durable DELETE across reopen for flows that dispose without an explicit flush. + // Transactional deletes deferred past their commit flush (e.g. dispose-without-flush + // flows) are compacted here; no-op when none are pending. CompactPendingDeletes(); // Dispose storage engine first diff --git a/src/SharpCoreDB/Interfaces/IStorage.cs b/src/SharpCoreDB/Interfaces/IStorage.cs index 71184cd9..3aecaf81 100644 --- a/src/SharpCoreDB/Interfaces/IStorage.cs +++ b/src/SharpCoreDB/Interfaces/IStorage.cs @@ -170,6 +170,15 @@ bool OverwriteRecordAtSameLength(string path, long offset, byte[] data) => /// bool AreRecordsEncrypted(string path) => false; + /// + /// Marks the record whose 4-byte length prefix sits at as deleted by + /// replacing the prefix with the NEGATIVE slot size (4-byte prefix + payload). Every record + /// enumerator treats a negative prefix as a deleted record and skips |value| bytes, so the + /// delete survives a reopen without rewriting the file. The default returns false (unsupported + /// layout / mock storage). + /// + bool TombstoneRecord(string path, long offset) => false; + /// /// Enumerates every record in a table data file, yielding the literal file offset of the /// 4-byte length prefix (the offset returned by ) together with the diff --git a/src/SharpCoreDB/Services/Storage.Append.cs b/src/SharpCoreDB/Services/Storage.Append.cs index a02d9cca..4693d029 100644 --- a/src/SharpCoreDB/Services/Storage.Append.cs +++ b/src/SharpCoreDB/Services/Storage.Append.cs @@ -563,6 +563,56 @@ public bool HasBufferedOverwrite(string path) => /// public bool AreRecordsEncrypted(string path) => UseRecordEncryption && FileHasEncryptedHeader(path); + /// + public bool TombstoneRecord(string path, long offset) + { + // Read the current slot size so the marker can encode the exact number of bytes to skip + // (4-byte prefix + payload), keeping every record enumerator aligned. + int slotSize; + try + { + SafeFileHandle readHandle = GetOrOpenReadHandle(path); + Span lengthBuffer = stackalloc byte[4]; + if (RandomAccess.Read(readHandle, lengthBuffer, offset) != 4) + { + return false; + } + + int currentLength = BinaryPrimitives.ReadInt32LittleEndian(lengthBuffer); + if (currentLength <= 0) + { + return false; // already tombstoned or invalid + } + + slotSize = 4 + currentLength; + } + catch (IOException) + { + return false; + } + + Span marker = stackalloc byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(marker, -slotSize); + + try + { + WriteRecordInPlace(path, offset, marker, ReadOnlySpan.Empty); + } + catch (IOException) + { + return false; + } + + // Invalidate the app-level page cache (mirrors the other in-place writers). + if (this.pageCache != null) + { + int pageId = ComputePageId(path, offset); + this.pageCache.EvictPage(pageId); + } + + return true; + } + /// [MethodImpl(MethodImplOptions.AggressiveOptimization)] public long[] AppendBytesMultiple(string path, List dataBlocks) @@ -924,6 +974,19 @@ private void FlushBufferedOverwrites() } int length = BinaryPrimitives.ReadInt32LittleEndian(lengthBuffer); + if (length < 0) + { + // Tombstoned (deleted) record: the prefix stores the negative slot size to skip. + int slotSize = -length; + if (slotSize < 4) + { + yield break; + } + + position += slotSize; + continue; + } + if (length <= 0 || length > MaxRecordSize || position + 4 + length > fileLength) { yield break; // Invalid or incomplete record tail diff --git a/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs b/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs index bac79e4e..e0d4cdce 100644 --- a/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs +++ b/src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs @@ -214,8 +214,21 @@ public void Delete(string tableName, long storageReference) // Read record length (4 bytes, little-endian) int recordLength = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian( allData.AsSpan((int)position, 4)); - - if (recordLength <= 0 || position + 4 + recordLength > allData.Length) + + if (recordLength < 0) + { + // Tombstoned (deleted) record: the prefix stores the negative slot size to skip. + int slotSize = -recordLength; + if (slotSize < 4) + { + break; + } + + position += slotSize; + continue; + } + + if (recordLength == 0 || position + 4 + recordLength > allData.Length) { break; // Invalid or incomplete record } @@ -354,10 +367,23 @@ public long CompactTable(string tableName, List activePositions) int recordLength = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian( allData.AsSpan((int)position, 4)); - - if (recordLength <= 0 || position + 4 + recordLength > allData.Length) + + if (recordLength < 0) + { + // Tombstoned (deleted) record: the prefix stores the negative slot size to skip. + int slotSize = -recordLength; + if (slotSize < 4) + { + break; + } + + position += slotSize; + continue; + } + + if (recordLength == 0 || position + 4 + recordLength > allData.Length) break; - + // Check if this position is active if (activeSet.Contains(position)) {