From 43141e3c7d02fa1692f9d02a23bfc3c475202242 Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Thu, 3 Sep 2026 16:04:13 +0200 Subject: [PATCH] fix: make Columnar deletes durable across reopen (flush-time compaction) Columnar deletes were logical only (PK/hash-index removal), so the on-load PK-index rebuild resurrected deleted rows from the untouched .dat after a reopen. Logically deleted rows are now tracked (_pendingLogicalDeletes, incremented by DeleteRecordsCore and the contiguous bulk-delete fast path) and physically compacted at Table.Flush/Dispose via CompactPendingDeletes (Columnar + PK, outside a transaction). Regression test DeletesPersistAcrossReopen_AfterFlushCompaction deletes half the rows, flushes, reopens and asserts exactly the remaining rows come back. --- docs/CHANGELOG.md | 5 ++++ src/SharpCoreDB/DataStructures/Table.CRUD.cs | 6 ++++ .../DataStructures/Table.Compaction.cs | 30 +++++++++++++++++++ src/SharpCoreDB/DataStructures/Table.cs | 11 +++++++ .../FixedWidthBulkDeleteTests.cs | 28 +++++++++++++++++ 5 files changed, 80 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2159e681..257c1318 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 now replaced by their in-order successor from the right subtree's leftmost leaf (leaf underflow is harmless), with empty-neighbour fallbacks keeping child counts consistent. Full suite **1655/1655**. +- **DELETE now survives a reopen (durability)** ÔÇö Columnar deletes were logical only (index removal), + so the on-load PK-index rebuild resurrected deleted rows from the untouched `.dat`. Logically + deleted rows are now counted (`_pendingLogicalDeletes`) and physically compacted at flush/dispose + (`Table.CompactPendingDeletes`, outside a transaction, Columnar tables with a PK). Regression: + delete half the rows, `Flush`, reopen ÔÇö exactly the remaining rows come back. - **Dedicated SQL batch-INSERT fast path (WP14)** ÔÇö `ExecuteBatchSQL` INSERTs no longer build a per-row `Dictionary`; VALUES clauses are parsed directly into column-ordered `object[]` rows (`PreparedInsertStatement.ParseValuesToArray`) and inserted via the new diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index 9a8f37d0..7b7325b1 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -2586,6 +2586,11 @@ private bool HasExplicitNamedIndex(string column) foreach (var (storagePosition, _) in recordsToDelete) engine.Delete(Name, storagePosition); } + else + { + // Track Columnar logical deletes so flush-time compaction makes them durable. + Interlocked.Add(ref _pendingLogicalDeletes, recordsToDelete.Count); + } // Primary-key B-tree cleanup. if (this.PrimaryKeyIndex >= 0) @@ -3180,6 +3185,7 @@ this.storage is null || } Interlocked.Add(ref _cachedRowCount, -count); + Interlocked.Add(ref _pendingLogicalDeletes, count); Interlocked.Increment(ref _bulkContiguousDeleteBatches); return true; } diff --git a/src/SharpCoreDB/DataStructures/Table.Compaction.cs b/src/SharpCoreDB/DataStructures/Table.Compaction.cs index ba4271d2..c77eddb8 100644 --- a/src/SharpCoreDB/DataStructures/Table.Compaction.cs +++ b/src/SharpCoreDB/DataStructures/Table.Compaction.cs @@ -51,6 +51,36 @@ public void TryAutoCompact() } } + /// + /// Physically removes rows that were logically deleted since the last flush (Columnar tables + /// with a primary key) so DELETE survives a reopen — the on-load PK-index rebuild would otherwise + /// resurrect them from the untouched .dat. Runs synchronously at flush/dispose, outside a + /// transaction, when any logical deletes are pending. + /// + public void CompactPendingDeletes() + { + if (StorageMode != SharpCoreDB.Storage.Hybrid.StorageMode.Columnar || + this.PrimaryKeyIndex < 0 || + Interlocked.Read(ref _pendingLogicalDeletes) == 0) + { + return; + } + + if (this.storage is { IsInTransaction: true }) + { + return; // defer until the transaction commits (the next flush will run again) + } + + try + { + CompactStorage(); + } + finally + { + Interlocked.Exchange(ref _pendingLogicalDeletes, 0); + } + } + /// /// Compacts the table storage by removing deleted and stale rows. /// Only applicable for columnar (append-only) storage mode. diff --git a/src/SharpCoreDB/DataStructures/Table.cs b/src/SharpCoreDB/DataStructures/Table.cs index eeacc478..b18462f6 100644 --- a/src/SharpCoreDB/DataStructures/Table.cs +++ b/src/SharpCoreDB/DataStructures/Table.cs @@ -289,6 +289,9 @@ private Dictionary GetColumnIndexCache() private long _updatedRowCount = 0; private long COMPACTION_THRESHOLD = 1000; // Default; can be overridden by DatabaseConfig + // Logical deletes awaiting physical compaction at flush (durable DELETE across reopen). + private long _pendingLogicalDeletes = 0; + // ✅ NEW: Dictionary pooling for SELECT operations (Phase 1 optimization) // Reduces allocations by 60% during full table scans private readonly ObjectPool> _dictPool; @@ -730,6 +733,11 @@ public void Flush() } } + // Durable DELETE across reopen: physically compact rows that were logically deleted + // since the last flush (Columnar + PK, outside a transaction). Runs after the engine + // and any transaction buffer have been flushed. + CompactPendingDeletes(); + // Flush indexes if (indexManager != null) { @@ -775,6 +783,9 @@ protected virtual void Dispose(bool disposing) { if (disposing) { + // Durable DELETE across reopen for flows that dispose without an explicit flush. + CompactPendingDeletes(); + // Dispose storage engine first DisposeStorageEngine(); diff --git a/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs b/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs index 89550df3..2ae0b69f 100644 --- a/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs +++ b/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs @@ -202,4 +202,32 @@ public void GappedDeletes_FallBackToGenericLoop_AndStayCorrect() } finally { (db as IDisposable)?.Dispose(); } } + + [Fact] + public void DeletesPersistAcrossReopen_AfterFlushCompaction() + { + IDatabase? db = CreateDb(); + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + InsertDocs(db, 1, 100); + db.Flush(); + + var stmts = new List(50); + for (int i = 1; i <= 50; i++) + { + stmts.Add(string.Format(CultureInfo.InvariantCulture, "DELETE FROM docs WHERE id = {0}", i)); + } + + db.ExecuteBatchSQL(stmts); + db.Flush(); // flush-time compaction must physically remove the deleted rows + (db as IDisposable)?.Dispose(); + + db = CreateDb(); + var scan = db.ExecuteQuery("SELECT id FROM docs ORDER BY id"); + Assert.Equal(50, scan.Count); + Assert.Equal(51L, Convert.ToInt64(scan[0]["id"])); + Assert.Equal(100L, Convert.ToInt64(scan[^1]["id"])); + var c = db.ExecuteQuery("SELECT COUNT(*) AS n FROM docs"); + Assert.Equal(50L, Convert.ToInt64(c[0].Values.First())); + (db as IDisposable)?.Dispose(); + } }