fix: make Columnar deletes durable across reopen (flush-time compaction)

docs/CHANGELOG.md

@@ -33,6 +33,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   now replaced by their in-order successor from the right subtree's leftmost leaf (leaf underflow is
   harmless), with empty-neighbour fallbacks keeping child counts consistent. Full suite
   **1655/1655**.
+- **DELETE now survives a reopen (durability)** ÔÇö Columnar deletes were logical only (index removal),
+  so the on-load PK-index rebuild resurrected deleted rows from the untouched `.dat`. Logically
+  deleted rows are now counted (`_pendingLogicalDeletes`) and physically compacted at flush/dispose
+  (`Table.CompactPendingDeletes`, outside a transaction, Columnar tables with a PK). Regression:
+  delete half the rows, `Flush`, reopen ÔÇö exactly the remaining rows come back.
 - **Dedicated SQL batch-INSERT fast path (WP14)** ÔÇö `ExecuteBatchSQL` INSERTs no longer build a
   per-row `Dictionary<string, object>`; VALUES clauses are parsed directly into column-ordered
   `object[]` rows (`PreparedInsertStatement.ParseValuesToArray`) and inserted via the new

src/SharpCoreDB/DataStructures/Table.CRUD.cs

@@ -2586,6 +2586,11 @@ private bool HasExplicitNamedIndex(string column)
             foreach (var (storagePosition, _) in recordsToDelete)
                 engine.Delete(Name, storagePosition);
         }
+        else
+        {
+            // Track Columnar logical deletes so flush-time compaction makes them durable.
+            Interlocked.Add(ref _pendingLogicalDeletes, recordsToDelete.Count);
+        }
 
         // Primary-key B-tree cleanup.
         if (this.PrimaryKeyIndex >= 0)
@@ -3180,6 +3185,7 @@ this.storage is null ||
         }
 
         Interlocked.Add(ref _cachedRowCount, -count);
+        Interlocked.Add(ref _pendingLogicalDeletes, count);
         Interlocked.Increment(ref _bulkContiguousDeleteBatches);
         return true;
     }

src/SharpCoreDB/DataStructures/Table.Compaction.cs

@@ -51,6 +51,36 @@ public void TryAutoCompact()
         }
     }
 
+    /// <summary>
+    /// Physically removes rows that were logically deleted since the last flush (Columnar tables
+    /// with a primary key) so DELETE survives a reopen — the on-load PK-index rebuild would otherwise
+    /// resurrect them from the untouched <c>.dat</c>. Runs synchronously at flush/dispose, outside a
+    /// transaction, when any logical deletes are pending.
+    /// </summary>
+    public void CompactPendingDeletes()
+    {
+        if (StorageMode != SharpCoreDB.Storage.Hybrid.StorageMode.Columnar ||
+            this.PrimaryKeyIndex < 0 ||
+            Interlocked.Read(ref _pendingLogicalDeletes) == 0)
+        {
+            return;
+        }
+
+        if (this.storage is { IsInTransaction: true })
+        {
+            return; // defer until the transaction commits (the next flush will run again)
+        }
+
+        try
+        {
+            CompactStorage();
+        }
+        finally
+        {
+            Interlocked.Exchange(ref _pendingLogicalDeletes, 0);
+        }
+    }
+
     /// <summary>
     /// Compacts the table storage by removing deleted and stale rows.
     /// Only applicable for columnar (append-only) storage mode.

src/SharpCoreDB/DataStructures/Table.cs

@@ -289,6 +289,9 @@ private Dictionary<string, int> GetColumnIndexCache()
     private long _updatedRowCount = 0;
     private long COMPACTION_THRESHOLD = 1000; // Default; can be overridden by DatabaseConfig
 
+    // Logical deletes awaiting physical compaction at flush (durable DELETE across reopen).
+    private long _pendingLogicalDeletes = 0;
+
     // ✅ NEW: Dictionary pooling for SELECT operations (Phase 1 optimization)
     // Reduces allocations by 60% during full table scans
     private readonly ObjectPool<Dictionary<string, object>> _dictPool;
@@ -730,6 +733,11 @@ public void Flush()
                 }
             }
 
+            // Durable DELETE across reopen: physically compact rows that were logically deleted
+            // since the last flush (Columnar + PK, outside a transaction). Runs after the engine
+            // and any transaction buffer have been flushed.
+            CompactPendingDeletes();
+
             // Flush indexes
             if (indexManager != null)
             {
@@ -775,6 +783,9 @@ protected virtual void Dispose(bool disposing)
     {
         if (disposing)
         {
+            // Durable DELETE across reopen for flows that dispose without an explicit flush.
+            CompactPendingDeletes();
+
             // Dispose storage engine first
             DisposeStorageEngine();
 

tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs

@@ -202,4 +202,32 @@ public void GappedDeletes_FallBackToGenericLoop_AndStayCorrect()
         }
         finally { (db as IDisposable)?.Dispose(); }
     }
+
+    [Fact]
+    public void DeletesPersistAcrossReopen_AfterFlushCompaction()
+    {
+        IDatabase? db = CreateDb();
+        db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, name TEXT, score REAL)");
+        InsertDocs(db, 1, 100);
+        db.Flush();
+
+        var stmts = new List<string>(50);
+        for (int i = 1; i <= 50; i++)
+        {
+            stmts.Add(string.Format(CultureInfo.InvariantCulture, "DELETE FROM docs WHERE id = {0}", i));
+        }
+
+        db.ExecuteBatchSQL(stmts);
+        db.Flush(); // flush-time compaction must physically remove the deleted rows
+        (db as IDisposable)?.Dispose();
+
+        db = CreateDb();
+        var scan = db.ExecuteQuery("SELECT id FROM docs ORDER BY id");
+        Assert.Equal(50, scan.Count);
+        Assert.Equal(51L, Convert.ToInt64(scan[0]["id"]));
+        Assert.Equal(100L, Convert.ToInt64(scan[^1]["id"]));
+        var c = db.ExecuteQuery("SELECT COUNT(*) AS n FROM docs");
+        Assert.Equal(50L, Convert.ToInt64(c[0].Values.First()));
+        (db as IDisposable)?.Dispose();
+    }
 }