fix(storage): reopen data-integrity fixes for 2.0.0.2

docs/CHANGELOG.md

@@ -196,6 +196,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   at 0.69ÔÇô0.85├ù of SQLite). Full report:
   `docs/benchmarks/AVX512_2026-09-01.md` (+ raw per-run `.md`/`.json` in `docs/benchmarks/avx512-2026-09-01/`).
 
+### Fixed
+
+- **Fixed-width overflow arena silently dropped later rows after an empty value on reopen**
+  - the per-table `.ovf` loader (`Storage.ReadAllRecords`) treated a valid zero-length block (written
+    for an empty TEXT/BLOB value, e.g. the CQRS outbox `last_error`/`next_attempt_utc` columns) as the
+    end of the file. After a reopen every arena block written after such an empty value was never
+    loaded, so later rows came back with empty string/BLOB fields and later in-place UPDATE payloads
+    read as empty. The persisted `.dat`/`.ovf` bytes were intact - only the reload scan stopped early.
+  - Fixed by parsing length-0 records as valid empty records and continuing the scan in both
+    `Storage.ReadAllRecords` and the default `IStorage.ReadAllRecords`.
+  - Reproduced and verified via the SharpCoreDB.CQRS outbox integration tests
+    (`GetUnpublishedAsync` scheduled-future exclusion, `RecordFailureAsync` retry metadata,
+    `RequeueDeadLetterAsync` attempt reset) - all previously failed, all green again. Regression
+    tests `DirectoryFixedWidthDefaultTests.DefaultConfig_EmptyTextValue_DoesNotHideLaterRowsAfterReopen`
+    and `..._UpdateToEmptyAndReopen_KeepsAllRowsIntact`.
+
+- **Single-file (.scdb) fixed-width overflow arena lost values after a reopen with freed blocks**
+  - freed arena blocks were serialized as zero-filled gaps; the sequential arena loader misreads the
+    first dead region as a corrupt/truncated stream and silently drops every later block, so updated
+    and later values came back empty after a reopen (same writer/reader edge-value class as the
+    directory-mode fix above, found by the new reopen round-trip matrix). Fixed by persisting freed
+    slots as negative-length tombstone markers that are tracked across sessions (`_deadSlots`) and
+    skipped on load - the byte stream stays aligned and dead space is reclaimed by the existing
+    copy-on-compact pass.
+
+- **New reopen round-trip matrix (`ReopenRoundTripMatrixTests`)** - four storage variants
+  (directory fixed-width default, directory legacy variable-length, single-file JSON,
+  single-file fixed-width) run insert/update/delete cycles with empty TEXT values interleaved with
+  non-empty ones across three reopen + content-verification rounds.
+
+- **Legacy (1.x-upgrade) variable-length delete-after-update resurrection fixed (critical)** - on a
+  directory-mode table without fixed-width records (`AutoFixedWidthRecords = false`, i.e. 1.x /
+  pre-B7 databases that have not been migrated), an UPDATE appends a new version and the durable
+  DELETE tombstone only marked the newest version. An older stale version of the key then won the
+  reopen index rebuild ("keep latest position") and the deleted row reappeared after a reopen -
+  any 1.x database upgraded to 2.0 was exposed to this on the normal update-then-delete workflow.
+  Fixed by purging every remaining (non-tombstoned) record of a deleted key at DELETE time for
+  legacy columnar tables (single buffered scan for plaintext files, storage-layer fallback for
+  per-record encrypted files). Fixed-width tables (the 2.0 default for new PK tables) were
+  unaffected. Regression: `ReopenRoundTripMatrixTests.DirectoryLegacy_UpdateThenDelete_DoesNotResurrectAfterReopen`
+  and the round-trip matrix legacy variant now exercises delete-after-update across reopen.
+
+  Full core suite 1777 tests, 0 failed; SharpCoreDB.CQRS.Tests 64/64.
+
 ## [2.0.0.1] - 2026-09-01
 
 ### Fixed

src/SharpCoreDB/DataStructures/Table.CRUD.cs

@@ -2704,6 +2704,16 @@ private bool HasExplicitNamedIndex(string column)
             {
                 // Durable DELETE: physically mark the removed records so a reopen skips them.
                 TombstoneDeletedPositions(positions);
+
+                // Legacy variable-length (non-fixed-width) columnar tables keep older stale versions
+                // of a key in the file (UPDATE appends a new version). Tombstoning only the newest
+                // version would let an older stale version win the reopen index rebuild ("keep the
+                // latest position per key") and resurrect the deleted row — purge the remaining
+                // records that carry a deleted key too.
+                if (StorageMode == StorageMode.Columnar && !_fixedWidthRecords && PrimaryKeyIndex >= 0)
+                {
+                    TombstoneRemainingVersionsOfDeletedKeys(recordsToDelete);
+                }
             }
 
             TryAutoCompact();
@@ -2712,6 +2722,171 @@ private bool HasExplicitNamedIndex(string column)
         Interlocked.Add(ref _cachedRowCount, -recordsToDelete.Count);
     }
 
+    /// <summary>
+    /// Legacy (variable-length) columnar tables append a new row version on every UPDATE, leaving
+    /// older stale versions of the same key in the data file. A durable DELETE tombstones only the
+    /// newest version; an older stale version would then win the reopen index rebuild and resurrect
+    /// the deleted row. This scans the file once and tombstones every remaining (non-tombstoned)
+    /// record whose primary key was just deleted. Fixed-width tables are unaffected (their UPDATEs
+    /// are in-place overwrites, so at most one live version per key exists).
+    /// </summary>
+    private void TombstoneRemainingVersionsOfDeletedKeys(List<(long storagePosition, Dictionary<string, object> row)> recordsToDelete)
+    {
+        if (this.storage is null || recordsToDelete.Count == 0)
+        {
+            return;
+        }
+
+        var pkCol = this.Columns[this.PrimaryKeyIndex];
+        var deletedKeys = new HashSet<string>(StringComparer.Ordinal);
+        foreach (var (_, row) in recordsToDelete)
+        {
+            if (row.TryGetValue(pkCol, out var pkValue) && pkValue != null)
+            {
+                deletedKeys.Add(pkValue.ToString() ?? string.Empty);
+            }
+        }
+
+        if (deletedKeys.Count == 0 || !File.Exists(DataFile))
+        {
+            return;
+        }
+
+        List<long>? remainingPositions = null;
+        if (!this.storage.AreRecordsEncrypted(DataFile))
+        {
+            remainingPositions = ScanLegacyPlaintextRemainingKeys(DataFile, pkCol, deletedKeys);
+        }
+
+        remainingPositions ??= ScanLegacyRemainingKeysViaStorage(DataFile, pkCol, deletedKeys);
+        if (remainingPositions is { Count: > 0 })
+        {
+            TombstoneDeletedPositions(remainingPositions.ToArray());
+        }
+    }
+
+    /// <summary>
+    /// Single buffered pass over a plaintext legacy data file: parse the length-prefixed record
+    /// stream inline (skipping tombstone markers) and decode only the PK column. Returns the
+    /// physical offsets of records whose PK is in <paramref name="deletedKeys"/>, or
+    /// <see langword="null"/> when the raw layout could not be parsed safely (the caller then falls
+    /// back to the storage-layer scan, which understands per-record encryption).
+    /// </summary>
+    private List<long>? ScanLegacyPlaintextRemainingKeys(string dataFile, string pkCol, HashSet<string> deletedKeys)
+    {
+        var matches = new List<long>();
+        try
+        {
+            using var fs = new FileStream(
+                dataFile, FileMode.Open, FileAccess.Read,
+                FileShare.ReadWrite | FileShare.Delete, 65536, FileOptions.SequentialScan);
+
+            Span<byte> lengthBuf = stackalloc byte[4];
+            long position = 0;
+            while (fs.Position < fs.Length)
+            {
+                if (fs.Read(lengthBuf) < 4)
+                {
+                    break;
+                }
+
+                int length = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(lengthBuf);
+                if (length < 0)
+                {
+                    int slotSize = -length;
+                    if (slotSize < 4 || position + slotSize > fs.Length)
+                    {
+                        break;
+                    }
+
+                    fs.Seek(slotSize - 4, SeekOrigin.Current);
+                    position += slotSize;
+                    continue;
+                }
+
+                if (length == 0)
+                {
+                    // Valid zero-length record (empty payload): nothing to read, keep scanning.
+                    position += 4;
+                    continue;
+                }
+
+                if (position + 4 + length > fs.Length)
+                {
+                    break;
+                }
+
+                byte[] recordData = new byte[length];
+                if (fs.Read(recordData) < length)
+                {
+                    break;
+                }
+
+                if (TryReadPrimaryKeyFromLegacyRecord(recordData, pkCol, out var pkStr) && deletedKeys.Contains(pkStr))
+                {
+                    matches.Add(position);
+                }
+
+                position += 4 + length;
+            }
+        }
+        catch (IOException)
+        {
+            return null;
+        }
+
+        return matches;
+    }
+
+    /// <summary>
+    /// Storage-layer scan used for encrypted per-record data files (and as the fallback when the
+    /// plaintext raw scan is unavailable): iterates the records through
+    /// <c>storage.ReadAllRecords</c>, which decrypts payloads and already skips tombstone markers.
+    /// </summary>
+    private List<long> ScanLegacyRemainingKeysViaStorage(string dataFile, string pkCol, HashSet<string> deletedKeys)
+    {
+        var matches = new List<long>();
+        foreach (var (recordOffset, recordData) in this.storage!.ReadAllRecords(dataFile))
+        {
+            if (TryReadPrimaryKeyFromLegacyRecord(recordData, pkCol, out var pkStr) && deletedKeys.Contains(pkStr))
+            {
+                matches.Add(recordOffset);
+            }
+        }
+
+        return matches;
+    }
+
+    /// <summary>
+    /// Walks a legacy variable-length record and returns the value of the primary-key column
+    /// (the same layout walk used by the reopen index rebuild).
+    /// </summary>
+    private bool TryReadPrimaryKeyFromLegacyRecord(byte[] recordData, string pkCol, out string? pkValue)
+    {
+        pkValue = null;
+        try
+        {
+            int offset = 0;
+            for (int i = 0; i < Columns.Count && offset < recordData.Length; i++)
+            {
+                var value = ReadTypedValueFromSpan(recordData.AsSpan(offset), ColumnTypes[i], out int bytesRead);
+                if (i == PrimaryKeyIndex && value != null)
+                {
+                    pkValue = value.ToString();
+                    return true;
+                }
+
+                offset += bytesRead;
+            }
+        }
+        catch
+        {
+            // Corrupt / unexpected record — mirror the index-rebuild tolerance.
+        }
+
+        return false;
+    }
+
     /// <summary>
     /// Deletes rows from the table that match the WHERE condition.
     /// Routes through storage engine with different semantics:

src/SharpCoreDB/Interfaces/IStorage.cs

@@ -247,7 +247,17 @@ void TombstoneRecords(string path, long[] offsets)
         while (position + 4 <= data.Length)
         {
             int length = BitConverter.ToInt32(data, (int)position);
-            if (length <= 0 || length > MaxRecordSizeLocal || position + 4 + length > data.Length)
+            if (length == 0)
+            {
+                // Valid zero-length record (e.g. an overflow-arena block for an empty TEXT/BLOB
+                // value). Yield an empty payload and keep scanning so later records/blocks are
+                // not silently dropped on reload.
+                yield return (position, []);
+                position += 4;
+                continue;
+            }
+
+            if (length < 0 || length > MaxRecordSizeLocal || position + 4 + length > data.Length)
             {
                 yield break;
             }

src/SharpCoreDB/Services/Storage.Append.cs

@@ -1346,7 +1346,18 @@ private bool TryFlushBufferedOverwritesBatched(string path, Dictionary<long, byt
                 continue;
             }
 
-            if (length <= 0 || length > MaxRecordSize || position + 4 + length > fileLength)
+            if (length == 0)
+            {
+                // Valid empty record (a zero-length payload — e.g. an overflow-arena block written
+                // for an empty TEXT/BLOB value). There are no payload bytes to read, but the block
+                // occupies a real offset, so yield an empty payload and keep scanning. Treating it
+                // as the end-of-file would silently drop every later record/block on reload.
+                yield return (position, []);
+                position += 4;
+                continue;
+            }
+
+            if (length > MaxRecordSize || position + 4 + length > fileLength)
             {
                 yield break; // Invalid or incomplete record tail
             }

src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs

@@ -21,6 +21,10 @@ public sealed class SingleFileOverflowArena : IOverflowArena
     private readonly Dictionary<long, byte[]> _blocks = new();
     private readonly Dictionary<int, List<long>> _freeByLength = new();
     private readonly Dictionary<string, long> _contentIndex = new(System.StringComparer.Ordinal);
+    // Dead (freed) block slots that must be re-emitted as tombstone markers on every serialize so
+    // the byte stream stays aligned for the sequential deserializer. Populated on load (markers)
+    // and on Free; cleared when a slot is reused in place or the arena is compacted.
+    private readonly Dictionary<long, int> _deadSlots = new();
     private long _nextOffset;
     private int _blockReuses;
 
@@ -101,6 +105,7 @@ public long Write(byte[] payload)
                 _freeByLength.Remove(payload.Length);
             }
 
+            _deadSlots.Remove(offset); // reused in place: no longer a dead slot
             _blocks[offset] = payload;
             _contentIndex[contentKey] = offset;
             _blockReuses++;
@@ -138,6 +143,7 @@ public void Free(long offset)
         }
 
         offsets.Add(offset);
+        _deadSlots[offset] = 4 + payload.Length;
     }
 
     /// <summary>Serializes all blocks (live and freed) as a contiguous <c>[length][payload]</c> stream.</summary>
@@ -155,6 +161,18 @@ public byte[] Serialize()
             payload.CopyTo(buffer, (int)offset + 4);
         }
 
+        // Dead slots (freed blocks from this session or loaded from a previous one) are written as
+        // tombstone markers (negative length = total slot span to skip). Without them a dead region
+        // would be serialized as a zero-filled gap that misaligns the sequential deserializer and
+        // drops every later block on reload.
+        foreach (var (offset, slotSize) in _deadSlots)
+        {
+            if (offset >= 0 && offset + 4 <= buffer.Length)
+            {
+                BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan((int)offset, 4), -slotSize);
+            }
+        }
+
         return buffer;
     }
 
@@ -175,7 +193,32 @@ public static SingleFileOverflowArena Deserialize(byte[]? data)
         while (position + 4 <= data.Length)
         {
             int length = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan((int)position, 4));
-            if (length < 0 || position + 4 + length > data.Length)
+            if (length < 0)
+            {
+                // Tombstone marker: the negative value encodes the whole slot span to skip
+                // (freed overflow blocks are serialized as markers, not zero-filled gaps).
+                int slotSize = -length;
+                if (slotSize < 4 || position + slotSize > data.Length)
+                {
+                    break; // truncated / corrupt
+                }
+
+                // Track the dead slot so this flush re-emits the marker (a marker consumed on load
+                // would otherwise come back as a zero-filled gap on the next serialize).
+                arena._deadSlots[position] = slotSize;
+                int deadPayloadLength = slotSize - 4;
+                if (!arena._freeByLength.TryGetValue(deadPayloadLength, out var deadOffsets))
+                {
+                    deadOffsets = [];
+                    arena._freeByLength[deadPayloadLength] = deadOffsets;
+                }
+
+                deadOffsets.Add(position);
+                position += slotSize;
+                continue;
+            }
+
+            if (position + 4 + length > data.Length)
             {
                 break; // truncated / corrupt
             }
@@ -213,6 +256,7 @@ public Dictionary<long, long> Compact(IReadOnlyCollection<long> activeOffsets)
 
         _blocks.Clear();
         _freeByLength.Clear();
+        _deadSlots.Clear();
         _contentIndex.Clear();
         foreach (var (newOffset, payload) in newBlocks)
         {