diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f275e223..d86865de 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -196,6 +196,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 at 0.69ÔÇô0.85├ù of SQLite). Full report: `docs/benchmarks/AVX512_2026-09-01.md` (+ raw per-run `.md`/`.json` in `docs/benchmarks/avx512-2026-09-01/`). +### Fixed + +- **Fixed-width overflow arena silently dropped later rows after an empty value on reopen** + - the per-table `.ovf` loader (`Storage.ReadAllRecords`) treated a valid zero-length block (written + for an empty TEXT/BLOB value, e.g. the CQRS outbox `last_error`/`next_attempt_utc` columns) as the + end of the file. After a reopen every arena block written after such an empty value was never + loaded, so later rows came back with empty string/BLOB fields and later in-place UPDATE payloads + read as empty. The persisted `.dat`/`.ovf` bytes were intact - only the reload scan stopped early. + - Fixed by parsing length-0 records as valid empty records and continuing the scan in both + `Storage.ReadAllRecords` and the default `IStorage.ReadAllRecords`. + - Reproduced and verified via the SharpCoreDB.CQRS outbox integration tests + (`GetUnpublishedAsync` scheduled-future exclusion, `RecordFailureAsync` retry metadata, + `RequeueDeadLetterAsync` attempt reset) - all previously failed, all green again. Regression + tests `DirectoryFixedWidthDefaultTests.DefaultConfig_EmptyTextValue_DoesNotHideLaterRowsAfterReopen` + and `..._UpdateToEmptyAndReopen_KeepsAllRowsIntact`. + +- **Single-file (.scdb) fixed-width overflow arena lost values after a reopen with freed blocks** + - freed arena blocks were serialized as zero-filled gaps; the sequential arena loader misreads the + first dead region as a corrupt/truncated stream and silently drops every later block, so updated + and later values came back empty after a reopen (same writer/reader edge-value class as the + directory-mode fix above, found by the new reopen round-trip matrix). Fixed by persisting freed + slots as negative-length tombstone markers that are tracked across sessions (`_deadSlots`) and + skipped on load - the byte stream stays aligned and dead space is reclaimed by the existing + copy-on-compact pass. + +- **New reopen round-trip matrix (`ReopenRoundTripMatrixTests`)** - four storage variants + (directory fixed-width default, directory legacy variable-length, single-file JSON, + single-file fixed-width) run insert/update/delete cycles with empty TEXT values interleaved with + non-empty ones across three reopen + content-verification rounds. + +- **Legacy (1.x-upgrade) variable-length delete-after-update resurrection fixed (critical)** - on a + directory-mode table without fixed-width records (`AutoFixedWidthRecords = false`, i.e. 1.x / + pre-B7 databases that have not been migrated), an UPDATE appends a new version and the durable + DELETE tombstone only marked the newest version. An older stale version of the key then won the + reopen index rebuild ("keep latest position") and the deleted row reappeared after a reopen - + any 1.x database upgraded to 2.0 was exposed to this on the normal update-then-delete workflow. + Fixed by purging every remaining (non-tombstoned) record of a deleted key at DELETE time for + legacy columnar tables (single buffered scan for plaintext files, storage-layer fallback for + per-record encrypted files). Fixed-width tables (the 2.0 default for new PK tables) were + unaffected. Regression: `ReopenRoundTripMatrixTests.DirectoryLegacy_UpdateThenDelete_DoesNotResurrectAfterReopen` + and the round-trip matrix legacy variant now exercises delete-after-update across reopen. + + Full core suite 1777 tests, 0 failed; SharpCoreDB.CQRS.Tests 64/64. + ## [2.0.0.1] - 2026-09-01 ### Fixed diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index dfee24df..958791f6 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -2704,6 +2704,16 @@ private bool HasExplicitNamedIndex(string column) { // Durable DELETE: physically mark the removed records so a reopen skips them. TombstoneDeletedPositions(positions); + + // Legacy variable-length (non-fixed-width) columnar tables keep older stale versions + // of a key in the file (UPDATE appends a new version). Tombstoning only the newest + // version would let an older stale version win the reopen index rebuild ("keep the + // latest position per key") and resurrect the deleted row — purge the remaining + // records that carry a deleted key too. + if (StorageMode == StorageMode.Columnar && !_fixedWidthRecords && PrimaryKeyIndex >= 0) + { + TombstoneRemainingVersionsOfDeletedKeys(recordsToDelete); + } } TryAutoCompact(); @@ -2712,6 +2722,171 @@ private bool HasExplicitNamedIndex(string column) Interlocked.Add(ref _cachedRowCount, -recordsToDelete.Count); } + /// + /// Legacy (variable-length) columnar tables append a new row version on every UPDATE, leaving + /// older stale versions of the same key in the data file. A durable DELETE tombstones only the + /// newest version; an older stale version would then win the reopen index rebuild and resurrect + /// the deleted row. This scans the file once and tombstones every remaining (non-tombstoned) + /// record whose primary key was just deleted. Fixed-width tables are unaffected (their UPDATEs + /// are in-place overwrites, so at most one live version per key exists). + /// + private void TombstoneRemainingVersionsOfDeletedKeys(List<(long storagePosition, Dictionary row)> recordsToDelete) + { + if (this.storage is null || recordsToDelete.Count == 0) + { + return; + } + + var pkCol = this.Columns[this.PrimaryKeyIndex]; + var deletedKeys = new HashSet(StringComparer.Ordinal); + foreach (var (_, row) in recordsToDelete) + { + if (row.TryGetValue(pkCol, out var pkValue) && pkValue != null) + { + deletedKeys.Add(pkValue.ToString() ?? string.Empty); + } + } + + if (deletedKeys.Count == 0 || !File.Exists(DataFile)) + { + return; + } + + List? remainingPositions = null; + if (!this.storage.AreRecordsEncrypted(DataFile)) + { + remainingPositions = ScanLegacyPlaintextRemainingKeys(DataFile, pkCol, deletedKeys); + } + + remainingPositions ??= ScanLegacyRemainingKeysViaStorage(DataFile, pkCol, deletedKeys); + if (remainingPositions is { Count: > 0 }) + { + TombstoneDeletedPositions(remainingPositions.ToArray()); + } + } + + /// + /// Single buffered pass over a plaintext legacy data file: parse the length-prefixed record + /// stream inline (skipping tombstone markers) and decode only the PK column. Returns the + /// physical offsets of records whose PK is in , or + /// when the raw layout could not be parsed safely (the caller then falls + /// back to the storage-layer scan, which understands per-record encryption). + /// + private List? ScanLegacyPlaintextRemainingKeys(string dataFile, string pkCol, HashSet deletedKeys) + { + var matches = new List(); + try + { + using var fs = new FileStream( + dataFile, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, 65536, FileOptions.SequentialScan); + + Span lengthBuf = stackalloc byte[4]; + long position = 0; + while (fs.Position < fs.Length) + { + if (fs.Read(lengthBuf) < 4) + { + break; + } + + int length = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(lengthBuf); + if (length < 0) + { + int slotSize = -length; + if (slotSize < 4 || position + slotSize > fs.Length) + { + break; + } + + fs.Seek(slotSize - 4, SeekOrigin.Current); + position += slotSize; + continue; + } + + if (length == 0) + { + // Valid zero-length record (empty payload): nothing to read, keep scanning. + position += 4; + continue; + } + + if (position + 4 + length > fs.Length) + { + break; + } + + byte[] recordData = new byte[length]; + if (fs.Read(recordData) < length) + { + break; + } + + if (TryReadPrimaryKeyFromLegacyRecord(recordData, pkCol, out var pkStr) && deletedKeys.Contains(pkStr)) + { + matches.Add(position); + } + + position += 4 + length; + } + } + catch (IOException) + { + return null; + } + + return matches; + } + + /// + /// Storage-layer scan used for encrypted per-record data files (and as the fallback when the + /// plaintext raw scan is unavailable): iterates the records through + /// storage.ReadAllRecords, which decrypts payloads and already skips tombstone markers. + /// + private List ScanLegacyRemainingKeysViaStorage(string dataFile, string pkCol, HashSet deletedKeys) + { + var matches = new List(); + foreach (var (recordOffset, recordData) in this.storage!.ReadAllRecords(dataFile)) + { + if (TryReadPrimaryKeyFromLegacyRecord(recordData, pkCol, out var pkStr) && deletedKeys.Contains(pkStr)) + { + matches.Add(recordOffset); + } + } + + return matches; + } + + /// + /// Walks a legacy variable-length record and returns the value of the primary-key column + /// (the same layout walk used by the reopen index rebuild). + /// + private bool TryReadPrimaryKeyFromLegacyRecord(byte[] recordData, string pkCol, out string? pkValue) + { + pkValue = null; + try + { + int offset = 0; + for (int i = 0; i < Columns.Count && offset < recordData.Length; i++) + { + var value = ReadTypedValueFromSpan(recordData.AsSpan(offset), ColumnTypes[i], out int bytesRead); + if (i == PrimaryKeyIndex && value != null) + { + pkValue = value.ToString(); + return true; + } + + offset += bytesRead; + } + } + catch + { + // Corrupt / unexpected record — mirror the index-rebuild tolerance. + } + + return false; + } + /// /// Deletes rows from the table that match the WHERE condition. /// Routes through storage engine with different semantics: diff --git a/src/SharpCoreDB/Interfaces/IStorage.cs b/src/SharpCoreDB/Interfaces/IStorage.cs index d617bece..fa8ee55f 100644 --- a/src/SharpCoreDB/Interfaces/IStorage.cs +++ b/src/SharpCoreDB/Interfaces/IStorage.cs @@ -247,7 +247,17 @@ void TombstoneRecords(string path, long[] offsets) while (position + 4 <= data.Length) { int length = BitConverter.ToInt32(data, (int)position); - if (length <= 0 || length > MaxRecordSizeLocal || position + 4 + length > data.Length) + if (length == 0) + { + // Valid zero-length record (e.g. an overflow-arena block for an empty TEXT/BLOB + // value). Yield an empty payload and keep scanning so later records/blocks are + // not silently dropped on reload. + yield return (position, []); + position += 4; + continue; + } + + if (length < 0 || length > MaxRecordSizeLocal || position + 4 + length > data.Length) { yield break; } diff --git a/src/SharpCoreDB/Services/Storage.Append.cs b/src/SharpCoreDB/Services/Storage.Append.cs index 29e05c87..b6a01898 100644 --- a/src/SharpCoreDB/Services/Storage.Append.cs +++ b/src/SharpCoreDB/Services/Storage.Append.cs @@ -1346,7 +1346,18 @@ private bool TryFlushBufferedOverwritesBatched(string path, Dictionary MaxRecordSize || position + 4 + length > fileLength) + if (length == 0) + { + // Valid empty record (a zero-length payload — e.g. an overflow-arena block written + // for an empty TEXT/BLOB value). There are no payload bytes to read, but the block + // occupies a real offset, so yield an empty payload and keep scanning. Treating it + // as the end-of-file would silently drop every later record/block on reload. + yield return (position, []); + position += 4; + continue; + } + + if (length > MaxRecordSize || position + 4 + length > fileLength) { yield break; // Invalid or incomplete record tail } diff --git a/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs b/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs index 55fe52ea..cf49e555 100644 --- a/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs +++ b/src/SharpCoreDB/Storage/Scdb/SingleFileOverflowArena.cs @@ -21,6 +21,10 @@ public sealed class SingleFileOverflowArena : IOverflowArena private readonly Dictionary _blocks = new(); private readonly Dictionary> _freeByLength = new(); private readonly Dictionary _contentIndex = new(System.StringComparer.Ordinal); + // Dead (freed) block slots that must be re-emitted as tombstone markers on every serialize so + // the byte stream stays aligned for the sequential deserializer. Populated on load (markers) + // and on Free; cleared when a slot is reused in place or the arena is compacted. + private readonly Dictionary _deadSlots = new(); private long _nextOffset; private int _blockReuses; @@ -101,6 +105,7 @@ public long Write(byte[] payload) _freeByLength.Remove(payload.Length); } + _deadSlots.Remove(offset); // reused in place: no longer a dead slot _blocks[offset] = payload; _contentIndex[contentKey] = offset; _blockReuses++; @@ -138,6 +143,7 @@ public void Free(long offset) } offsets.Add(offset); + _deadSlots[offset] = 4 + payload.Length; } /// Serializes all blocks (live and freed) as a contiguous [length][payload] stream. @@ -155,6 +161,18 @@ public byte[] Serialize() payload.CopyTo(buffer, (int)offset + 4); } + // Dead slots (freed blocks from this session or loaded from a previous one) are written as + // tombstone markers (negative length = total slot span to skip). Without them a dead region + // would be serialized as a zero-filled gap that misaligns the sequential deserializer and + // drops every later block on reload. + foreach (var (offset, slotSize) in _deadSlots) + { + if (offset >= 0 && offset + 4 <= buffer.Length) + { + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan((int)offset, 4), -slotSize); + } + } + return buffer; } @@ -175,7 +193,32 @@ public static SingleFileOverflowArena Deserialize(byte[]? data) while (position + 4 <= data.Length) { int length = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan((int)position, 4)); - if (length < 0 || position + 4 + length > data.Length) + if (length < 0) + { + // Tombstone marker: the negative value encodes the whole slot span to skip + // (freed overflow blocks are serialized as markers, not zero-filled gaps). + int slotSize = -length; + if (slotSize < 4 || position + slotSize > data.Length) + { + break; // truncated / corrupt + } + + // Track the dead slot so this flush re-emits the marker (a marker consumed on load + // would otherwise come back as a zero-filled gap on the next serialize). + arena._deadSlots[position] = slotSize; + int deadPayloadLength = slotSize - 4; + if (!arena._freeByLength.TryGetValue(deadPayloadLength, out var deadOffsets)) + { + deadOffsets = []; + arena._freeByLength[deadPayloadLength] = deadOffsets; + } + + deadOffsets.Add(position); + position += slotSize; + continue; + } + + if (position + 4 + length > data.Length) { break; // truncated / corrupt } @@ -213,6 +256,7 @@ public Dictionary Compact(IReadOnlyCollection activeOffsets) _blocks.Clear(); _freeByLength.Clear(); + _deadSlots.Clear(); _contentIndex.Clear(); foreach (var (newOffset, payload) in newBlocks) { diff --git a/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs b/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs index 9ae58e1f..d6e14960 100644 --- a/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs +++ b/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs @@ -129,4 +129,76 @@ public void ExplicitFixedWidthConfig_NoPrimaryKey_StillFixedWidth() } finally { (db as IDisposable)?.Dispose(); } } + + [Fact] + public void DefaultConfig_EmptyTextValue_DoesNotHideLaterRowsAfterReopen() + { + // Regression: the overflow arena stores an empty TEXT/BLOB value as a valid zero-length + // record, and reloading an arena file treated that record as the end of the file — every + // later row (and every later arena block of the same row) came back empty after a reopen. + IDatabase? db = null; + try + { + db = CreateDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, tag TEXT, payload TEXT)"); + Assert.True(IsFixedWidth(db, "t")); + + // Row 1 contains an empty TEXT value whose arena block precedes all later blocks. + db.ExecuteSQL("INSERT INTO t VALUES (1, '', 'first')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b2', 'second')"); + } + finally { (db as IDisposable)?.Dispose(); } + + try + { + db = CreateDb(); + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal(2, rows.Count); + Assert.Equal(1, Convert.ToInt32(rows[0]["id"])); + Assert.Equal(string.Empty, rows[0]["tag"]); + Assert.Equal("first", rows[0]["payload"]); + Assert.Equal(2, Convert.ToInt32(rows[1]["id"])); + Assert.Equal("b2", rows[1]["tag"]); + Assert.Equal("second", rows[1]["payload"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void DefaultConfig_UpdateToEmptyAndReopen_KeepsAllRowsIntact() + { + // Follow-up hardening on the same arena reload bug: a value that changes from empty to + // non-empty (and vice versa) writes/rewrites arena blocks that live after the first + // zero-length block. After reopen every block must still resolve. + IDatabase? db = null; + try + { + db = CreateDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, tag TEXT, payload TEXT, score INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'x', 'y', 10)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, '', '', 20)"); + + // Row 2: empty -> non-empty appends a payload after the row's earlier empty blocks. + // Row 1: non-empty -> empty exercises an empty payload write on a fresh update. + db.ExecuteSQL("UPDATE t SET payload = 'now' WHERE id = 2"); + db.ExecuteSQL("UPDATE t SET tag = '' WHERE id = 1"); + } + finally { (db as IDisposable)?.Dispose(); } + + try + { + db = CreateDb(); + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal(2, rows.Count); + Assert.Equal(1, Convert.ToInt32(rows[0]["id"])); + Assert.Equal(string.Empty, rows[0]["tag"]); + Assert.Equal("y", rows[0]["payload"]); + Assert.Equal(10, Convert.ToInt32(rows[0]["score"])); + Assert.Equal(2, Convert.ToInt32(rows[1]["id"])); + Assert.Equal(string.Empty, rows[1]["tag"]); + Assert.Equal("now", rows[1]["payload"]); + Assert.Equal(20, Convert.ToInt32(rows[1]["score"])); + } + finally { (db as IDisposable)?.Dispose(); } + } } diff --git a/tests/SharpCoreDB.Tests/ReopenRoundTripMatrixTests.cs b/tests/SharpCoreDB.Tests/ReopenRoundTripMatrixTests.cs new file mode 100644 index 00000000..a444bb5a --- /dev/null +++ b/tests/SharpCoreDB.Tests/ReopenRoundTripMatrixTests.cs @@ -0,0 +1,230 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + +/// +/// Round-trip matrix across storage variants (directory fixed-width default, directory legacy +/// variable-length, single-file JSON, single-file fixed-width) for values and operations that +/// exercise the length-prefixed overflow-arena layout: empty TEXT values interleaved with +/// non-empty ones, UPDATEs that flip a value between empty and non-empty, INSERTs and DELETEs +/// after such values, each followed by a full reopen + content verification. +/// Guards against "valid writer output treated as end-of-data by a reader" regressions +/// (e.g. zero-length arena records silently truncating an .ovf reload). +/// +public sealed class ReopenRoundTripMatrixTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + private readonly string _scdbPath; + + public ReopenRoundTripMatrixTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_RoundTrip_Dir_{Guid.NewGuid():N}"); + _scdbPath = Path.Combine(Path.GetTempPath(), $"SCDB_RoundTrip_Scdb_{Guid.NewGuid():N}.scdb"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + try { if (File.Exists(_scdbPath)) File.Delete(_scdbPath); } catch { } + } + + public static TheoryData Cases => new() + { + { "directory-fixedwidth-default", new DatabaseConfig() }, + { "directory-legacy-variable", new DatabaseConfig { AutoFixedWidthRecords = false } }, + { "singlefile-legacy-json", new DatabaseConfig() }, + { "singlefile-fixedwidth", new DatabaseConfig { FixedWidthRecordLayout = true } }, + }; + + [Theory] + [MemberData(nameof(Cases))] + public void RoundTrip_ReopenWithEmptyValues_KeepsDataIntact(string variant, DatabaseConfig config) + { + var path = variant.StartsWith("singlefile", StringComparison.Ordinal) ? _scdbPath : _dirPath; + RunScenario(variant, path, config); + } + + [Fact] + public void DirectoryLegacy_UpdateThenDelete_DoesNotResurrectAfterReopen() + { + // Regression for the legacy (variable-length) delete-after-update resurrection: an UPDATE + // appends a new version, DELETE only tombstones the newest version, and an older stale + // version used to win the reopen index rebuild. The delete-time purge must tombstone the + // remaining older versions of the deleted key too. + string path = Path.Combine(Path.GetTempPath(), $"SCDB_RoundTrip_LegacyDelete_{Guid.NewGuid():N}"); + var config = new DatabaseConfig { AutoFixedWidthRecords = false }; + try + { + WithDb(path, config, db => + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, tag TEXT, note TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'a', 'keep-1')"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b', 'keep-2')"); + db.ExecuteSQL("UPDATE t SET tag = 'a2' WHERE id = 1"); + db.ExecuteSQL("UPDATE t SET tag = 'a3' WHERE id = 1"); // two stale versions + }); + + // Delete the updated row in a fresh session, then verify it stays deleted after reopen. + WithDb(path, config, db => + { + AssertRowsDirect(db, [(1L, "a3", "keep-1"), (2L, "b", "keep-2")]); + db.ExecuteSQL("DELETE FROM t WHERE id = 1"); + }); + + WithDb(path, config, db => + { + AssertRowsDirect(db, [(2L, "b", "keep-2")]); + }); + } + finally + { + try { if (Directory.Exists(path)) Directory.Delete(path, true); } catch { } + } + } + + private static void AssertRowsDirect(IDatabase db, params (long Id, string Tag, string Note)[] expected) + { + var rows = db.ExecuteQuery("SELECT id, tag, note FROM t ORDER BY id", new Dictionary()); + Assert.True(rows.Count == expected.Length, $"expected {expected.Length} rows, got {rows.Count}"); + for (int i = 0; i < expected.Length; i++) + { + Assert.True(Convert.ToInt64(rows[i]["id"]) == expected[i].Id, "id mismatch"); + Assert.True(string.Equals(Convert.ToString(rows[i]["tag"]), expected[i].Tag, StringComparison.Ordinal), "tag mismatch"); + Assert.True(string.Equals(Convert.ToString(rows[i]["note"]), expected[i].Note, StringComparison.Ordinal), "note mismatch"); + } + } + + private void RunScenario(string variant, string path, DatabaseConfig config) + { + // Fresh storage for every variant run (theory cases may share a class instance). + if (Directory.Exists(path)) Directory.Delete(path, true); + if (File.Exists(path)) File.Delete(path); + + // Phase 0: seed rows where empty TEXT values are interleaved with non-empty ones. + WithDb(path, config, db => + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, tag TEXT, note TEXT, score INTEGER)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, '', 'first', 10)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'b2', '', 20)"); + db.ExecuteSQL("INSERT INTO t VALUES (3, 'c3', 'third', 30)"); + db.ExecuteSQL("INSERT INTO t VALUES (4, '', '', 40)"); + db.ExecuteSQL("INSERT INTO t VALUES (5, 'a considerably longer tag value for the overflow block', 'five', 50)"); + }); + + WithDb(path, config, db => + { + AssertTableEquals(db, variant, + Row(1, "", "first", 10), Row(2, "b2", "", 20), Row(3, "c3", "third", 30), + Row(4, "", "", 40), Row(5, "a considerably longer tag value for the overflow block", "five", 50)); + }); + + // Phase 1: UPDATEs that flip empty <-> non-empty (both directions) then reopen again. + WithDb(path, config, db => + { + db.ExecuteSQL("UPDATE t SET note = 'changed' WHERE id = 2"); + db.ExecuteSQL("UPDATE t SET tag = '' WHERE id = 3"); + db.ExecuteSQL("UPDATE t SET note = '' WHERE id = 5"); + db.ExecuteSQL("UPDATE t SET tag = '4-updated' WHERE id = 4"); + + // Same-session probe: the row cache must already reflect the updates. + AssertTableEquals(db, variant, + Row(1, "", "first", 10), Row(2, "b2", "changed", 20), Row(3, "", "third", 30), + Row(4, "4-updated", "", 40), Row(5, "a considerably longer tag value for the overflow block", "", 50)); + }); + + WithDb(path, config, db => + { + AssertTableEquals(db, variant, + Row(1, "", "first", 10), Row(2, "b2", "changed", 20), Row(3, "", "third", 30), + Row(4, "4-updated", "", 40), Row(5, "a considerably longer tag value for the overflow block", "", 50)); + }); + + // Phase 2: delete the previously-updated row (id 3), insert more rows after empty values, + // update again, and reopen a third time. The legacy variable-length variant exercises the + // delete-after-update purge (older stale versions of a deleted key must not resurrect). + WithDb(path, config, db => + { + db.ExecuteSQL("DELETE FROM t WHERE id = 3"); + db.ExecuteSQL("INSERT INTO t VALUES (6, '', 'six', 60)"); + db.ExecuteSQL("INSERT INTO t VALUES (7, 'g7', '', 70)"); + db.ExecuteSQL("UPDATE t SET tag = 's1' WHERE id = 1"); + + // Same-session probe: the row cache must already reflect the DELETE. + AssertTableEquals(db, variant, + Row(1, "s1", "first", 10), Row(2, "b2", "changed", 20), + Row(4, "4-updated", "", 40), Row(5, "a considerably longer tag value for the overflow block", "", 50), + Row(6, "", "six", 60), Row(7, "g7", "", 70)); + }); + + WithDb(path, config, db => + { + AssertTableEquals(db, variant, + Row(1, "s1", "first", 10), Row(2, "b2", "changed", 20), + Row(4, "4-updated", "", 40), Row(5, "a considerably longer tag value for the overflow block", "", 50), + Row(6, "", "six", 60), Row(7, "g7", "", 70)); + }); + } + + private void WithDb(string path, DatabaseConfig config, Action action) + { + var db = Open(path, config); + try + { + action(db); + db.Flush(); + db.ForceSave(); + } + finally { (db as IDisposable)?.Dispose(); } + } + + private IDatabase Open(string path, DatabaseConfig config) + => _factory.Create(path, "pw", isReadOnly: false, config: config); + + private static (long Id, string Tag, string Note, long Score) Row(long id, string tag, string note, long score) + => (id, tag, note, score); + + private static void AssertTableEquals( + IDatabase db, + string variant, + params (long Id, string Tag, string Note, long Score)[] expected) + { + var rows = db.ExecuteQuery("SELECT id, tag, note, score FROM t ORDER BY id", new Dictionary()); + string actualSummary = string.Join( + " | ", + rows.Select(r => $"id={Convert.ToString(r["id"])},tag=[{Convert.ToString(r["tag"])}],note=[{Convert.ToString(r["note"])}],score={Convert.ToString(r["score"])}")); + Assert.True( + rows.Count == expected.Length, + $"[{variant}] row count mismatch: expected {expected.Length}, actual {rows.Count}. rows: {actualSummary}"); + + for (int i = 0; i < expected.Length; i++) + { + var actual = rows[i]; + var want = expected[i]; + Assert.True( + Convert.ToInt64(actual["id"]) == want.Id, + $"[{variant}] row {i}: id mismatch (expected {want.Id}, actual {Convert.ToString(actual["id"])}). rows: {actualSummary}"); + Assert.True( + string.Equals(Convert.ToString(actual["tag"]), want.Tag, StringComparison.Ordinal), + $"[{variant}] row {i}: tag mismatch (expected [{want.Tag}], actual [{Convert.ToString(actual["tag"])}]). rows: {actualSummary}"); + Assert.True( + string.Equals(Convert.ToString(actual["note"]), want.Note, StringComparison.Ordinal), + $"[{variant}] row {i}: note mismatch (expected [{want.Note}], actual [{Convert.ToString(actual["note"])}]). rows: {actualSummary}"); + Assert.True( + Convert.ToInt64(actual["score"]) == want.Score, + $"[{variant}] row {i}: score mismatch (expected {want.Score}, actual {Convert.ToString(actual["score"])}). rows: {actualSummary}"); + } + } +} \ No newline at end of file