diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f15414c0..45317920 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,6 +27,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 runtime `AreRecordsEncrypted` gate (so default-config plaintext databases benefit without `NoEncryptMode`). Fixed-size hash-indexed SET columns are re-pointed with one lock per index (`HashIndex.RemoveBatchKeys`/`AddBatchKeys`). +- **Sequential ascending-PK batch resolution for legacy DELETE (Fase B: legacy fast paths)** - + `DeleteMultipleKeys` on a legacy (variable-length, plaintext, non-fixed-width) Columnar table now + resolves a strictly-ascending INTEGER-PK literal batch with a single sequential decode pass that + starts at the first target's position and early-exits once every target matched — instead of one + B-tree search + row decode per target. Strictly gated: page-based/fixed-width/encrypted layouts, + non-PK or non-ascending keys, sparse batches spanning more than 2 MB and physically unordered + files (detected by a monotonicity pre-pass) all fall back to the existing per-row path, so the + result is identical. Two regression tests cover an ascending batch over an **unordered** physical + layout (must delete exactly the requested keys across reopen) and re-validate the existing + legacy prefix delete; full suite 1766 tests, 0 failed. Legacy `--pk` DELETE stays within noise on + this harness; the gate is groundwork for the wide-row legacy arms. - **Buffered in-place UPDATE overwrites are now flushed per storage page (C6, Fase B)** - the UPDATE commit path buffered one record per row (B7) and flushed each with two pwrites (length prefix + payload), so ~10K-row UPDATEs were dominated by per-row write syscalls diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index eaefa638..dc4ba92e 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -3102,8 +3102,16 @@ internal void DeleteMultipleKeys(List<(string Column, string Literal)> condition var recordsToDelete = new List<(long storagePosition, Dictionary row)>(); - foreach (var (col, literal) in conditions) + // Sequential ascending-INTEGER-PK batch resolution (legacy variable-length layout): one + // decode pass with an early exit instead of one B-tree search + row decode per target. + if (TryResolvePkBatchSequentially(conditions, wholeFile, deleteKeyColumns, out var sequentialRecords)) { + recordsToDelete = sequentialRecords; + } + else + { + foreach (var (col, literal) in conditions) + { string value = UnquoteSqlLiteral(literal); // Issue #7 PK fast path (structured: no WHERE-string re-parse). @@ -3223,6 +3231,7 @@ _btreeManager is null && recordsToDelete.Add((storageRef, row)); } } + } } if (recordsToDelete.Count == 0) return; @@ -3235,6 +3244,180 @@ _btreeManager is null && } } + /// + /// Sequential ascending-INTEGER-PK batch resolution for the legacy (variable-length, plaintext, + /// Columnar) DELETE path. When every condition is a strictly-ascending literal on an INTEGER PK + /// and the physical byte range between the first and last target is compact (≤ 2 MB), all + /// targets are resolved with a single decode pass starting at the first target's position + /// instead of one B-tree search + row decode per target (the per-target cost of the issue-#7 + /// fast path). The pass early-exits as soon as every target matched while the file has proven + /// physically PK-ordered; if an out-of-order row is seen the scan continues to EOF so the + /// result stays identical to the per-row search path. Handled batches return true with the + /// matched records (possibly empty); any shape/type/layout deviation returns false so the + /// caller falls back to the existing per-condition resolution. + /// + private bool TryResolvePkBatchSequentially( + List<(string Column, string Literal)> conditions, + byte[]? wholeFile, + int[] deleteKeyColumns, + out List<(long storagePosition, Dictionary row)> records) + { + records = []; + + if (StorageMode == StorageMode.PageBased || wholeFile is null || _fixedWidthRecords) + { + return false; + } + + if (this.PrimaryKeyIndex < 0 || this.ColumnTypes[this.PrimaryKeyIndex] != DataType.Integer || conditions.Count < 32) + { + return false; + } + + // All conditions must target the PK column with strictly ascending, duplicate-free integer + // literals (numeric ordering equals the INTEGER PK key ordering). + var targets = new long[conditions.Count]; + for (int i = 0; i < conditions.Count; i++) + { + var (col, literal) = conditions[i]; + if (!string.Equals(col, this.Columns[this.PrimaryKeyIndex], StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var value = UnquoteSqlLiteral(literal).Trim(); + if (!long.TryParse(value, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var parsed)) + { + return false; + } + + if (i > 0 && parsed <= targets[i - 1]) + { + return false; + } + + targets[i] = parsed; + } + + // Bound the worst-case scan to a compact physical region: locate the first and last target + // through the PK B-tree and require their byte distance to stay below 2 MB. Sparse or + // full-file-spanning target sets stay on the per-row search path. + var firstSearch = this.Index.Search(targets[0].ToString(CultureInfo.InvariantCulture)); + var lastSearch = this.Index.Search(targets[^1].ToString(CultureInfo.InvariantCulture)); + if (!firstSearch.Found || !lastSearch.Found || lastSearch.Value - firstSearch.Value > 2 * 1024 * 1024) + { + return false; + } + + string pkCol = this.Columns[this.PrimaryKeyIndex]; + + // Pre-pass: verify the file is physically PK-ordered from offset 0 up to the first + // target's position. Only then may the main pass skip those leading rows — an unordered + // file could otherwise place a later target *before* the first target's position, which a + // forward-only scan would never see. Any disorder falls back to the per-row path. + { + var pkWantedPre = new[] { this.PrimaryKeyIndex }; + long walk = 0; + long prev = long.MinValue; + while (walk + 4 <= wholeFile.Length && walk < firstSearch.Value) + { + int len = BinaryPrimitives.ReadInt32LittleEndian(wholeFile.AsSpan((int)walk, 4)); + if (len > 0 && walk + 4 + len <= wholeFile.Length) + { + var r = DeserializeDeleteKeyRow(wholeFile.AsSpan((int)walk + 4, len), pkWantedPre); + if (r != null && r.TryGetValue(pkCol, out var v) && v is not null && v is not DBNull) + { + long pk = Convert.ToInt64(v, CultureInfo.InvariantCulture); + if (pk < prev) + { + return false; // physically unordered file -> per-row resolution + } + + prev = pk; + } + + walk += 4 + len; + } + else + { + if (len == 0) + { + return false; + } + + // Tombstone marker: the negative value already encodes the whole slot span + // (4-byte prefix + payload), so skipping by |len| lands exactly on the next + // record's prefix. + walk += Math.Abs(len); + } + } + } + + var remaining = new HashSet(targets); // set-based matching keeps unordered files correct + int matchedCount = 0; + long previousPk = long.MinValue; + bool ordered = true; + var pkWanted = new[] { this.PrimaryKeyIndex }; + + long position = firstSearch.Value; + while (position + 4 <= wholeFile.Length) + { + int recordLength = BinaryPrimitives.ReadInt32LittleEndian(wholeFile.AsSpan((int)position, 4)); + int advance; + if (recordLength > 0 && position + 4 + recordLength <= wholeFile.Length) + { + advance = 4 + recordLength; + + var pkRow = DeserializeDeleteKeyRow(wholeFile.AsSpan((int)position + 4, recordLength), pkWanted); + if (pkRow != null && pkRow.TryGetValue(pkCol, out var pkValue) && pkValue is not null && pkValue is not DBNull) + { + long pk = Convert.ToInt64(pkValue, CultureInfo.InvariantCulture); + if (pk < previousPk) + { + ordered = false; + } + + previousPk = pk; + + if (remaining.Remove(pk)) + { + matchedCount++; + var row = DeserializeDeleteKeyRowFromFile(wholeFile, position, deleteKeyColumns); + if (row != null) + { + records.Add((position, row)); + } + } + } + } + else + { + // Tombstoned record (negative prefix) or truncated tail. The marker encodes the whole + // slot span (4-byte prefix + payload) as its magnitude, so |recordLength| skips + // exactly to the next record's prefix; a zero-length prefix cannot be walked safely. + if (recordLength == 0) + { + return false; + } + + advance = Math.Abs(recordLength); + if (advance <= 4) + { + return false; + } + } + + if (ordered && matchedCount == targets.Length) + { + break; // early exit: monotonically ordered file, every target resolved + } + + position += advance; + } + + return true; + } + private bool AllPkLiteralConditions(List<(string Column, string Literal)> conditions) { var pkCol = this.Columns[this.PrimaryKeyIndex]; diff --git a/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs b/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs index f010f900..e1c89563 100644 --- a/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs +++ b/tests/SharpCoreDB.Tests/FixedWidthBulkDeleteTests.cs @@ -311,6 +311,72 @@ public void CanonicalBatchDelete_EngagesStructuredPath() finally { (db as IDisposable)?.Dispose(); } } + [Fact] + public void LegacyUnorderedLayout_AscendingPkBatchDelete_StaysCorrectAcrossReopen() + { + // Sequential ascending-PK batch resolution may not assume physical row order: when rows + // were inserted in a shuffled order the batch must still delete exactly the requested keys + // (the sequential scan detects the disorder, keeps scanning to EOF and matches by set). + IDatabase? db = _factory.Create(_dirPath, "pw", isReadOnly: false, + config: new DatabaseConfig { NoEncryptMode = true, AutoFixedWidthRecords = false }); + try + { + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + db.ExecuteSQL("CREATE INDEX idx_docs_name ON docs(name)"); + + // Shuffle 1..2000 with a fixed seed so the physical file order is not PK-ordered. + var ids = new List(2000); + for (int i = 1; i <= 2000; i++) ids.Add(i); + var rng = new Random(20260904); + for (int i = ids.Count - 1; i > 0; i--) + { + int j = rng.Next(i + 1); + (ids[i], ids[j]) = (ids[j], ids[i]); + } + + var stmts = new List(2000); + foreach (int id in ids) + { + stmts.Add(string.Format(CultureInfo.InvariantCulture, + "INSERT INTO docs VALUES ({0}, 'user{0}', {1})", id, id * 0.5)); + } + + db.ExecuteBatchSQL(stmts); + db.Flush(); + Assert.Equal(2000, db.ExecuteQuery("SELECT id FROM docs").Count); + + // Ascending pk batch over a physically unordered file: keys 101..140. + var dels = new List(40); + for (int i = 101; i <= 140; i++) + { + dels.Add(string.Format(CultureInfo.InvariantCulture, "DELETE FROM docs WHERE id = {0}", i)); + } + + db.ExecuteBatchSQL(dels); + db.Flush(); + + Assert.Equal(1960, db.ExecuteQuery("SELECT id FROM docs").Count); + Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE id = 120")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 100")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 141")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE name = 'user150'")); + } + finally { (db as IDisposable)?.Dispose(); } + + // Reopen: the deleted rows stay gone, live rows stay reachable. + db = _factory.Create(_dirPath, "pw", isReadOnly: false, + config: new DatabaseConfig { NoEncryptMode = true, AutoFixedWidthRecords = false }); + try + { + Assert.Equal(1960, db.ExecuteQuery("SELECT id FROM docs").Count); + Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE id = 101")); + Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE id = 140")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 100")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 141")); + } + finally { (db as IDisposable)?.Dispose(); } + } + [Fact] public void BatchDelete_CommitTimeTombstones_SurviveReopenWithoutExplicitFlush() {