From d8201f516e6748bed3084a7cba4de8a48e70507b Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Fri, 4 Sep 2026 13:31:09 +0200 Subject: [PATCH] docs+test: upgrade/downgrade compatibility policy and format-compat regressions Add docs/manual/upgrade-and-downgrade.md recording the compatibility matrix: new versions read legacy (variable-length, pre-marker) databases; downgrading below the marker-introducing release is NOT supported for databases that contain commit-time tombstone markers (negative length prefixes). Includes the recommended read-only-first upgrade order and notes the planned cross-version CI job. FormatCompatPolicyTests locks in the forward-compatibility guarantees: (1) a legacy variable-length file reads back and accepts current-version marker writes across reopens, (2) commit-time tombstone markers stay stable across reopen cycles while rows appended afterwards coexist. --- docs/CHANGELOG.md | 9 ++ docs/manual/upgrade-and-downgrade.md | 43 ++++++ .../FormatCompatPolicyTests.cs | 137 ++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 docs/manual/upgrade-and-downgrade.md create mode 100644 tests/SharpCoreDB.Tests/FormatCompatPolicyTests.cs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3d0c7356..1645e886 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Hardening +- **Upgrade/downgrade policy documented + format-compat regression tests** - new + `docs/manual/upgrade-and-downgrade.md` records the compatibility matrix: reading legacy + (variable-length, pre-marker) databases with the current version is supported; opening a database + that already contains commit-time tombstone markers with a version that predates them is **not** + supported (negative length-prefix markers), and the recommended read-only-first upgrade order. + `FormatCompatPolicyTests` locks in the two forward-compatibility guarantees: legacy files read + back and accept marker writes across reopens, and commit-time tombstone markers stay stable + across reopen cycles while rows appended afterwards coexist. + - **Auto engine selection no longer lands on PageBased (production hardening)** - with default configuration (`StorageEngineType.Auto` + `WorkloadHint.General`) `GetOptimalStorageEngine` returned PageBased, which is not yet OLTP-ready (measured UPDATE ~26K ops/s vs ~245K ops/s on the diff --git a/docs/manual/upgrade-and-downgrade.md b/docs/manual/upgrade-and-downgrade.md new file mode 100644 index 00000000..f3568ea8 --- /dev/null +++ b/docs/manual/upgrade-and-downgrade.md @@ -0,0 +1,43 @@ +# Upgrade & downgrade compatibility + +Status: **backward compatible** (a new version opens databases written by older versions). +Downgrade below this line is **not supported** after the database contains certain on-disk markers. + +## Compatibility matrix + +| Scenario | Supported? | Notes | +|---|---|---| +| Open a legacy (pre-fixed-width, variable-length) database with the current version | ✅ Yes | Plaintext + encrypted-header legacy files are read natively; a table without `IsFixedWidthRecords` stays variable-length (the persisted flag is authoritative). | +| Open a current fixed-width Columnar database with the current version after reopen cycles | ✅ Yes | Covered by `FormatCompatPolicyTests` / `FixedWidthMigrationTests`; per-table flag persisted. | +| Migrate a legacy table to fixed-width | ✅ Yes (opt-in) | `DatabaseConfig.FixedWidthRecordLayout = true` triggers `MigrateToFixedWidth()` on open (never on read-only opens). | +| Open a database written by the **current** version (contains commit-time tombstone markers) with an **older** version that predates markers | ❌ **Not supported** | Deleted rows are stored as **negative length-prefix markers** (introduced with commit-time tombstones). An older binary that only knows positive length prefixes cannot skip these records and must not be pointed at such a file. | +| Open a fixed-width table with a version that only understands variable-length records | ❌ Not supported | Downgrade requires a migration/export; none is shipped. | + +## Downgrade boundary + +The on-disk markers (negative length prefixes, written at COMMIT for transactional deletes) make a +database **forward-compatible only**. If you must keep the ability to downgrade, either: + +1. Keep a separate pre-upgrade copy of the database, or +2. Do not upgrade software that writes tombstones in place on a database you need to open again + with the old software, or +3. Export/re-import data (SQL dump) instead of copying `.dat`/metadata files across versions. + +Recommended upgrade order: +1. Back up the database directory (or single-file database). +2. Open it with the new version once **read-only** first (this never rewrites data). +3. Open read-write and run the normal DML regression/verification. +4. Only then let the new version write to the file. + +## Verification + +- In-repo: `FormatCompatPolicyTests`, `FixedWidthMigrationTests`, + `DefaultEngineSelectionTests`, and the full suite (currently 1768+ tests) cover legacy reads, + opt-in migration, marker durability across reopen cycles, and the default fast-path engine. +- Planned (CI): a true **cross-version** job that writes a database with a pinned older commit and + reads it with `master` (requires an old-binary generator; tracked as follow-up). + +## Changelog + +See `docs/CHANGELOG.md` → `[Unreleased]` → **Hardening** for the marker/downgrade notes that +accompanied the tombstone work (PRs #367/#368) and this policy document. diff --git a/tests/SharpCoreDB.Tests/FormatCompatPolicyTests.cs b/tests/SharpCoreDB.Tests/FormatCompatPolicyTests.cs new file mode 100644 index 00000000..78d37544 --- /dev/null +++ b/tests/SharpCoreDB.Tests/FormatCompatPolicyTests.cs @@ -0,0 +1,137 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + +/// +/// Encoding of the compatibility policy in docs/manual/upgrade-and-downgrade.md: +/// (1) files written in the legacy variable-length format (the pre-marker / pre-fixed-width +/// representation) must read back and keep working across reopen cycles, and (2) commit-time +/// tombstone markers written by the current version must stay stable across repeated reopen cycles +/// and coexist with rows appended afterwards. These are the forward-compatibility guarantees that +/// make downgrade the only unsupported direction. +/// +public sealed class FormatCompatPolicyTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public FormatCompatPolicyTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_FormatCompat_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + private static void InsertRange(IDatabase db, int from, int to) + { + var stmts = new List(to - from + 1); + for (int i = from; i <= to; i++) + { + stmts.Add($"INSERT INTO docs VALUES ({i}, 'user{i}', {i * 0.5})"); + } + + db.ExecuteBatchSQL(stmts); + db.Flush(); + } + + private static void DeleteRange(IDatabase db, int from, int to) + { + var stmts = new List(to - from + 1); + for (int i = from; i <= to; i++) + { + stmts.Add($"DELETE FROM docs WHERE id = {i}"); + } + + db.ExecuteBatchSQL(stmts); + db.Flush(); + } + + [Fact] + public void LegacyVariableLengthFile_ReadsBackAndKeepsWorking_AcrossReopens() + { + // A legacy variable-length database (no fixed-width layout) is byte-compatible with the + // pre-fixed-width / pre-marker format. It must open, round-trip and then accept marker + // writes from the current version. + IDatabase? db = _factory.Create(_dirPath, "pw", isReadOnly: false, + config: new DatabaseConfig { NoEncryptMode = true, AutoFixedWidthRecords = false, FixedWidthRecordLayout = false }); + try + { + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + InsertRange(db, 1, 300); + Assert.Equal(300, db.ExecuteQuery("SELECT id FROM docs").Count); + } + finally { (db as IDisposable)?.Dispose(); } + + // First reopen = "old file read by new version" (no markers written yet). + db = _factory.Create(_dirPath, "pw", isReadOnly: false, + config: new DatabaseConfig { NoEncryptMode = true, AutoFixedWidthRecords = false, FixedWidthRecordLayout = false }); + try + { + Assert.Equal(300, db.ExecuteQuery("SELECT id FROM docs").Count); + DeleteRange(db, 1, 40); + Assert.Equal(260, db.ExecuteQuery("SELECT id FROM docs").Count); + } + finally { (db as IDisposable)?.Dispose(); } + + // Final reopen: the legacy rows plus current-version tombstone markers coexist. + db = _factory.Create(_dirPath, "pw", isReadOnly: false, + config: new DatabaseConfig { NoEncryptMode = true, AutoFixedWidthRecords = false, FixedWidthRecordLayout = false }); + try + { + Assert.Equal(260, db.ExecuteQuery("SELECT id FROM docs").Count); + Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE id = 10")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 41")); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void CommitTimeTombstoneMarkers_StableAcrossReopenCycles_AndCoexistWithAppends() + { + IDatabase? db = _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig()); + try + { + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + InsertRange(db, 1, 1000); + DeleteRange(db, 1, 500); + Assert.Equal(500, db.ExecuteQuery("SELECT id FROM docs").Count); + } + finally { (db as IDisposable)?.Dispose(); } + + db = _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig()); + try + { + Assert.Equal(500, db.ExecuteQuery("SELECT id FROM docs").Count); + InsertRange(db, 1001, 1200); // appends after the marker region + Assert.Equal(700, db.ExecuteQuery("SELECT id FROM docs").Count); + DeleteRange(db, 1001, 1100); + } + finally { (db as IDisposable)?.Dispose(); } + + db = _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig()); + try + { + Assert.Equal(600, db.ExecuteQuery("SELECT id FROM docs").Count); + Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE id = 1")); + Assert.Empty(db.ExecuteQuery("SELECT id FROM docs WHERE id = 1050")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 1101")); + Assert.Single(db.ExecuteQuery("SELECT id FROM docs WHERE id = 1200")); + } + finally { (db as IDisposable)?.Dispose(); } + } +}