diff --git a/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs b/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs index 10d9c112..87930e08 100644 --- a/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs +++ b/src/SharpCoreDB/DataStructures/FixedWidthCodec.cs @@ -29,31 +29,54 @@ public static byte[] SerializeRow( for (int i = 0; i < columns.Count; i++) { - var slot = span.Slice(layout.Offsets[i], layout.SlotSizes[i]); var value = row.TryGetValue(columns[i], out var v) ? v : DBNull.Value; + WriteSlot(span.Slice(layout.Offsets[i], layout.SlotSizes[i]), layout.IsVariable[i], types[i], value, arena); + } - if (layout.IsVariable[i]) - { - if (value == null || value == DBNull.Value) - { - slot[0] = 0; - BinaryPrimitives.WriteInt32LittleEndian(slot[1..], 0); - } - else - { - var payload = Table.EncodeVariablePayload(types[i], value); - var offset = arena.Write(payload); - slot[0] = 1; - BinaryPrimitives.WriteInt32LittleEndian(slot[1..], (int)offset); - } - } - else + return buffer; + } + + /// Serializes a column-ordered object[] row (full table column order) with the fixed-width codec. + public static byte[] SerializeRow( + object[] row, + IReadOnlyList types, + FixedWidthRecordLayout layout, + IOverflowArena arena) + { + var buffer = new byte[layout.FixedSize]; + var span = buffer.AsSpan(); + + for (int i = 0; i < row.Length && i < types.Count; i++) + { + WriteSlot(span.Slice(layout.Offsets[i], layout.SlotSizes[i]), layout.IsVariable[i], types[i], row[i], arena); + } + + return buffer; + } + + /// + /// Writes one column value into its fixed-width slot: variable-length types store an out-of-line + /// arena block reference, fixed-size types store their payload inline. + /// + private static void WriteSlot(Span slot, bool isVariable, DataType type, object? value, IOverflowArena arena) + { + if (isVariable) + { + if (value == null || value == DBNull.Value) { - _ = Table.WriteTypedValueToSpan(slot, value, types[i]); + slot[0] = 0; + BinaryPrimitives.WriteInt32LittleEndian(slot[1..], 0); + return; } + + var payload = Table.EncodeVariablePayload(type, value); + var offset = arena.Write(payload); + slot[0] = 1; + BinaryPrimitives.WriteInt32LittleEndian(slot[1..], (int)offset); + return; } - return buffer; + _ = Table.WriteTypedValueToSpan(slot, value, type); } /// Deserializes a fixed-width record into a row dictionary (variable values ← arena). diff --git a/src/SharpCoreDB/DataStructures/Table.Serialization.cs b/src/SharpCoreDB/DataStructures/Table.Serialization.cs index 5925b4bf..448139dc 100644 --- a/src/SharpCoreDB/DataStructures/Table.Serialization.cs +++ b/src/SharpCoreDB/DataStructures/Table.Serialization.cs @@ -512,6 +512,10 @@ internal static object DecodeVariablePayload(DataType type, byte[] payload) private byte[] SerializeRowFixedWidth(Dictionary row) => FixedWidthCodec.SerializeRow(row, Columns, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena()); + /// Serializes a column-ordered row (full table column order) with the fixed-width layout. + private byte[] SerializeRowFixedWidth(object[] row) + => FixedWidthCodec.SerializeRow(row, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena()); + /// Deserializes a fixed-width record into a row dictionary (variable values read from the overflow arena). private Dictionary DeserializeRowFixedWidth(ReadOnlySpan data) => FixedWidthCodec.DeserializeRow(data, Columns, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena()); @@ -645,10 +649,19 @@ private byte[] SerializeRowExact(Dictionary row) /// /// Column-ordered array variant of /// for the dedicated batch-INSERT path (no dictionary allocation / lookups). + /// Fixed-width tables must use the fixed-width codec here too — writing legacy variable-length + /// records into a fixed-width-flagged table would make every subsequent scan misread them. /// [MethodImpl(MethodImplOptions.AggressiveOptimization)] private byte[] SerializeRowExact(object[] values) { + // Fixed-width record layout (out-of-line overflow): constant-size record, variable values + // stored in the table's overflow arena. + if (_fixedWidthRecords) + { + return SerializeRowFixedWidth(values); + } + byte[] buffer = new byte[ComputeExactRowSize(values)]; int bytesWritten = WriteRowGeneric(buffer.AsSpan(), values); return bytesWritten == buffer.Length diff --git a/src/SharpCoreDB/DatabaseConfig.cs b/src/SharpCoreDB/DatabaseConfig.cs index 3ca71f85..34babc4d 100644 --- a/src/SharpCoreDB/DatabaseConfig.cs +++ b/src/SharpCoreDB/DatabaseConfig.cs @@ -30,6 +30,17 @@ public class DatabaseConfig /// public bool FixedWidthRecordLayout { get; init; } = false; + /// + /// Gets a value indicating whether NEW columnar tables that declare a PRIMARY KEY default to + /// the fixed-width record layout with out-of-line overflow (see + /// ), so UPDATE/DELETE by key are in-place overwrites. + /// This only affects tables created after the setting is in effect — existing tables keep + /// their persisted record format until an explicit opt-in () + /// or MigrateTableToFixedWidth converts them. Set to to keep + /// creating every new table with the legacy variable-length records. + /// + public bool AutoFixedWidthRecords { get; init; } = true; + /// /// Gets a value indicating whether SQLite integer type affinity is used for DDL type mapping. /// When (opt-in), INTEGER maps to (Int64), diff --git a/src/SharpCoreDB/Services/SqlParser.DDL.cs b/src/SharpCoreDB/Services/SqlParser.DDL.cs index a49e6d72..14e1ff30 100644 --- a/src/SharpCoreDB/Services/SqlParser.DDL.cs +++ b/src/SharpCoreDB/Services/SqlParser.DDL.cs @@ -386,6 +386,24 @@ private void ExecuteCreateTable(string sql, string[] parts, IWAL? wal) }); table.Name = tableName; + + // B7+: new columnar tables with an explicitly declared PRIMARY KEY default to the fixed-width + // record layout (DatabaseConfig.AutoFixedWidthRecords, default true) so keyed UPDATE/DELETE + // become in-place overwrites. Tables without a declared PK (which get the hidden _rowid + // fallback) keep the legacy variable-length records. The per-table flag is persisted in + // metadata, so existing tables are never rewritten and FixedWidthRecordLayout stays the + // explicit force/auto-migrate switch. Applies to directory-mode (Columnar) tables only — + // PageBased tables and the single-file (.scdb) layout are untouched by this default. + if (table is Table fixedWidthCandidate && + primaryKeyIndex >= 0 && + !hasInternalRowId && + storageMode == StorageMode.Columnar && + !fixedWidthCandidate.IsFixedWidthRecords && + (this.config?.AutoFixedWidthRecords ?? true)) + { + fixedWidthCandidate.IsFixedWidthRecords = true; + } + this.tables[tableName] = table; // ✅ NEW: Wire database reference so Table.Insert can call SetLastInsertRowId / RecordBatchInsert diff --git a/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs b/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs new file mode 100644 index 00000000..9ae58e1f --- /dev/null +++ b/tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs @@ -0,0 +1,132 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.Tests; + +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using System; +using System.IO; +using Xunit; + +/// +/// B7+: DatabaseConfig.AutoFixedWidthRecords (default true) — NEW columnar tables that declare a +/// PRIMARY KEY are created with the fixed-width record layout even when +/// is left false. Existing tables are never +/// rewritten by this default; the persisted per-table format stays authoritative on reopen. +/// +public sealed class DirectoryFixedWidthDefaultTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _dirPath; + + public DirectoryFixedWidthDefaultTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_FixedWidthDefault_{Guid.NewGuid():N}"); + } + + public void Dispose() + { + try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } + } + + private IDatabase CreateDb(DatabaseConfig? config = null) + => _factory.Create(_dirPath, "pw", isReadOnly: false, config: config ?? new DatabaseConfig()); + + private static bool IsFixedWidth(IDatabase db, string tableName) + => db.TryGetTable(tableName, out var t) && t.IsFixedWidthRecords; + + private string DatPath(string table) => Path.Combine(_dirPath, $"{table}.dat"); + + [Fact] + public void DefaultConfig_PkTable_IsCreatedFixedWidth() + { + var db = CreateDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL)"); + Assert.True(IsFixedWidth(db, "t")); + Assert.True(db.TryGetTable("t", out var t) && t.PrimaryKeyIndex >= 0); + + db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha', 1.5)"); + db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta', 2.5)"); + + // Growing variable-column UPDATE is an in-place overwrite: the .dat never grows. + long sizeBefore = new FileInfo(DatPath("t")).Length; + db.ExecuteSQL("UPDATE t SET name = 'a considerably longer name value' WHERE id = 2"); + Assert.Equal(sizeBefore, new FileInfo(DatPath("t")).Length); + + var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY id"); + Assert.Equal(2, rows.Count); + Assert.Equal("a considerably longer name value", rows[1]["name"]); + Assert.Equal(2.5, Convert.ToDouble(rows[1]["score"])); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void DefaultConfig_PkTable_FormatPersistsAcrossReopen() + { + IDatabase? db = null; + try + { + db = CreateDb(); + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + db.ExecuteSQL("INSERT INTO t VALUES (1, 'persisted')"); + Assert.True(IsFixedWidth(db, "t")); + } + finally { (db as IDisposable)?.Dispose(); } + + // Reopen WITHOUT any fixed-width config: the persisted per-table flag is authoritative. + db = null; + try + { + db = CreateDb(); + Assert.True(IsFixedWidth(db, "t")); + var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1"); + Assert.Single(row); + Assert.Equal("persisted", row[0]["name"]); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void DefaultConfig_NoPrimaryKey_StaysLegacyVariableLength() + { + var db = CreateDb(); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, score REAL)"); + Assert.False(IsFixedWidth(db, "t")); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void AutoFixedWidthOptOut_PkTable_StaysLegacyVariableLength() + { + var db = CreateDb(new DatabaseConfig { AutoFixedWidthRecords = false }); + try + { + db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + Assert.False(IsFixedWidth(db, "t")); + } + finally { (db as IDisposable)?.Dispose(); } + } + + [Fact] + public void ExplicitFixedWidthConfig_NoPrimaryKey_StillFixedWidth() + { + var db = CreateDb(new DatabaseConfig { FixedWidthRecordLayout = true }); + try + { + db.ExecuteSQL("CREATE TABLE t (name TEXT, score REAL)"); + Assert.True(IsFixedWidth(db, "t")); + } + finally { (db as IDisposable)?.Dispose(); } + } +} diff --git a/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs b/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs index 401f8e01..f4c6accf 100644 --- a/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs +++ b/tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs @@ -35,7 +35,12 @@ public void Dispose() try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { } } - private IDatabase CreateLegacyDb() => _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig()); + private IDatabase CreateLegacyDb() => _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig + { + // AutoFixedWidthRecords defaults to true since B7+; this fixture deliberately simulates a + // pre-fixed-width (1.x / variable-length records) database, so it opts out. + AutoFixedWidthRecords = false, + }); private IDatabase CreateFixedWidthDb() => _factory.Create( _dirPath, "pw", isReadOnly: false, config: new DatabaseConfig { FixedWidthRecordLayout = true }); diff --git a/tests/SharpCoreDB.Tests/KnownIssuesFixTests.cs b/tests/SharpCoreDB.Tests/KnownIssuesFixTests.cs index ac1ada13..6c98d115 100644 --- a/tests/SharpCoreDB.Tests/KnownIssuesFixTests.cs +++ b/tests/SharpCoreDB.Tests/KnownIssuesFixTests.cs @@ -151,9 +151,15 @@ public void Issue1_NoEncryptMode_RemainsPlaintext_BackwardCompatible() db.Flush(); // The payload is still readable as plaintext on disk (guarantee for NoEncrypt users). + // Fixed-width tables keep TEXT values out-of-line in the per-table .ovf arena, so scan + // both the record file and the overflow arena for the plaintext guarantee. var tableFile = Path.Combine(dbPath, "plain.dat"); Assert.True(File.Exists(tableFile)); - var content = File.ReadAllText(tableFile, Encoding.UTF8); + var dataFiles = new[] { tableFile, Path.ChangeExtension(tableFile, ".ovf") } + .Where(File.Exists) + .ToArray(); + Assert.NotEmpty(dataFiles); + var content = string.Join("\n", dataFiles.Select(f => File.ReadAllText(f, Encoding.UTF8))); Assert.Contains(payload, content); // And the engine still reads it back correctly. diff --git a/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs b/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs index 5ce9ef8d..5e917cc3 100644 --- a/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs +++ b/tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs @@ -101,7 +101,9 @@ public void SqlUpdate_Parameterized_FixedWidth_OverwritesInPlace() [Fact] public void SqlUpdate_VariableWidth_GrowsWhenStoredLengthChanges_StillCorrect() { - var db = _factory.Create(_dirPath, "pw"); + // This test asserts the LEGACY variable-length layout's append-on-grow semantics, so the + // table must not be auto-promoted to the fixed-width layout (AutoFixedWidthRecords default). + var db = _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig { AutoFixedWidthRecords = false }); try { db.ExecuteSQL("CREATE TABLE vw (id INTEGER PRIMARY KEY, name TEXT, val INTEGER)");