perf: default new columnar PK tables to the fixed-width record layout

src/SharpCoreDB/DataStructures/FixedWidthCodec.cs

@@ -29,31 +29,54 @@ public static byte[] SerializeRow(
 
         for (int i = 0; i < columns.Count; i++)
         {
-            var slot = span.Slice(layout.Offsets[i], layout.SlotSizes[i]);
             var value = row.TryGetValue(columns[i], out var v) ? v : DBNull.Value;
+            WriteSlot(span.Slice(layout.Offsets[i], layout.SlotSizes[i]), layout.IsVariable[i], types[i], value, arena);
+        }
 
-            if (layout.IsVariable[i])
-            {
-                if (value == null || value == DBNull.Value)
-                {
-                    slot[0] = 0;
-                    BinaryPrimitives.WriteInt32LittleEndian(slot[1..], 0);
-                }
-                else
-                {
-                    var payload = Table.EncodeVariablePayload(types[i], value);
-                    var offset = arena.Write(payload);
-                    slot[0] = 1;
-                    BinaryPrimitives.WriteInt32LittleEndian(slot[1..], (int)offset);
-                }
-            }
-            else
+        return buffer;
+    }
+
+    /// <summary>Serializes a column-ordered object[] row (full table column order) with the fixed-width codec.</summary>
+    public static byte[] SerializeRow(
+        object[] row,
+        IReadOnlyList<DataType> types,
+        FixedWidthRecordLayout layout,
+        IOverflowArena arena)
+    {
+        var buffer = new byte[layout.FixedSize];
+        var span = buffer.AsSpan();
+
+        for (int i = 0; i < row.Length && i < types.Count; i++)
+        {
+            WriteSlot(span.Slice(layout.Offsets[i], layout.SlotSizes[i]), layout.IsVariable[i], types[i], row[i], arena);
+        }
+
+        return buffer;
+    }
+
+    /// <summary>
+    /// Writes one column value into its fixed-width slot: variable-length types store an out-of-line
+    /// arena block reference, fixed-size types store their payload inline.
+    /// </summary>
+    private static void WriteSlot(Span<byte> slot, bool isVariable, DataType type, object? value, IOverflowArena arena)
+    {
+        if (isVariable)
+        {
+            if (value == null || value == DBNull.Value)
             {
-                _ = Table.WriteTypedValueToSpan(slot, value, types[i]);
+                slot[0] = 0;
+                BinaryPrimitives.WriteInt32LittleEndian(slot[1..], 0);
+                return;
             }
+
+            var payload = Table.EncodeVariablePayload(type, value);
+            var offset = arena.Write(payload);
+            slot[0] = 1;
+            BinaryPrimitives.WriteInt32LittleEndian(slot[1..], (int)offset);
+            return;
         }
 
-        return buffer;
+        _ = Table.WriteTypedValueToSpan(slot, value, type);
     }
 
     /// <summary>Deserializes a fixed-width record into a row dictionary (variable values ← arena).</summary>

src/SharpCoreDB/DataStructures/Table.Serialization.cs

@@ -512,6 +512,10 @@ internal static object DecodeVariablePayload(DataType type, byte[] payload)
     private byte[] SerializeRowFixedWidth(Dictionary<string, object> row)
         => FixedWidthCodec.SerializeRow(row, Columns, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena());
 
+    /// <summary>Serializes a column-ordered row (full table column order) with the fixed-width layout.</summary>
+    private byte[] SerializeRowFixedWidth(object[] row)
+        => FixedWidthCodec.SerializeRow(row, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena());
+
     /// <summary>Deserializes a fixed-width record into a row dictionary (variable values read from the overflow arena).</summary>
     private Dictionary<string, object> DeserializeRowFixedWidth(ReadOnlySpan<byte> data)
         => FixedWidthCodec.DeserializeRow(data, Columns, ColumnTypes, GetFixedWidthLayout(), GetOverflowArena());
@@ -645,10 +649,19 @@ private byte[] SerializeRowExact(Dictionary<string, object> row)
     /// <summary>
     /// Column-ordered array variant of <see cref="SerializeRowExact(Dictionary{string,object})"/>
     /// for the dedicated batch-INSERT path (no dictionary allocation / lookups).
+    /// Fixed-width tables must use the fixed-width codec here too — writing legacy variable-length
+    /// records into a fixed-width-flagged table would make every subsequent scan misread them.
     /// </summary>
     [MethodImpl(MethodImplOptions.AggressiveOptimization)]
     private byte[] SerializeRowExact(object[] values)
     {
+        // Fixed-width record layout (out-of-line overflow): constant-size record, variable values
+        // stored in the table's overflow arena.
+        if (_fixedWidthRecords)
+        {
+            return SerializeRowFixedWidth(values);
+        }
+
         byte[] buffer = new byte[ComputeExactRowSize(values)];
         int bytesWritten = WriteRowGeneric(buffer.AsSpan(), values);
         return bytesWritten == buffer.Length

src/SharpCoreDB/DatabaseConfig.cs

@@ -30,6 +30,17 @@ public class DatabaseConfig
     /// </summary>
     public bool FixedWidthRecordLayout { get; init; } = false;
 
+    /// <summary>
+    /// Gets a value indicating whether NEW columnar tables that declare a PRIMARY KEY default to
+    /// the fixed-width record layout with out-of-line overflow (see
+    /// <see cref="FixedWidthRecordLayout"/>), so UPDATE/DELETE by key are in-place overwrites.
+    /// This only affects tables created after the setting is in effect — existing tables keep
+    /// their persisted record format until an explicit opt-in (<see cref="FixedWidthRecordLayout"/>)
+    /// or <c>MigrateTableToFixedWidth</c> converts them. Set to <see langword="false"/> to keep
+    /// creating every new table with the legacy variable-length records.
+    /// </summary>
+    public bool AutoFixedWidthRecords { get; init; } = true;
+
     /// <summary>
     /// Gets a value indicating whether SQLite integer type affinity is used for DDL type mapping.
     /// When <see langword="true"/> (opt-in), <c>INTEGER</c> maps to <see cref="DataType.Long"/> (Int64),

src/SharpCoreDB/Services/SqlParser.DDL.cs

@@ -386,6 +386,24 @@ private void ExecuteCreateTable(string sql, string[] parts, IWAL? wal)
         });
 
         table.Name = tableName;
+
+        // B7+: new columnar tables with an explicitly declared PRIMARY KEY default to the fixed-width
+        // record layout (DatabaseConfig.AutoFixedWidthRecords, default true) so keyed UPDATE/DELETE
+        // become in-place overwrites. Tables without a declared PK (which get the hidden _rowid
+        // fallback) keep the legacy variable-length records. The per-table flag is persisted in
+        // metadata, so existing tables are never rewritten and FixedWidthRecordLayout stays the
+        // explicit force/auto-migrate switch. Applies to directory-mode (Columnar) tables only —
+        // PageBased tables and the single-file (.scdb) layout are untouched by this default.
+        if (table is Table fixedWidthCandidate &&
+            primaryKeyIndex >= 0 &&
+            !hasInternalRowId &&
+            storageMode == StorageMode.Columnar &&
+            !fixedWidthCandidate.IsFixedWidthRecords &&
+            (this.config?.AutoFixedWidthRecords ?? true))
+        {
+            fixedWidthCandidate.IsFixedWidthRecords = true;
+        }
+
         this.tables[tableName] = table;
 
         // ✅ NEW: Wire database reference so Table.Insert can call SetLastInsertRowId / RecordBatchInsert

tests/SharpCoreDB.Tests/DirectoryFixedWidthDefaultTests.cs

@@ -0,0 +1,132 @@
+// <copyright file="DirectoryFixedWidthDefaultTests.cs" company="MPCoreDeveloper">
+// Copyright (c) 2026 MPCoreDeveloper. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for full license information.
+// </copyright>
+namespace SharpCoreDB.Tests;
+
+using Microsoft.Extensions.DependencyInjection;
+using SharpCoreDB.Interfaces;
+using System;
+using System.IO;
+using Xunit;
+
+/// <summary>
+/// B7+: DatabaseConfig.AutoFixedWidthRecords (default true) — NEW columnar tables that declare a
+/// PRIMARY KEY are created with the fixed-width record layout even when
+/// <see cref="DatabaseConfig.FixedWidthRecordLayout"/> is left false. Existing tables are never
+/// rewritten by this default; the persisted per-table format stays authoritative on reopen.
+/// </summary>
+public sealed class DirectoryFixedWidthDefaultTests : IDisposable
+{
+    private readonly DatabaseFactory _factory;
+    private readonly string _dirPath;
+
+    public DirectoryFixedWidthDefaultTests()
+    {
+        var services = new ServiceCollection();
+        services.AddSharpCoreDB();
+        _factory = services.BuildServiceProvider().GetRequiredService<DatabaseFactory>();
+        _dirPath = Path.Combine(Path.GetTempPath(), $"SCDB_FixedWidthDefault_{Guid.NewGuid():N}");
+    }
+
+    public void Dispose()
+    {
+        try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { }
+    }
+
+    private IDatabase CreateDb(DatabaseConfig? config = null)
+        => _factory.Create(_dirPath, "pw", isReadOnly: false, config: config ?? new DatabaseConfig());
+
+    private static bool IsFixedWidth(IDatabase db, string tableName)
+        => db.TryGetTable(tableName, out var t) && t.IsFixedWidthRecords;
+
+    private string DatPath(string table) => Path.Combine(_dirPath, $"{table}.dat");
+
+    [Fact]
+    public void DefaultConfig_PkTable_IsCreatedFixedWidth()
+    {
+        var db = CreateDb();
+        try
+        {
+            db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT, score REAL)");
+            Assert.True(IsFixedWidth(db, "t"));
+            Assert.True(db.TryGetTable("t", out var t) && t.PrimaryKeyIndex >= 0);
+
+            db.ExecuteSQL("INSERT INTO t VALUES (1, 'alpha', 1.5)");
+            db.ExecuteSQL("INSERT INTO t VALUES (2, 'beta', 2.5)");
+
+            // Growing variable-column UPDATE is an in-place overwrite: the .dat never grows.
+            long sizeBefore = new FileInfo(DatPath("t")).Length;
+            db.ExecuteSQL("UPDATE t SET name = 'a considerably longer name value' WHERE id = 2");
+            Assert.Equal(sizeBefore, new FileInfo(DatPath("t")).Length);
+
+            var rows = db.ExecuteQuery("SELECT * FROM t ORDER BY id");
+            Assert.Equal(2, rows.Count);
+            Assert.Equal("a considerably longer name value", rows[1]["name"]);
+            Assert.Equal(2.5, Convert.ToDouble(rows[1]["score"]));
+        }
+        finally { (db as IDisposable)?.Dispose(); }
+    }
+
+    [Fact]
+    public void DefaultConfig_PkTable_FormatPersistsAcrossReopen()
+    {
+        IDatabase? db = null;
+        try
+        {
+            db = CreateDb();
+            db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)");
+            db.ExecuteSQL("INSERT INTO t VALUES (1, 'persisted')");
+            Assert.True(IsFixedWidth(db, "t"));
+        }
+        finally { (db as IDisposable)?.Dispose(); }
+
+        // Reopen WITHOUT any fixed-width config: the persisted per-table flag is authoritative.
+        db = null;
+        try
+        {
+            db = CreateDb();
+            Assert.True(IsFixedWidth(db, "t"));
+            var row = db.ExecuteQuery("SELECT * FROM t WHERE id = 1");
+            Assert.Single(row);
+            Assert.Equal("persisted", row[0]["name"]);
+        }
+        finally { (db as IDisposable)?.Dispose(); }
+    }
+
+    [Fact]
+    public void DefaultConfig_NoPrimaryKey_StaysLegacyVariableLength()
+    {
+        var db = CreateDb();
+        try
+        {
+            db.ExecuteSQL("CREATE TABLE t (name TEXT, score REAL)");
+            Assert.False(IsFixedWidth(db, "t"));
+        }
+        finally { (db as IDisposable)?.Dispose(); }
+    }
+
+    [Fact]
+    public void AutoFixedWidthOptOut_PkTable_StaysLegacyVariableLength()
+    {
+        var db = CreateDb(new DatabaseConfig { AutoFixedWidthRecords = false });
+        try
+        {
+            db.ExecuteSQL("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)");
+            Assert.False(IsFixedWidth(db, "t"));
+        }
+        finally { (db as IDisposable)?.Dispose(); }
+    }
+
+    [Fact]
+    public void ExplicitFixedWidthConfig_NoPrimaryKey_StillFixedWidth()
+    {
+        var db = CreateDb(new DatabaseConfig { FixedWidthRecordLayout = true });
+        try
+        {
+            db.ExecuteSQL("CREATE TABLE t (name TEXT, score REAL)");
+            Assert.True(IsFixedWidth(db, "t"));
+        }
+        finally { (db as IDisposable)?.Dispose(); }
+    }
+}

tests/SharpCoreDB.Tests/FixedWidthMigrationTests.cs

@@ -35,7 +35,12 @@ public void Dispose()
         try { if (Directory.Exists(_dirPath)) Directory.Delete(_dirPath, true); } catch { }
     }
 
-    private IDatabase CreateLegacyDb() => _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig());
+    private IDatabase CreateLegacyDb() => _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig
+    {
+        // AutoFixedWidthRecords defaults to true since B7+; this fixture deliberately simulates a
+        // pre-fixed-width (1.x / variable-length records) database, so it opts out.
+        AutoFixedWidthRecords = false,
+    });
 
     private IDatabase CreateFixedWidthDb() => _factory.Create(
         _dirPath, "pw", isReadOnly: false, config: new DatabaseConfig { FixedWidthRecordLayout = true });

tests/SharpCoreDB.Tests/KnownIssuesFixTests.cs

@@ -151,9 +151,15 @@ public void Issue1_NoEncryptMode_RemainsPlaintext_BackwardCompatible()
             db.Flush();
 
             // The payload is still readable as plaintext on disk (guarantee for NoEncrypt users).
+            // Fixed-width tables keep TEXT values out-of-line in the per-table .ovf arena, so scan
+            // both the record file and the overflow arena for the plaintext guarantee.
             var tableFile = Path.Combine(dbPath, "plain.dat");
             Assert.True(File.Exists(tableFile));
-            var content = File.ReadAllText(tableFile, Encoding.UTF8);
+            var dataFiles = new[] { tableFile, Path.ChangeExtension(tableFile, ".ovf") }
+                .Where(File.Exists)
+                .ToArray();
+            Assert.NotEmpty(dataFiles);
+            var content = string.Join("\n", dataFiles.Select(f => File.ReadAllText(f, Encoding.UTF8)));
             Assert.Contains(payload, content);
 
             // And the engine still reads it back correctly.

tests/SharpCoreDB.Tests/SqlInPlaceUpdateTests.cs

@@ -101,7 +101,9 @@ public void SqlUpdate_Parameterized_FixedWidth_OverwritesInPlace()
     [Fact]
     public void SqlUpdate_VariableWidth_GrowsWhenStoredLengthChanges_StillCorrect()
     {
-        var db = _factory.Create(_dirPath, "pw");
+        // This test asserts the LEGACY variable-length layout's append-on-grow semantics, so the
+        // table must not be auto-promoted to the fixed-width layout (AutoFixedWidthRecords default).
+        var db = _factory.Create(_dirPath, "pw", isReadOnly: false, config: new DatabaseConfig { AutoFixedWidthRecords = false });
         try
         {
             db.ExecuteSQL("CREATE TABLE vw (id INTEGER PRIMARY KEY, name TEXT, val INTEGER)");