From 91c875882e2c5336843799a12db324657806093e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 11:07:43 +0000 Subject: [PATCH 1/5] wip(driver-sql): dialect-correct JSON column type --- packages/drivers/driver-sql/src/sql-driver.ts | 96 ++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 76cd790654..3c26e19a3b 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -11083,7 +11083,26 @@ export class SqlDriver implements IDataDriver { return honoured; } - /** Map an introspected SQLite column to a knex builder for the rebuilt table. */ + /** + * Map an introspected SQLite column to a knex builder for the rebuilt table. + * + * ⚠️ [#12738] The `json` arm below is DELIBERATELY not routed through + * {@link jsonColumn}, and this is the one place in the driver where that + * reads like an oversight. The input here is a column's EXISTING declared + * type, read back from `pragma_table_info` — not a field's metadata — and + * this method's whole contract is to hand back the column the table already + * had. A legacy column declared `json` therefore stays `json` through a + * rebuild, which is exactly the migration shape #12738 ruled: new columns + * only, existing columns keep their declared type. + * + * Routing it through the new emitter would make an UNRELATED drift rebuild + * (a relaxed NOT NULL, a dropped column) silently re-declare the column as + * `text` and, because the rebuild's `insert … select` copies every row + * through the new affinity, rewrite the stored bytes of live data — a + * conversion nobody asked for, ridden in on an unrelated op. Columns created + * AFTER #12738 arrive here declared `text` already and take the `text` arm + * below, so the rebuild is stable in both eras without special-casing. + */ protected buildRebuiltColumn(t: Knex.CreateTableBuilder, c: IntrospectedColumn): any { if (c.name === 'id') return t.string('id').primary(); const ty = (c.type || 'text').toLowerCase(); @@ -15078,6 +15097,77 @@ export class SqlDriver implements IDataDriver { }); } + /** + * [#12738] Declare a JSON-valued column as the type that is semantically + * correct **for this dialect** — the single seam both JSON routes in + * {@link createColumn} go through. + * + * ## The ruling + * + * Maintainer, 2026-08-28 (recorded on #12738): each dialect declares the + * semantically correct JSON column type. Server dialects keep their NATIVE + * JSON type; the SQLite family declares `text`. Two directions were on the + * table and BOTH were refused — unifying everything onto `json` (it hands + * SQLite the affinity trap below) and changing the shared arm for all + * dialects (it would strip Postgres's json/jsonb and MySQL's JSON, which are + * correct and stay). + * + * ## Why SQLite is the exception, measured rather than argued + * + * SQLite has no JSON type. It derives a column's AFFINITY from substrings of + * the declared type name, and `json` contains none of the markers (`INT`, + * `CHAR`/`CLOB`/`TEXT`, `BLOB`, `REAL`/`FLOA`/`DOUB`), so it falls through to + * **NUMERIC** and converts number-like input on the way in. Measured through + * raw SQL on a column declared `json`: a bare `'0123'` is stored as the + * INTEGER `123`. `text` takes TEXT affinity and converts nothing — and TEXT + * is what SQLite's own JSON1 functions operate on, so this is the type the + * database actually means. + * + * That exposure is the one #12380 had to defeat on this driver's SQLite half + * by making the `Field.json` codec injective. This change removes it at the + * ROOT for new columns instead of encoding around it. + * + * ⛔ Measured, not assumed — knex's own compilers, read and then run: + * `ColumnCompiler.prototype.json = 'text'` is the BASE, and the sqlite3 + * dialect is the one that overrides it (`ColumnCompiler_SQLite3.prototype + * .json = 'json'`). Compiled DDL for `t.json('v')`: `better-sqlite3` and + * `sqlite3` → `` `v` json ``; `pg` → `"v" json`; `mysql2` → `` `v` json ``. + * `t.text('v')` under sqlite → `` `v` text ``. So this branch is not a + * preference between two spellings — it is the one dialect where knex's + * answer names a type the engine does not have. + * + * ## Why `this.isSqlite` is the right discriminator + * + * It covers every SQLite face this monorepo ships, and the coverage was + * verified rather than presumed: + * + * - plain {@link SqlDriver} on `sqlite3` / `sqlite` / `better-sqlite3` + * (`SQLITE_EMIT_CLIENTS`); + * - `TursoDriver` in all three transport modes — local, embedded-replica + * and remote all configure `client: 'better-sqlite3'`; + * - `SqliteWasmDriver`, which passes a knex Client CONSTRUCTOR (so the + * string match answers `''`) and therefore **overrides `isSqlite` to + * `true`** for exactly this reason. A `config.client` string test written + * inline here would have silently missed it. + * + * ## What this does NOT do + * + * It changes what NEW columns are declared as. It never rewrites an existing + * column: the schema sync is additive, and a legacy `json` column keeps its + * declared type, its NUMERIC affinity, and the #12380 encoding that defeats + * it. Nothing starts reporting drift over the difference either — the base- + * type finding in `schema-drift.ts` is gated on + * `multiValueColumnTypeIsLoadBearing(dialect)`, which is `postgres || mysql` + * — and no read path consults the physical type: `isJsonField` answers from + * METADATA, so decoding is identical on both spellings. + * + * @see turso-json-column-type-asymmetry.test.ts — the #12586 pin, INVERTED by + * this change per its own header (it now pins the convergence). + */ + protected jsonColumn(table: Knex.CreateTableBuilder, name: string): any { + return this.isSqlite ? table.text(name) : table.json(name); + } + protected createColumn( table: Knex.CreateTableBuilder, name: string, @@ -15101,7 +15191,7 @@ export class SqlDriver implements IDataDriver { if (field.reference_to !== undefined) refuseRejectedReferenceAlias(name); if (field.multiple) { - table.json(name); + this.jsonColumn(table, name); return; } @@ -15388,7 +15478,7 @@ export class SqlDriver implements IDataDriver { // // A type that genuinely wants the bound belongs in the string-family // case above, named — never acquired by falling through to here. - col = JSON_COLUMN_TYPES.has(type) ? table.json(name) : table.string(name); + col = JSON_COLUMN_TYPES.has(type) ? this.jsonColumn(table, name) : table.string(name); } if (col) { From 655079f3901a4d47495b1da025c6fee1e229bb46 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 11:27:53 +0000 Subject: [PATCH 2/5] test(driver-turso): invert the #12586 asymmetry pin onto the convergence --- packages/drivers/driver-sql/probe3.mjs | 4 + .../turso-json-column-type-asymmetry.test.ts | 252 +++++++++++------- 2 files changed, 163 insertions(+), 93 deletions(-) create mode 100644 packages/drivers/driver-sql/probe3.mjs diff --git a/packages/drivers/driver-sql/probe3.mjs b/packages/drivers/driver-sql/probe3.mjs new file mode 100644 index 0000000000..9d468d4a9b --- /dev/null +++ b/packages/drivers/driver-sql/probe3.mjs @@ -0,0 +1,4 @@ +import knexLib from 'knex'; +const k = knexLib({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); +await k.schema.createTable('p', (t) => { t.text('a'); t.json('b'); t.string('c'); }); +console.log('sqlite_master:', (await k.raw("select sql from sqlite_master where name='p'"))[0].sql); diff --git a/packages/drivers/driver-turso/src/turso-json-column-type-asymmetry.test.ts b/packages/drivers/driver-turso/src/turso-json-column-type-asymmetry.test.ts index 49fe5deb47..5c6406239d 100644 --- a/packages/drivers/driver-turso/src/turso-json-column-type-asymmetry.test.ts +++ b/packages/drivers/driver-turso/src/turso-json-column-type-asymmetry.test.ts @@ -1,81 +1,86 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. /** - * [#12586] **The declared physical column type for a `Field.json` is DIFFERENT - * on this driver's two transports, and that is intended.** This file is the - * record of that decision and the pin that fails if either side moves. + * [#12738] **INVERTED.** This file was created by #12586 to pin an ASYMMETRY — + * a declared `Field.json` became a `json` column on the local transport and a + * `TEXT` column on the remote one. That asymmetry is **gone**, deliberately, + * and this file now pins the convergence and the affinity that replaced it. * * ```text - * declared field local (TursoDriver extends SqlDriver, knex) remote (RemoteTransport) - * --------------- ------------------------------------------- ------------------------ - * { type: 'json' } json TEXT - * { multiple: true} json TEXT + * declared field local (TursoDriver extends SqlDriver, knex) remote (RemoteTransport) + * ---------------- ------------------------------------------- ------------------------ + * { type: 'json' } TEXT (was: json) TEXT (unchanged) + * { multiple: true} TEXT (was: json) TEXT (unchanged) * ``` * - * ## Why a reader should not "fix" this + * ⚠️ Those are CATALOG readings, and the local one is worth a sentence because + * it surprises. knex writes `` `v_json` text `` (lower case) into the stored + * `CREATE TABLE` — measured in `sqlite_master` — and `pragma_table_info` + * reports it back as `TEXT`. So the convergence is not merely onto one affinity + * class, which is all the ruling required: the two transports turn out to + * declare the **byte-identical** type string. The affinity assertion below is + * still written as the primary one, because affinity is the property that + * matters and an exact-string pin would go red over a cosmetic re-spelling. * - * The two halves are written by two different authors of DDL. Local mode - * inherits `SqlDriver.createColumn`, which hands the column to knex's - * `table.json(name)`; remote mode does not go through knex at all and spells - * its own SQLite types in `RemoteTransport.mapFieldTypeToSQL`. **Every** column - * type in this driver is spelled differently by the two — `varchar(255)` vs - * `TEXT`, `float` vs `REAL`, `boolean` vs `INTEGER` — and for all of those the - * difference is cosmetic, because SQLite derives a column's *affinity* from - * substrings of the declared type name and the two spellings land in the same - * affinity class. + * ## Why the pin was inverted rather than re-baselined * - * `json` is the one that does not. It contains none of SQLite's affinity - * markers (`INT`, `CHAR`/`CLOB`/`TEXT`, `BLOB`, `REAL`/`FLOA`/`DOUB`), so it - * takes **NUMERIC** affinity and converts a number-like value on the way in; - * `TEXT` takes **TEXT** affinity and converts nothing. So this is the one - * column where the two transports disagree about *what a value becomes on - * disk*, and that is a different question from the spelling. + * The #12586 header instructed exactly this: *"When convergence is genuinely + * taken on, delete or invert that pin as part of the change; never edit its + * expectations to match new output."* The expectations below are not the old + * ones nudged — every one of them states the OPPOSITE fact, on purpose, and the + * old fact is quoted beside it so the change is legible. * - * ## Why it is safe today — and why "safe" is not the same as "the same" + * ## The ruling that moved it * - * Both transports round-trip every `VALUE_ROUNDTRIP_CASES` value faithfully - * (`turso-value-roundtrip-conformance.test.ts`, both halves green). They get - * there by different routes: #12380 made the local `Field.json` codec - * injective, so what reaches the NUMERIC-affinity column is an encoded form - * that affinity has nothing to convert; the remote transport's own - * `serializeValue`/`mapRows` reach the same answer over a column where no - * conversion was available in the first place. + * Maintainer, 2026-08-28 (recorded on #12738): each dialect declares the + * semantically correct JSON column type. Server dialects keep their native JSON + * types; the **SQLite family declares `text`** — what SQLite's JSON1 functions + * actually operate on, and what `RemoteTransport.mapFieldTypeToSQL` had spelled + * all along. So the two transports converge onto the side **without** the + * affinity trap: it is the LOCAL half that moved. * - * ⚠️ Equal answers, unequal bytes. Measured on the fixture below: of the cases - * in the shared table, the two transports store **different storage classes** - * for `n_int` and `n_real` — an INTEGER/REAL cell locally, a TEXT cell - * remotely — while both `find()` calls answer `123` and `1.5`. That is the - * #11535 class in its quiet phase: two paths that agree on every visible - * answer while standing on different ground, so the next codec change has no - * reason to be kind to both. PR #12585's ablation is the loud phase — restoring - * the pre-#12380 SQLite `json` branch broke the two transports by DIFFERENT - * counts, diverging on `s_0123`, because only the local column's NUMERIC - * affinity was there to destroy a bare `'0123'`. + * ⛔ The other direction was refused by name. Converging onto `json` would have + * given the remote transport NUMERIC affinity — `json` contains none of + * SQLite's affinity markers (`INT`, `CHAR`/`CLOB`/`TEXT`, `BLOB`, + * `REAL`/`FLOA`/`DOUB`) — i.e. the measured `'0123'` → `123` exposure that + * #12380 had to defeat on the local half, imported into the half that never had + * it. * - * ## ⛔ What to do when this file goes red + * ## The instrument is AFFINITY-LEVEL, and that is a requirement, not a taste + * + * #12738's own content is a negative result: the shared `VALUE_ROUNDTRIP` + * case-set is **green on both sides of this decision** (88 passed, unchanged + * under convergence), because both transports round-trip every value faithfully + * either way. It therefore cannot adjudicate the direction, and using it as + * evidence is an error. What this file asserts instead, on **both** transports: + * + * - the declared type each side records in the catalog; + * - `typeof(col)` — the **storage class** SQLite actually put in each cell, + * which is the observable consequence of affinity; + * - a raw-SQL bare `'0123'`, which must survive as **text** on both sides. + * + * The last one is the whole decision in one statement: before this change that + * probe returned the integer `123` locally and the string `'0123'` remotely. * - * It goes red when one transport's emitted type for `Field.json` moves and the - * other's does not, or when they are made to converge. Both are real decisions - * and this pin exists so they are taken **knowingly**: + * ## What did NOT change * - * - **Converging them** (#12586 disposition 1) changes what new columns are - * physically declared as, and is deliberately NOT done here: it needs "why - * did remote choose `TEXT`?" answered first, which is un-measured. File it, - * measure it, and then **delete or invert this pin** as part of that change. - * - ⛔ **Never patch the expectations green.** Editing the tables below to - * match whatever the code now emits turns this file from a record of a - * decision into a mirror of the code, which is the one thing it cannot be - * and still be worth running. + * Only what NEW columns are declared as. A column created before this change + * keeps its `json` declaration, keeps NUMERIC affinity, and keeps being + * defended by #12380's injective codec — `SqlDriver.buildRebuiltColumn` still + * re-declares an introspected `json` column as `json`, so not even a drift + * rebuild converts one. Nothing on the read path consults the physical type + * (`isJsonField` answers from metadata), so decoding is identical either way. * - * ## The instrument + * ## ⛔ What to do when this file goes red * - * The fixture is `VALUE_ROUNDTRIP_FIELDS` / `VALUE_ROUNDTRIP_CASES` — the same - * shared table `turso-value-roundtrip-conformance.test.ts` drives, for the - * reason #12586 named it: the case-set already covers both transports, so the - * pin and the round-trip conformance answer questions about the same columns - * and the same written values, and cannot drift onto different fixtures. + * It goes red when the two transports stop agreeing, or when the type they + * agree on stops being TEXT-affinity. Both are real decisions. ⛔ Never patch + * the expectations green — editing the tables below to match whatever the code + * now emits turns this file from a record of a decision into a mirror of the + * code, which is the one thing it cannot be and still be worth running. * - * @see https://github.com/objectstack-ai/objectstack/issues/12586 (this pin) + * @see https://github.com/objectstack-ai/objectstack/issues/12738 (this inversion) + * @see https://github.com/objectstack-ai/objectstack/issues/12586 (the pin this replaces) * @see https://github.com/objectstack-ai/objectstack/issues/12380 (the injective local codec) * @see https://github.com/objectstack-ai/objectstack/issues/11535 (the class) */ @@ -89,7 +94,7 @@ import { import { TursoDriver } from './turso-driver.js'; import { makeLibsqlSqliteStub, type LibsqlSqliteStub } from './libsql-sqlite-stub.testkit.js'; -const TABLE = 'asymmetry_12586'; +const TABLE = 'convergence_12738'; const OBJECT = { name: TABLE, fields: { ...VALUE_ROUNDTRIP_FIELDS } }; /** The fixture's own declared columns — the builtins are a different question. */ @@ -128,7 +133,7 @@ const storageClassOf = (rows: any[]): Map => { return out; }; -describe('[#12586] driver-turso — the Field.json column type is deliberately asymmetric across the two transports', () => { +describe('[#12738] driver-turso — the two transports declare ONE physical column type for Field.json', () => { let local: TursoDriver; let remote: TursoDriver; let stub: LibsqlSqliteStub; @@ -168,18 +173,23 @@ describe('[#12586] driver-turso — the Field.json column type is deliberately a // ─── §1 What each side declares ────────────────────────────────────────── - it('local mode declares the fixture through knex — `json` for both JSON routes', () => { + it('local mode declares the fixture through knex — `text` for both JSON routes (#12738; was `json`)', () => { + // INVERTED. Before #12738 both JSON routes read `json` here — knex's + // sqlite3 dialect overrides the base compiler's `text` with `json` + // (`ColumnCompiler_SQLite3.prototype.json = 'json'`), naming a type SQLite + // does not have. `SqlDriver.jsonColumn` now routes the SQLite family to + // `table.text()` instead. Every non-JSON row below is untouched. expect(localTypes).toEqual({ label: 'varchar(255)', - v_json: 'json', - v_multi: 'json', + v_json: 'TEXT', + v_multi: 'TEXT', v_string: 'varchar(255)', v_number: 'float', v_boolean: 'boolean', }); }); - it("remote mode declares the fixture through RemoteTransport.mapFieldTypeToSQL — `TEXT` for both JSON routes", () => { + it("remote mode declares the fixture through RemoteTransport.mapFieldTypeToSQL — `TEXT` for both JSON routes, UNCHANGED by #12738", () => { expect(remoteTypes).toEqual({ label: 'TEXT', v_json: 'TEXT', @@ -190,25 +200,39 @@ describe('[#12586] driver-turso — the Field.json column type is deliberately a }); }); - // ─── §2 The asymmetry, named ───────────────────────────────────────────── + // ─── §2 The convergence, named ─────────────────────────────────────────── - it('THE ASYMMETRY: one declared Field.json, two physical column types — intended, not an oversight', () => { - expect( - [localTypes.v_json, remoteTypes.v_json], - 'The local/remote pair for a declared `Field.json`. If this is red because the two now AGREE, ' + - 'that is #12586 disposition 1 (convergence) — a decision this pin exists to make deliberate. ' + - 'Delete or invert this file as part of that change; do NOT edit the expectation to match.', - ).toEqual(['json', 'TEXT']); - expect( - [localTypes.v_multi, remoteTypes.v_multi], - 'The same asymmetry reached by the OTHER route: `multiple: true` short-circuits before the ' + - 'type switch on both sides, so it can move independently of `type: "json"`.', - ).toEqual(['json', 'TEXT']); + it('THE CONVERGENCE: one declared Field.json, ONE affinity class on both transports (#12738)', () => { + // INVERTED from `['json', 'TEXT']`. The pair is compared by AFFINITY, not + // by spelling: the two authors of DDL still write different words for every + // column (`varchar(255)`/`TEXT`, `float`/`REAL`, `boolean`/`INTEGER`), and + // that was never the question. What #12586 recorded — and what #12738 fixed + // — is that `json` was the one spelling that landed in a DIFFERENT affinity + // class than its opposite number. Both now contain `TEXT`, so both take + // TEXT affinity by SQLite's own rule. + for (const [route, note] of [ + ['v_json', 'the `type: "json"` route'], + ['v_multi', 'the `multiple: true` route, which short-circuits before the type switch on both sides'], + ] as const) { + const pair = [localTypes[route], remoteTypes[route]]; + expect( + pair.map((t) => /char|clob|text/i.test(t)), + `${route} — ${note}. Both transports must declare a TEXT-affinity type. If this is red ` + + 'because they DISAGREE again, one side moved: that is a decision, not a broken ' + + 'expectation. Fix the emitter, or invert this file again with the reason recorded — ' + + 'do NOT edit the expectation to match new output.', + ).toEqual([true, true]); + } + // The spellings themselves, pinned so a silent move is legible in the diff. + // They happen to be identical here — see the header note; that is a + // measured bonus, not the contract the affinity check above states. + expect([localTypes.v_json, remoteTypes.v_json]).toEqual(['TEXT', 'TEXT']); + expect([localTypes.v_multi, remoteTypes.v_multi]).toEqual(['TEXT', 'TEXT']); }); // ─── §3 The mechanism, measured rather than argued ─────────────────────── - it('THE MECHANISM: the local column converts a bare number-like string on the way in, the remote one does not', async () => { + it('THE MECHANISM: a bare number-like string survives as TEXT on BOTH transports now (#12738)', async () => { // Written with raw SQL on purpose: this asks what the COLUMN does, so it // has to bypass the driver codec whose job is to make the column's answer // not matter. The value is the ablation's own divergent case. @@ -223,35 +247,77 @@ describe('[#12586] driver-turso — the Field.json column type is deliberately a stub.raw.prepare(`insert into "${probe}" ("v") values ('${bare}')`).run(); const remoteCell: any = stub.raw.prepare(`select typeof(v) as ty, v as v from "${probe}"`).all(); + // INVERTED. Before #12738 this read `local: { ty: 'integer', v: 123 }` — + // the local column's NUMERIC affinity ate the leading zero — against + // `remote: { ty: 'text', v: '0123' }`. That difference IS the decision + // #12738 took, and it is now closed in the safe direction: the local half + // moved to the remote half's behaviour, never the reverse. + // + // ⛔ This is the assertion that would go red if anyone converged these two + // onto `json` instead. It is stated as the RAW-SQL truth on purpose: it + // asks what the COLUMN does, bypassing the driver codec whose job is to + // make the column's answer not matter. #12380's codec still runs and is + // still required — for LEGACY columns, which keep their `json` declaration + // and therefore keep NUMERIC affinity. expect( { local: localCell[0], remote: remoteCell[0] }, - 'NUMERIC affinity (local `json`) destroys the leading zero; TEXT affinity (remote `TEXT`) does not. ' + - 'This is the "why it is safe today" of #12586: the local codec never hands this column a bare ' + - 'number-like string, and #12380 is what made that true.', + 'A bare number-like string written by raw SQL must come back as TEXT, byte for byte, on BOTH ' + + 'transports. If the local half reads `{ ty: "integer", v: 123 }` again, the SQLite JSON ' + + 'column has regained NUMERIC affinity — the exposure #12380 defeated and #12738 removed at ' + + 'the root. Fix the emitter; do NOT edit this expectation.', ).toEqual({ - local: { ty: 'integer', v: 123 }, + local: { ty: 'text', v: bare }, remote: { ty: 'text', v: bare }, }); }); + it('THE MECHANISM, CONTROL: a column still declared `json` DOES eat it — so the probe can fail', async () => { + // The negative control the inverted mechanism test needs. Without it, a + // probe that returned 'text' because the write silently stopped happening + // would read exactly like a pass. This creates the LEGACY shape by hand — + // a column explicitly declared `json`, which is what every pre-#12738 + // deployment holds — and shows the affinity is real, present, and still + // eating bare values on this very SQLite build. + const bare = VALUE_ROUNDTRIP_CASES.find((c) => c.name === 's_0123')!.wrote as string; + const legacy = `${TABLE}_legacy_json`; + + await local.execute(`create table "${legacy}" ("v" json)`); + await local.execute(`insert into "${legacy}" ("v") values ('${bare}')`); + const cell: any = await local.execute(`select typeof(v) as ty, v as v from "${legacy}"`); + + expect( + cell[0], + 'NUMERIC affinity on a `json`-declared SQLite column, measured on the same build that answers ' + + '`text` above. This is why legacy columns keep #12380 in force, and why #12738 changes only ' + + 'what NEW columns are declared as.', + ).toEqual({ ty: 'integer', v: 123 }); + }); + // ─── §4 The consequence on the values the drivers really write ─────────── - it('THE CONSEQUENCE: equal answers, unequal bytes — the two transports diverge on exactly two cases', () => { + it('THE CONSEQUENCE: equal answers AND equal bytes — the transports diverge on NOTHING (#12738)', () => { const divergent = VALUE_ROUNDTRIP_CASES.filter( (c) => localStorage.get(c.name) !== remoteStorage.get(c.name), ).map((c) => `${c.name}(${c.column}): local=${localStorage.get(c.name)} remote=${remoteStorage.get(c.name)}`); + // INVERTED. This list held exactly two entries before #12738 — + // `n_int(v_json): local=integer remote=text` and + // `n_real(v_json): local=real remote=text` — the #11535 class in its quiet + // phase: two paths agreeing on every visible ANSWER while standing on + // different ground. The ground is now the same, so the list is empty. + // + // ⚠️ Note what this is NOT evidence of. The round-trip case-set is green on + // both sides of the decision (88 passed, unchanged under convergence), so + // it could never have adjudicated the direction. THIS is the measurement + // that can: it compares the storage class, not the answer. expect( divergent, `Every value in VALUE_ROUNDTRIP_CASES written through each transport's own create(), compared by ` + - `the storage class it landed in. Both transports READ every one of them back faithfully — that is ` + - `turso-value-roundtrip-conformance.test.ts, and it is green either way, which is exactly why this ` + - `pin is a separate file. An EMPTY list here means the transports have converged; a LONGER list ` + - `means one side moved. Both are decisions, neither is a broken expectation.`, - ).toEqual([ - 'n_int(v_json): local=integer remote=text', - 'n_real(v_json): local=real remote=text', - ]); + `the storage class it landed in. An EMPTY list is the #12738 post-state: the two transports ` + + `store the same bytes as well as answering the same values. A NON-EMPTY list means one side ` + + `moved — a decision, not a broken expectation. Fix the emitter, or invert this file with the ` + + `reason recorded.`, + ).toEqual([]); }); it('CONTROL: every non-JSON column agrees on storage class across the transports, though none agrees on spelling', () => { From 97fbb15aa38b2b3d27badbe68b59e44303d56dd3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 11:36:54 +0000 Subject: [PATCH 3/5] test(driver-sql): invert the #12380 affinity pins onto legacy columns --- .../schema-drift.base-type-mismatch.test.ts | 24 ++- .../sql-driver-12380-json-roundtrip.test.ts | 161 +++++++++++++++--- 2 files changed, 163 insertions(+), 22 deletions(-) diff --git a/packages/drivers/driver-sql/src/schema-drift.base-type-mismatch.test.ts b/packages/drivers/driver-sql/src/schema-drift.base-type-mismatch.test.ts index 93553edd90..cda1b21639 100644 --- a/packages/drivers/driver-sql/src/schema-drift.base-type-mismatch.test.ts +++ b/packages/drivers/driver-sql/src/schema-drift.base-type-mismatch.test.ts @@ -370,13 +370,31 @@ function declareBaseTypeDriftSuite(cell: DialectCell): void { // sync had migrated it, they would all pass for the wrong reason. expect(physicalType).toMatch(/char|text/i); - // And a table built from the SAME metadata on a fresh database gets json — - // which is what makes the column above stale rather than simply correct. + // And a table built from the SAME metadata on a fresh database gets the + // JSON column type FOR THIS DIALECT — which is what makes the column above + // stale rather than simply correct. const fresh = `${TABLE}_fresh`; await driver.execute(`drop table if exists ${fresh}`).catch(() => {}); await driver.initObjects([{ name: fresh, fields: { tags: { type: 'string', multiple: true } } }] as any); const freshType = (await driver.columnsOf(fresh)).find((c) => c.name === 'tags')!.type; - expect(freshType).toMatch(/json/i); + + // [#12738] INVERTED on SQLite only, and the inversion REINFORCES this + // suite rather than weakening it. `createColumn` used to emit `json` on + // every dialect; it now emits the dialect-correct type, and SQLite — which + // has no JSON type — gets `text`. So on SQLite a fresh column and the + // stale one below are now the same AFFINITY CLASS, differing only in + // spelling (`TEXT` vs `varchar(255)`). + // + // That is exactly why `multiValueColumnTypeIsLoadBearing()` excludes + // SQLite and why the next case asserts SQLite does NOT corrupt: on this + // dialect the column type was never load-bearing, and after #12738 the + // emitter agrees with the differ instead of merely being excused by it. + // ⛔ Do not "restore" `/json/i` here — that would assert SQLite declares a + // type it does not have. + expect(freshType).toMatch(cell.id === 'sqlite' ? /char|clob|text/i : /json/i); + + // Still a real difference on every dialect — on the enforcing ones it is a + // difference of TYPE, on SQLite only of spelling. expect(freshType).not.toBe(physicalType); await driver.execute(`drop table if exists ${fresh}`).catch(() => {}); }); diff --git a/packages/drivers/driver-sql/src/sql-driver-12380-json-roundtrip.test.ts b/packages/drivers/driver-sql/src/sql-driver-12380-json-roundtrip.test.ts index b7a5c74740..10b23661d5 100644 --- a/packages/drivers/driver-sql/src/sql-driver-12380-json-roundtrip.test.ts +++ b/packages/drivers/driver-sql/src/sql-driver-12380-json-roundtrip.test.ts @@ -172,6 +172,35 @@ async function catalogType(driver: SqlDriver, cell: DialectCell, table: string): return String(rows[0].ty).toLowerCase(); } +/** + * [#12738] The stored `CREATE TABLE` statement for a table, from SQLite's own + * catalog. Used to build a LEGACY-shaped table out of the table the driver + * actually emits, so the legacy fixture cannot drift from the real one. + */ +async function createStatementOf(driver: SqlDriver, cell: DialectCell, table: string): Promise { + const rows = rowsOf(cell, await driver.execute(`select sql from sqlite_master where name = ?`, [table])); + expect(rows, `no CREATE statement for ${table}`).toHaveLength(1); + return String(rows[0].sql); +} + +/** + * [#12738] Turn a driver-emitted `CREATE TABLE` into the PRE-#12738 one: rename + * the table and put `val` back to its old `json` declaration. + * + * ⚠️ The rewrite is ASSERTED, not assumed. A regex that silently matched + * nothing would produce a table declared `text` and every "legacy" expectation + * below would then be measuring the new column while claiming to measure the + * old one — green, and about the wrong table. + */ +function legacyDdlFrom(create: string, from: string, to: string): string { + const renamed = create.replace(new RegExp(`(["\`]?)${from}\\1`), `"${to}"`); + expect(renamed, `table ${from} was not renamed to ${to}`).toContain(to); + const legacy = renamed.replace(/(["`]?val["`]?)\s+text\b/i, '$1 json'); + expect(legacy, 'the `val` column was not put back to its legacy `json` declaration').toMatch(/val["`]?\s+json\b/i); + expect(legacy).not.toMatch(/val["`]?\s+text\b/i); + return legacy; +} + function declareRoundTrip(cell: DialectCell): void { describe(`[#12380] driver-sql — Field.json round-trips faithfully (${cell.label})`, () => { let driver: SqlDriver; @@ -251,69 +280,140 @@ describe(`[#12380] driver-sql — Field.json round-trips faithfully (${cell.labe // ─── §3 The column type, and the affinity it still carries ─────────────── - it('the physical column type is `json`, read from the catalog', async () => { - expect(await catalogType(driver, cell, TABLE)).toBe('json'); + it('the physical column type is the dialect-correct JSON type, read from the catalog', async () => { + // [#12738] INVERTED for SQLite only; `json` was asserted on all three + // dialects before. `createColumn` now emits the type each dialect actually + // has: Postgres `json` and MySQL `json` are native and UNTOUCHED, while + // SQLite — which has no JSON type — declares `text`, the type its own JSON1 + // functions operate on and the one that carries TEXT affinity instead of + // the NUMERIC affinity `json` silently picks up there. + expect(await catalogType(driver, cell, TABLE)).toBe(cell.id === 'sqlite' ? 'text' : 'json'); }); }); } /** - * ⚠️ SQLite only, and the point of the whole design: the DDL is UNCHANGED, so - * the `json`-declared column still has NUMERIC affinity. This proves the - * affinity is still in force AND that the driver's encoded form survives it — - * both in one statement, so neither half can be true of a different column. + * ⚠️ SQLite only. **[#12738] INVERTED, and split in two.** + * + * This block used to assert that the DDL was UNCHANGED — that a `Field.json` + * column was still declared `json`, still carried NUMERIC affinity, and that + * #12380's encoded form survived it. #12738 changed the DDL: the SQLite family + * now declares `text`, so NUMERIC affinity is gone from NEW columns and the + * exposure is closed at the root rather than encoded around. + * + * Both facts still need pinning, and they are now about two different columns, + * which is why this describe covers both: + * + * - **§A the column the driver emits today** — declared `text`, TEXT affinity, + * so a bare number-like value written by raw SQL is no longer destroyed. + * - **§B a LEGACY column** — declared `json` by hand, which is what every + * database created before #12738 holds. NUMERIC affinity is still in force + * there and #12380's encoding still defeats it. This is the half the #12738 + * ruling requires to stay green: existing columns keep their declared type, + * so the codec that protects them stays load-bearing forever. + * + * §B is also §A's negative control. Without it, a §A that answered `text` + * because the write silently stopped happening would read exactly like a pass. */ function declareAffinity(): void { -describe('[#12380] SQLite NUMERIC affinity is still in force, and the encoding defeats it', () => { +describe('[#12738] SQLite affinity: gone from new columns, still defeated on legacy ones', () => { const cell = dialectCell('sqlite'); const T = 'json_affinity_12380'; + const LEGACY_T = 'json_affinity_legacy_12738'; let driver: SqlDriver; beforeAll(async () => { driver = new SqlDriver(cell.config()); await driver.execute(`drop table if exists ${T}`).catch(() => {}); await driver.initObjects([{ name: T, fields: { ...FIELDS } }]); + + // The legacy shape, built by hand from the table the driver just made so + // nothing about it is guessed: same columns, same constraints, with `val` + // declared the pre-#12738 way. Reproducing the old deployment is the only + // way to keep measuring the affinity that old deployments still have. + await driver.execute(`drop table if exists ${LEGACY_T}`).catch(() => {}); + await driver.execute(legacyDdlFrom(await createStatementOf(driver, cell, T), T, LEGACY_T)); }, 60_000); afterAll(async () => { await driver.execute(`drop table if exists ${T}`).catch(() => {}); + await driver.execute(`drop table if exists ${LEGACY_T}`).catch(() => {}); await driver.disconnect(); }); - it('the declared type is still `json` — no DDL change was needed', async () => { - expect(await catalogType(driver, cell, T)).toBe('json'); + // ─── §A the column the driver emits today ──────────────────────────────── + + it('[#12738] the declared type is now `text` — the DDL DID change, and that is the fix', async () => { + // INVERTED from `toBe('json')`. ⛔ Do not restore it: `json` names a type + // SQLite does not have, and its affinity fallback is the whole defect. + expect(await catalogType(driver, cell, T)).toBe('text'); }); it.each(['123', ' 123 ', '0123', '1e5', '1.0', '-0'])( - 'raw %j becomes a numeric storage class, but its JSON encoding stays TEXT', + 'raw %j now SURVIVES as text on the emitted column, and so does its JSON encoding', async (raw) => { const bare = `bare_${raw}`; const enc = `enc_${raw}`; - // ONE statement, one column, two bindings: the pre-fix form and the form - // `formatInput` now produces. Bound through raw SQL so nothing but SQLite's - // own affinity rule can be responsible for the difference. + // ONE statement, one column, two bindings: the pre-#12380 form and the + // form `formatInput` produces. Bound through raw SQL so nothing but + // SQLite's own affinity rule can be responsible for the outcome. await driver.execute( `insert into "${T}" ("id", "label", "val") values (?, ?, ?), (?, ?, ?)`, [`i_${bare}`, bare, raw, `i_${enc}`, enc, JSON.stringify(raw)], ); const bareDisk = await diskCell(driver, cell, T, bare); const encDisk = await diskCell(driver, cell, T, enc); - // The affinity is REAL — this is the mechanism the card could not repair. - expect(['integer', 'real'], `bare ${raw} must be eaten by NUMERIC affinity`).toContain(bareDisk.t); - // …and the quotes defeat it, with no DDL change. + // INVERTED. This asserted `['integer','real']` — the affinity eating the + // value — before #12738. The bare value is now preserved BYTE FOR BYTE, + // which is what "removed at the root" means: raw SQL against this column + // no longer needs the codec to be safe. + expect(bareDisk.t, `bare ${raw} must now stay TEXT`).toBe('text'); + expect(bareDisk.v, `bare ${raw} bytes`).toBe(raw); + // …and the encoded form is unaffected by the change, as it always was. expect(encDisk.t, `encoded ${raw} must stay TEXT`).toBe('text'); expect(encDisk.v, `encoded ${raw} bytes`).toBe(JSON.stringify(raw)); expect(JSON.parse(encDisk.v!), `encoded ${raw} parses back`).toBe(raw); }, ); - it('a native number still lands as a numeric storage class — unchanged by this fix', async () => { + it('[#12738] a native number now lands as TEXT on disk — and still reads back as a number', async () => { + // INVERTED from `toBe('integer')`. The READ is the half that must not move, + // and it does not: `formatOutput` parses the encoded form, so the value is + // still the number 4200. Only the storage class changed. await driver.create(T, { label: 'num', val: 4200 }, { bypassTenantAudit: true }); const d = await diskCell(driver, cell, T, 'num'); - expect(d.t).toBe('integer'); + expect(d.t).toBe('text'); + expect(d.v).toBe('4200'); const rows = (await driver.find(T, { where: { label: 'num' } } as DriverQuery)) as any[]; expect(rows[0].val).toBe(4200); }); + + // ─── §B the legacy column — #12380 still in force, and still needed ─────── + + it('[#12738] a LEGACY column is still declared `json` — the fixture is real', async () => { + expect(await catalogType(driver, cell, LEGACY_T)).toBe('json'); + }); + + it.each(['123', ' 123 ', '0123', '1e5', '1.0', '-0'])( + 'raw %j is STILL eaten by NUMERIC affinity on a legacy column, and #12380 still defeats it', + async (raw) => { + const bare = `bare_${raw}`; + const enc = `enc_${raw}`; + await driver.execute( + `insert into "${LEGACY_T}" ("id", "label", "val") values (?, ?, ?), (?, ?, ?)`, + [`i_${bare}`, bare, raw, `i_${enc}`, enc, JSON.stringify(raw)], + ); + const bareDisk = await diskCell(driver, cell, LEGACY_T, bare); + const encDisk = await diskCell(driver, cell, LEGACY_T, enc); + // Unchanged from the pre-#12738 assertion, on purpose: this is the exact + // measurement #12380 made, still true, now correctly scoped to the + // columns it is still true OF. + expect(['integer', 'real'], `bare ${raw} must be eaten by NUMERIC affinity`).toContain(bareDisk.t); + expect(encDisk.t, `encoded ${raw} must stay TEXT`).toBe('text'); + expect(encDisk.v, `encoded ${raw} bytes`).toBe(JSON.stringify(raw)); + expect(JSON.parse(encDisk.v!), `encoded ${raw} parses back`).toBe(raw); + }, + ); }); } @@ -324,6 +424,16 @@ describe('[#12380] SQLite NUMERIC affinity is still in force, and the encoding d * Legacy rows are planted through raw SQL with the values the PRE-fix * `formatInput` would have bound — so the same affinity rule that produced the * legacy corpus produces this one. + * + * ⚠️ [#12738] The table is now built with an explicit `json` column instead of + * being taken from `initObjects`, and that is a STRENGTHENING rather than a + * workaround. The corpus's defining property is that its number-like cells were + * eaten by NUMERIC affinity — which only a `json`-declared column does. Since + * #12738 the emitter produces `text`, so letting `initObjects` create this table + * would have quietly produced a corpus with no legacy cells in it, and every + * assertion below would have passed while measuring nothing. The migration + * itself is unchanged and untouched by #12738: it selects on + * `typeof(col) = 'text' and json_valid(col) = 0`, never on the declared type. */ function declareMigration(): void { describe('[#12380] the storage-format migration over legacy SQLite json rows', () => { @@ -370,9 +480,22 @@ describe('[#12380] the storage-format migration over legacy SQLite json rows', ( beforeAll(async () => { driver = new SqlDriver(cell.config()); + const shape = `${LEGACY_TABLE}_shape`; await driver.execute(`drop table if exists ${LEGACY_TABLE}`).catch(() => {}); - // Pass 1 CREATES the table, so the backfill correctly does nothing. + await driver.execute(`drop table if exists ${shape}`).catch(() => {}); + // Pass 1 CREATES the table, so the backfill correctly does nothing. It is + // created in the PRE-#12738 shape: the driver emits the table, and `val` is + // put back to `json` so the corpus below really is a legacy corpus. + await driver.initObjects([{ name: shape, fields: { ...FIELDS } }]); + await driver.execute( + legacyDdlFrom(await createStatementOf(driver, cell, shape), shape, LEGACY_TABLE), + ); + await driver.execute(`drop table if exists ${shape}`).catch(() => {}); + // Register the metadata over the now-existing legacy table. Additive sync + // adds nothing (every column is present) and the backfill converts nothing + // (no rows yet), so this is the "old database boots on new code" moment. await driver.initObjects([{ name: LEGACY_TABLE, fields: { ...FIELDS } }]); + expect(await catalogType(driver, cell, LEGACY_TABLE), 'the legacy fixture must be `json`').toBe('json'); await plant(); beforeDisk = await snapshot(); beforeRead = await readAll(); From 2d5cf547cd26ce383eadd4bd62a24753ccd9a553 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 11:39:34 +0000 Subject: [PATCH 4/5] chore: drop a stray local probe script --- packages/drivers/driver-sql/probe3.mjs | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 packages/drivers/driver-sql/probe3.mjs diff --git a/packages/drivers/driver-sql/probe3.mjs b/packages/drivers/driver-sql/probe3.mjs deleted file mode 100644 index 9d468d4a9b..0000000000 --- a/packages/drivers/driver-sql/probe3.mjs +++ /dev/null @@ -1,4 +0,0 @@ -import knexLib from 'knex'; -const k = knexLib({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); -await k.schema.createTable('p', (t) => { t.text('a'); t.json('b'); t.string('c'); }); -console.log('sqlite_master:', (await k.raw("select sql from sqlite_master where name='p'"))[0].sql); From 8058e2b606a94776d49be771f6f4a7ea491a498f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 12:03:31 +0000 Subject: [PATCH 5/5] feat(driver-sql): SQLite-family JSON columns declare TEXT (#12738) Each dialect now declares the semantically correct JSON column type. Postgres and MySQL keep their native JSON types; the SQLite family (plain SqlDriver on sqlite3/better-sqlite3, TursoDriver in all three transport modes, and SqliteWasmDriver) declares text. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01F3jdziLbAPGeceVNmSox5L --- .../sqlite-json-columns-declare-text.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .changeset/sqlite-json-columns-declare-text.md diff --git a/.changeset/sqlite-json-columns-declare-text.md b/.changeset/sqlite-json-columns-declare-text.md new file mode 100644 index 0000000000..1b686ddcce --- /dev/null +++ b/.changeset/sqlite-json-columns-declare-text.md @@ -0,0 +1,44 @@ +--- +'@objectstack/driver-sql': minor +'@objectstack/driver-turso': minor +'@objectstack/driver-sqlite-wasm': minor +--- + +feat(driver-sql,driver-turso,driver-sqlite-wasm): SQLite-family JSON columns declare `TEXT`; server dialects keep native JSON (#12738) + +A `Field.json` column — and any field with `multiple: true` — is now declared as +the JSON column type the **target dialect actually has**. Postgres and MySQL are +untouched: their native `json`/`jsonb` and `JSON` types are correct and stay. +The SQLite family (plain `SqlDriver` on `sqlite3`/`better-sqlite3`, `TursoDriver` +in all three transport modes, and `SqliteWasmDriver`) now declares `text`. + +**Why.** SQLite has no JSON type. It derives a column's *affinity* from +substrings of the declared type name, and `json` contains none of the markers +(`INT`, `CHAR`/`CLOB`/`TEXT`, `BLOB`, `REAL`/`FLOA`/`DOUB`), so it fell through +to **NUMERIC** and converted number-like input on the way in — measured through +raw SQL, a bare `'0123'` was stored as the integer `123`. `text` takes TEXT +affinity, converts nothing, and is what SQLite's own JSON1 functions operate on. +It is also what `RemoteTransport.mapFieldTypeToSQL` had spelled all along, so +turso's two transports now agree instead of diverging on one column. + +## The migration shape + +- **New columns only.** The change is to the DDL emitter. Schema sync is + additive, so no existing column is altered, dropped or rewritten. +- **Existing columns keep their declared type.** A column created before this + release stays `json` and keeps NUMERIC affinity — including through an + unrelated SQLite drift rebuild, which re-declares an introspected `json` + column as `json` rather than converting it. +- **Platform write-path behaviour is unchanged.** The `Field.json` codec stays + injective and stays in force: what the platform writes and reads back is + identical before and after, on legacy and new columns alike. Nothing on the + read path consults the physical column type — `isJsonField` answers from + metadata — so decoding is the same on both spellings. +- **No new schema-drift findings.** The multi-value base-type finding is gated + on the dialect where the column type is load-bearing (Postgres and MySQL); + SQLite was already excluded, and the emitter now agrees with the differ + instead of merely being excused by it. +- **The visible difference is raw-SQL-only, and in the safe direction.** A value + written to a JSON column by raw SQL (bypassing the driver) is preserved as + text on a new column where it would previously have been coerced to a number. + Nothing that was preserved before is coerced now.