diff --git a/.changeset/filter-logic-no-value-exists-enrolment.md b/.changeset/filter-logic-no-value-exists-enrolment.md new file mode 100644 index 0000000000..36c74bf119 --- /dev/null +++ b/.changeset/filter-logic-no-value-exists-enrolment.md @@ -0,0 +1,36 @@ +--- +'@objectstack/spec': patch +--- + +test(spec): `FILTER_LOGIC_CASES` enrols the no-value negated-operator rows (`$nin` / `$notContains`) and `$exists` in both directions; the stale reversal-table cells are refreshed (#13540, #13531) + +The published conformance table gains four rows, all implementing rulings that +had already shipped on every scored backend: + +- `$nin` / `$notContains` **return the rows with no value** (`['2','3','4']`) — + the include direction, #5146 extended by #5298 option A, re-affirmed after + the #5299 reversal was costed and withdrawn. Enrolment was unblocked by + PR #13356, which realigned the last divergent surface (`driver-memory`'s + reference matcher). +- `$exists: true` / `$exists: false` **partition the table on has-value** + (`['1','2']` / `['3','4']`) — never key-presence (#5299 cell 2 / #5962). + Enrolment was unblocked by PR #13529, which moved the last three + key-presence exits. Both directions land together because a + single-direction row is the measured blind spot: `$exists: false` — the + harm the maintainer's ruling called the hardest live one — was invisible + while only `$exists: true` was recorded. + +The docblock's dated reversal-measurement table is refreshed in the same form +it was taken in: the `driver-memory` reference-matcher cells now read MATCH +(realigned by PR #13356), the three `$exists` cells read has-value (moved by +PR #13529), and the "a `$exists` row cannot be enrolled here yet" paragraph is +rewritten as a closed-gap record. + +**Grading: `patch`.** No runtime path changes in any package — both backend +realignments shipped and were graded in their own PRs (PR #13356, PR #13529). +What changes here is the published standard's coverage: a third-party driver +suite that runs the table now runs four more rows, and a red they produce is +the ratchet catching a real divergence from semantics that were already ruled, +not a new contract. (`driver-mongodb`'s server-free translation harness also +learns to model `$regex`, the vocabulary the `$notContains` row emits — a +test-only change, no bump.) diff --git a/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts b/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts index d99bfb62f7..e6bc0ddea8 100644 --- a/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts +++ b/packages/drivers/driver-mongodb/src/mongodb-filter-logic-translation.test.ts @@ -101,6 +101,17 @@ function matchOps(value: unknown, ops: Record): boolean { case '$exists': if ((value !== undefined) !== arg) return false; break; + case '$regex': { + // [#13540] Written by the translator for `$contains`/`$notContains` + // (an escaped literal pattern), so the conformance sweep produces it + // the moment those operators have enrolled rows. MongoDB applies + // `$regex` to STRING values only: a null, missing or non-string value + // never matches — which is exactly what makes the emitted + // `{ $not: { $regex } }` satisfy a no-value row. + if (typeof arg !== 'string') throw new UnsupportedShape('$regex with a non-string pattern'); + if (typeof value !== 'string' || !new RegExp(arg).test(value)) return false; + break; + } case '$not': { // The per-field negation — the only `$not` MongoDB accepts, and it // takes an operator document, never a plain value. @@ -258,10 +269,19 @@ describe('the in-process matcher discriminates', () => { }); it('refuses any operator it does not model, rather than treating it as true', () => { - expect(() => matchDoc(FILTER_LOGIC_ROWS[0], { a: { $regex: 'x' } })).toThrow(/unsupported/); + expect(() => matchDoc(FILTER_LOGIC_ROWS[0], { a: { $mod: [2, 0] } })).toThrow(/unsupported/); expect(() => matchDoc(FILTER_LOGIC_ROWS[0], { $expr: {} })).toThrow(/unsupported/); }); + it('models $regex the way the server applies it: strings only, so a negated pattern satisfies a no-value row', () => { + // [#13540] `$regex` joined the modelled vocabulary when the + // `$notContains` row enrolled (the translator emits `{ $not: { $regex } }` + // for it). Pin both halves of the string-only rule: the positive form + // never matches the null rows, and the negated form always does. + expect(select({ d: { $regex: 'v' } })).toEqual(['1', '2']); + expect(select({ d: { $not: { $regex: 'v1' } } })).toEqual(['2', '3', '4']); + }); + it('honours $nor and the per-field $not the translator is allowed to emit', () => { expect(select({ $nor: [{ a: 'x' }] })).toEqual(['3', '4']); expect(select({ a: { $not: { $eq: 'x' } } })).toEqual(['3', '4']); diff --git a/packages/spec/src/data/filter-logic-conformance.ts b/packages/spec/src/data/filter-logic-conformance.ts index 0be8707794..c9b32acbaa 100644 --- a/packages/spec/src/data/filter-logic-conformance.ts +++ b/packages/spec/src/data/filter-logic-conformance.ts @@ -57,11 +57,12 @@ * > OR-s its BRANCHES; it does not change how the contents within a branch * > combine. * - * The predicates are deliberately boring: string equality, `$in`, `$ne`, `$gte` - * / `$lt` on lexicographic strings. Dates, numeric coercion, `LIKE` escaping and - * case sensitivity are still out — those legitimately differ between a SQL - * engine and a JS matcher, and folding them in would make the table unpassable - * rather than more useful. Keep it that way: a case belongs here only if + * The predicates are deliberately boring: string equality, `$in` / `$nin`, + * `$ne`, `$gte` / `$lt` on lexicographic strings, `$notContains` over plain + * substrings, and the value-presence pair `$null` / `$exists`. Dates, numeric + * coercion, `LIKE` escaping and case sensitivity are still out — those + * legitimately differ between a SQL engine and a JS matcher, and folding them + * in would make the table unpassable rather than more useful. Keep it that way: a case belongs here only if * **every** backend must agree on it. * * **Null handling is IN, as of #5298** — it used to be excluded by the same @@ -70,7 +71,7 @@ * #5146 made `$not` NULL-safe and #5298 did the same for the non-negated * `$ne` / `$nin` / `$notContains`, so "the column has no value" now has ONE * cross-backend answer and belongs to the standard like any other. The - * {@link FilterLogicRow.d} column carries it, and the four `d`-column cases + * {@link FilterLogicRow.d} column carries it, and the eight `d`-column cases * below enforce it on every backend. * * ⚠️ That answer — the INCLUDE direction — was reversed by a ruling on @@ -155,33 +156,41 @@ * - **`$ne` / `$nin` / `$notContains` on a no-value row MATCH** — the include * direction, ruled by #5146 for `$not`, extended to the operator family by * #5298, shipped across all eleven surfaces below via `nullSafeNegative` and - * `nullValueSatisfiesOperator`, and enrolled in {@link FILTER_LOGIC_CASES}. - * Not merely unchallenged — challenged, measured, and re-affirmed with the - * reversal option on the table. + * `nullValueSatisfiesOperator`, and enrolled in {@link FILTER_LOGIC_CASES}: + * `$ne` / `$not` since #5903, `$nin` / `$notContains` since #13540 — the + * last surface holding those two back was the reference matcher itself, + * realigned by PR #13356 (#13166). Not merely unchallenged — challenged, + * measured, and re-affirmed with the reversal option on the table. * - **`$exists` means "has a value"** (`!= null`), never key-presence — cell 2, * the leg of the 07:33Z ruling that was never in conflict with #5298 and had - * already shipped in PR #5962. It stands. + * already shipped in PR #5962 on the surfaces the ruling named. It stands — + * and since PR #13529 (#13195) moved the last three key-presence exits, it + * is enforced here too: enrolled in {@link FILTER_LOGIC_CASES} in BOTH + * directions (#13531). * - * So the two enrolled rows `$ne returns the rows with no value` and `$not - * returns the rows with no value` state the affirmed answer, and the `['2']` - * that native three-valued SQL would give for `d <> 'v1'` is the answer this - * platform deliberately does NOT take. + * So the four enrolled negation rows — `$ne` / `$not` / `$nin` / + * `$notContains` `… returns the rows with no value` — state the affirmed + * answer, and the `['2']` that native three-valued SQL would give for + * `d <> 'v1'` is the answer this platform deliberately does NOT take. * * ### Why the reversal was declined — the measurement * * Taken on `60f0dd8` by adding the candidate rows to this table and running - * every suite that drives it. `MATCH` = a no-value row satisfies the operator, - * i.e. the include direction — the affirmed one: + * every suite that drives it; re-taken on `b997272` with the four candidate + * rows ENROLLED below, after PR #13356 and PR #13529 moved the last divergent + * cells — the dated per-cell notes say which cells moved and what moved them. + * `MATCH` = a no-value row satisfies the operator, i.e. the include direction + * — the affirmed one: * * | Surface | `$notContains` | `$nin` | `$exists: true` on a null value | * |---|---|---|---| * | `formula` `matchesFilterCondition` | MATCH | MATCH | no — already ruled-correct (#5962) | - * | `driver-memory` reference matcher | no — DIVERGENT, frozen (#5499) | MATCH on a null value, no on a missing key | no — ruled-correct (#5962) | - * | `driver-memory` live mingo path | MATCH | MATCH | MATCH — reads KEY-PRESENCE | - * | `driver-memory` analytics face | MATCH | MATCH | MATCH — reads KEY-PRESENCE | + * | `driver-memory` reference matcher | MATCH — realigned by PR #13356 | MATCH on both readings — realigned by PR #13356 | no — ruled-correct (#5962) | + * | `driver-memory` live mingo path | MATCH | MATCH | no — has-value since PR #13529 | + * | `driver-memory` analytics face | MATCH | MATCH | no — has-value since PR #13529 | * | `driver-sql` / `driver-sqlite-wasm` / `driver-turso` local | MATCH | MATCH | no — `IS NOT NULL` | * | `driver-turso` REMOTE | MATCH | MATCH | no — `IS NOT NULL` | - * | `driver-mongodb` `translateFilter` | MATCH | MATCH | MATCH — mongo `$exists` IS key-presence | + * | `driver-mongodb` `translateFilter` | MATCH | MATCH | no — has-value since PR #13529 (lowered to a null test, never mongo `$exists`) | * | `service-analytics` `read-scope-sql` | MATCH | MATCH | no — `IS NOT NULL` | * | `service-analytics` `filter-normalizer` | MATCH | MATCH | no — `IS NOT NULL` | * @@ -206,25 +215,29 @@ * silent absence for visible surplus. With no business pull behind it, the * maintainer kept include. * - * ⚠️ One cell of the three is still short of its ruling, and it is the cell that - * SURVIVED. `$exists` = has-value is settled and shipped on the surfaces the - * ruling named — `formula` and `driver-memory`'s reference matcher both read it - * that way (#5298 ③ / #5369, landed in #5962). Still reading key-presence: - * `driver-memory`'s live mingo path and `driver-mongodb`, both among the FIVE - * drivers this gate scores. The #5499 investment freeze that once explained why - * neither had moved was lifted on 2026-08-11 (recorded in - * `./aggregation-conformance.ts`), so the gap is now unexcused rather than - * deferred — but it is still a gap, so a `$exists` row cannot be enrolled here - * yet. - * - * ⛔ And the blocker on that row is NOT a missing wording, which is worth - * stating because the obvious workaround does not exist: **the DEBT ledger in + * ⚠️ The `$exists` column was the cell of the three that SURVIVED the + * withdrawal, and for a while it ran short of its own ruling. `$exists` = + * has-value shipped first on the surfaces the ruling named — `formula` and + * `driver-memory`'s reference matcher (#5298 ③ / #5369, landed in #5962) — + * while three exits still read key-presence: `driver-memory`'s live mingo + * path, its analytics face (a third divergent exit the earlier prose never + * named; measured in PR #13420), and `driver-mongodb`'s `translateFilter`. + * The #5499 investment freeze that once excused the lag dissolved on + * 2026-08-11 (recorded in `./aggregation-conformance.ts`), and PR #13529 + * (#13195) moved all three to has-value — the gap is closed, the stated + * blocker on enrolment is gone with it, and the two `$exists` rows below are + * enrolled in BOTH directions (#13531). + * + * ⛔ The granularity rule that made that wait mandatory is still the standing + * rule for every future row, and it is NOT a missing wording — the obvious + * workaround does not exist: **the DEBT ledger in * `scripts/check-driver-conformance.mjs` is per (driver × case-set), not per * case.** An entry says "this driver's suite does not import this marker at * all". There is no spelling for "this driver fails one row while passing the - * other thirty-five", so a row added ahead of a backend is simply a red gate — - * the thing #5903's note calls "a gate that reports a known red", which teaches - * every agent reading CI to discount the colour. + * rest", so a row added ahead of a backend is simply a red gate — the thing + * #5903's note calls "a gate that reports a known red", which teaches every + * agent reading CI to discount the colour. Enrolment stays all-green-first: + * every row in the table below landed only once every scored driver passed it. */ import type { FilterCondition } from './filter.zod'; @@ -407,9 +420,11 @@ export const FILTER_LOGIC_CASES: readonly FilterLogicCase[] = [ // Rows 3 and 4 have no `d`. The `$null` partition at the bottom of this // section is what makes that true of every harness: a fixture that quietly // stored `''` instead of NULL, or a `NOT NULL` column that rejected the seed, - // fails THERE rather than by turning one of the two cases here green for the - // wrong reason. Read the four together — a no-value row is either inside the - // negation or outside it, and the partition says which rows are which. + // fails THERE rather than by turning one of the negation cases here green + // for the wrong reason. Read the eight together — a no-value row is either + // inside the negation or outside it, and the two partition pairs (`$null` + // and `$exists`, opposite polarities of one question) say which rows are + // which. // // The family completed here in #5903, the PR that made `driver-turso` REMOTE // answer them: it was the last backend of eleven still returning `['2']`, and @@ -437,6 +452,39 @@ export const FILTER_LOGIC_CASES: readonly FilterLogicCase[] = [ expected: ['2', '3', '4'], note: '#5146: the same ruling reached through the combinator. `NOT (NULL = ?)` is UNKNOWN, so an unguarded negation drops rows 3-4 — and on a CEL `!expr` read scope that is one permission rule admitting different row sets per backend.', }, + + // [#13540] The negated OPERATOR forms of the same ruling, enrollable since + // PR #13356 realigned the reference matcher — the last of the eleven + // surfaces still answering either of them the pre-ruling way (#13166 + // measured the divergence; the freeze that once excused it dissolved on + // 2026-08-11). + // + // Reading scope, stated so the next author knows what these rows do and do + // not measure: every harness seeds rows 3-4 with a stored NULL (`d: null`), + // so these cases hold all backends to the stored-null reading of "no + // value". The key-ABSENT reading — the shape a partial write leaves, and + // the reading `$nin` actually diverged on — is only distinguishable on the + // document faces and has no spelling in this shared fixture; it is pinned + // per face, three operators x both readings, in driver-memory's + // `memory-matcher-no-value-negated-operators.test.ts` (#13166). + // + // ⛔ Deliberately NOT enrolled beside them: the `[null]`-comparand axis + // (`$in: [null]` / `$nin: [null]`). PR #13356 disclosed that the reference + // matcher's answer for `$nin: [null]` on a missing key moved with the fix, + // but that cell has never been ruled cross-surface — it is #13357's axis, + // and a conformance row is a ruling ratchet, not a way to mint one. + { + name: '$nin returns the rows with no value', + filter: { d: { $nin: ['v1'] } }, + expected: ['2', '3', '4'], + note: '#5298 option A: the list form of `$ne` — "not one of [v1]" is true of a row whose d is absent. A three-valued `NOT IN` drops rows 3-4; the reference matcher answered this way on a stored null but not on a missing key until PR #13356.', + }, + { + name: '$notContains returns the rows with no value', + filter: { d: { $notContains: 'v1' } }, + expected: ['2', '3', '4'], + note: '#5298 option A: a row with no value satisfies the negated substring test. The reference matcher failed BOTH readings of this one on the type test (`null` is not a string) until PR #13356; the SQL faces reach it through `nullSafeNegative`.', + }, { name: '$null true selects exactly the no-value rows', filter: { d: { $null: true } }, @@ -450,6 +498,30 @@ export const FILTER_LOGIC_CASES: readonly FilterLogicCase[] = [ note: 'The complement, so `$null` is pinned as a partition of the table rather than one half of one. Together with the row-count control this makes a NOT NULL fixture column fail loudly instead of quietly passing everything.', }, + // [#13531] The value-presence predicate, enrolled in BOTH directions once + // PR #13529 (#13195) moved the last three key-presence exits to has-value. + // The stored-null seeding is what makes these rows discriminating: a + // key-presence reading answers MATCH on rows 3-4 for `$exists: true` + // precisely because every harness stores `d: null` with the key present — + // every exit that historically diverged did so on this reading, while on a + // key-absent document the two readings agree. Single-direction enrolment is + // the recorded blind spot: `$exists: false` — the harm the maintainer's + // ruling called the hardest live one — stayed invisible for as long as + // only `$exists: true` was measured (PR #13420's pins were inverted on + // exactly that axis). + { + name: '$exists true selects exactly the valued rows', + filter: { d: { $exists: true } }, + expected: ['1', '2'], + note: '#5299 cell 2 / #5962: `$exists` means HAS A VALUE, never key-presence. A key-presence reading returns all four rows here, because the fixture stores `d: null` with the key present — the reading every divergent exit failed on.', + }, + { + name: '$exists false selects exactly the no-value rows', + filter: { d: { $exists: false } }, + expected: ['3', '4'], + note: 'The direction the ruling called the hardest live harm: a key-presence reading returns NOTHING here, silently emptying every "field is not set" scope. Enrolled beside its twin so a single-direction blind spot cannot rebuild.', + }, + // ── Shapes read scopes are actually written in ──────────────────────────── { name: 'read scope: own AND active, OR another owner\'s row',