diff --git a/.changeset/metadata-mutation-cluster-fanout.md b/.changeset/metadata-mutation-cluster-fanout.md new file mode 100644 index 0000000000..6415cfa825 --- /dev/null +++ b/.changeset/metadata-mutation-cluster-fanout.md @@ -0,0 +1,52 @@ +--- +"@objectstack/metadata-protocol": minor +"@objectstack/service-cluster": minor +--- + +feat(metadata-protocol,service-cluster): fan runtime metadata mutations out to peer replicas — a runtime-authored object no longer answers OBJECT_NOT_FOUND on every replica that did not perform the write (#13331) + +Measured on a live 3-replica EE deployment (ADR-0018 compose, redis driver): +an object authored through `PUT /api/v1/meta/object/...` persisted to the +shared `sys_metadata` (so `/api/v1/meta/*` answered 200 fleet-wide) but +registered with the ObjectQL engine registry of the writing replica only — +`/api/v1/data/` answered a hard 404 `OBJECT_NOT_FOUND` on the other +replicas, indefinitely (200 concurrent creates through the LB: 67×201 / +133×404; a boot-loaded control object: 0 errors; the only recovery was a full +fleet restart). The runtime authoring path lives entirely in the metadata +protocol and never touches the metadata service, so the existing +`metadata.changed` bridge — even when attached — never heard these writes. + +Maintainer-ruled design (2026-09-01, Option A): + +- **Publisher at the producer choke point.** The protocol's post-persistence + mutation funnel (`saveMetaItem` / `publishMetaItem` / `deleteMetaItem` — + the same seam `onMetadataMutation` subscribes) now also publishes the + mutation's ADDRESS on a new cluster channel `metadata.mutated` + (`METADATA_MUTATION_CLUSTER_CHANNEL`, payload + `ClusterMetadataMutationPayload`). Drafts are not published — they never + enter any replica's registry. +- **Peers converge from their own DB read.** On receipt, a replica re-reads + the row from its OWN `sys_metadata` and re-runs the registry write-through + (active row present) or the delete heal walk (no active row). The payload + is a signal, never trusted content — the shared database stays the single + source of truth, and duplicate or out-of-order delivery converges to the + row's current state by construction. After convergence the event replays + into the replica's local `onMetadataMutation` listeners (never + re-published), so boot-cached consumers such as the authored hook/action + re-bind re-sync on peers exactly as they do on the writer. +- **New attach seam, mirrored from the shipped bridges.** + `ObjectStackProtocolImplementation.attachMetadataMutationPubSub(pubsub, + nodeId)` — idempotent on the `(pubsub, nodeId)` pair, with loopback + suppression via `originNode`, shaped after + `MetadataManager.attachClusterPubSub()` and the engine's + `attachAuthzInvalidationPubSub()`. `MetadataClusterBridgePlugin` late-binds + it at `kernel:ready` as a second, independent lane beside the existing + metadata-service lane — the boot shape that lacks a manager-backed + `metadata` service (the TS-config host-config boot, exactly the shipped EE + shape) is the one that needs this lane most. The new lane skips the + in-process memory driver (nothing to fan out to), the guard the authz + sibling already carries. + +No shipped driver exceeds at-most-once delivery, so a lost message still +degrades to the pre-existing staleness bound (heal at next boot); this +channel narrows the window from "until restart" to one network hop. diff --git a/content/docs/permissions/system-context.mdx b/content/docs/permissions/system-context.mdx index be72c21552..e51f76faa8 100644 --- a/content/docs/permissions/system-context.mdx +++ b/content/docs/permissions/system-context.mdx @@ -112,7 +112,7 @@ that silently does not happen. | 18 | **`readonly` strip bypassed — UPDATE, single row** | objectql | Get: a `readonly` field CAN be written. Lose: the protection that stops a caller seeding e.g. `approval_status` | `objectql/src/engine.ts:10914` | | 19 | **`readonly` strip bypassed — UPDATE, bulk/predicate** | objectql | Same, on the multi-row path | `objectql/src/engine.ts:11076` | | 20 | **`readonly` strip bypassed — INSERT (engine pass)** | objectql | Same, on create | `objectql/src/engine.ts:9772` | -| 21 | **`readonly` strip bypassed — INSERT (protocol ingress)** | metadata-protocol | `isSystem` is the **only** exemption here. `preserveAudit` is deliberately not read on this path (#6640) — a non-system historical import is still stripped on create | `metadata-protocol/src/protocol.ts:1737` | +| 21 | **`readonly` strip bypassed — INSERT (protocol ingress)** | metadata-protocol | `isSystem` is the **only** exemption here. `preserveAudit` is deliberately not read on this path (#6640) — a non-system historical import is still stripped on create | `metadata-protocol/src/protocol.ts:1741` | | 22 | Strict-drop refusal never fires | objectql | Lose: a caller that opted into loud refusal gets **silence** — strict refuses exactly what the strip would have taken, and the strip took nothing | `objectql/src/engine.ts:9809`, `readonly-strict-errors.ts:66` | | 23 | **Referential-integrity check skipped** | objectql | Get: writes proceed against unreachable/unresolvable targets. Lose: an `isSystem` caller can write a **dangling reference** | `objectql/src/engine.ts:5730` | | 24 | Tenant-audit warning silenced; `bypassTenantAudit` threaded to the driver | objectql | Get: unscoped system writes stop warning. Lose: the signal that would flag a genuine user-path scoping bug | `objectql/src/engine.ts:3599`, `:3609`, `:3636` | diff --git a/packages/metadata-protocol/src/index.ts b/packages/metadata-protocol/src/index.ts index 246ea6b10e..9963d5e511 100644 --- a/packages/metadata-protocol/src/index.ts +++ b/packages/metadata-protocol/src/index.ts @@ -135,6 +135,13 @@ export type { UninstallCleanup, UninstallCleanupOutcome } from './protocol.js'; // against the producer's contract instead of restating it locally. export type { DeletePackageRequest, DeletePackageResponse } from './protocol.js'; export type { MetadataMutationEvent, MetadataMutationProjector, MutationProjectionOutcome } from './protocol.js'; +// [#13331] The cross-node half of the mutation notification: the cluster +// channel the protocol publishes post-persistence mutations on, and its +// address-only payload. Exported for the bridge's tests and for any peer-side +// consumer that must speak the channel by name — the payload is a SIGNAL, +// never trusted content (receipt re-reads `sys_metadata` locally). +export { METADATA_MUTATION_CLUSTER_CHANNEL } from './protocol.js'; +export type { ClusterMetadataMutationPayload } from './protocol.js'; // [#10219] The per-item publish notification the host bridges to the // kernel-wide `metadata:reloaded` announce. Exported for the same reason its // mutation sibling is: the subscriber lives in another package. diff --git a/packages/metadata-protocol/src/protocol.cluster-mutation-fanout.test.ts b/packages/metadata-protocol/src/protocol.cluster-mutation-fanout.test.ts new file mode 100644 index 0000000000..a77d29d333 --- /dev/null +++ b/packages/metadata-protocol/src/protocol.cluster-mutation-fanout.test.ts @@ -0,0 +1,466 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#13331] Cross-node registry convergence — the protocol's `metadata.mutated` + * fan-out, measured over the topology that failed in production: two protocol + * instances ("replicas") over ONE shared `sys_metadata` store, each with its + * OWN in-memory registry, joined by a bus that delivers every publish to every + * subscriber (publisher included — that is what a real remote driver does, and + * it is what the `originNode` loopback guard exists for). + * + * The defect this pins, measured on a live 3-replica EE deployment before the + * fix: a runtime-authored object persists to the shared DB but registers with + * the WRITING replica's engine registry only, so `/api/v1/data/` + * answers OBJECT_NOT_FOUND on every other replica, indefinitely (200 + * concurrent creates through the LB: 67×201 / 133×404; a boot-loaded control + * object under the identical harness: 0 errors). + * + * --------------------------------------------------------------------------- + * Two-arm design, directions declared BEFORE running + * --------------------------------------------------------------------------- + * • Arm B (bridge attached): the writer's publish reaches the peer and the + * peer's registry converges FROM ITS OWN DB READ -> GREEN + * • Arm A (control, no attach): the same write leaves the peer's registry + * EMPTY — the pre-fix production shape -> GREEN + * (constrains the instrument: Arm B's convergence is the bridge's doing, + * not a harness artifact that registers everywhere unconditionally) + * + * Ablation, direction declared in writing for the committed tree: reverting + * the publisher (the `publishMetadataMutation` call inside + * `emitMetadataMutation`) turns EXACTLY the Arm-B convergence cases red — + * "peer converges after a writer publish", "a delete on the writer heals the + * peer", "the peer's listeners hear a remote mutation" — while Arm A, the + * loopback case, the draft-silence case and every local-write assertion stay + * green: with no publisher nothing crosses the bus, which is indistinguishable + * from the shipped defect. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { assertEngineDeleteDispatch, assertEngineUpdateDispatch, assertEngineFindOnePredicate } from '@objectstack/metadata-core'; +import { + ObjectStackProtocolImplementation, + METADATA_MUTATION_CLUSTER_CHANNEL, + type ClusterMetadataMutationPayload, + type MetadataMutationEvent, +} from './protocol.js'; + +interface Row { + id: string; + type: string; + name: string; + organization_id: string | null; + package_id: string | null; + state: string; + metadata: string; + checksum?: string; + version?: number; +} + +interface HistoryRow { + id: string; + type: string; + name: string; + version: number; + organization_id: string | null; + operation_type: string; + metadata?: string | null; + recorded_at?: string; +} + +/** ADR-0048 overlay key — (type, name, org, state, package_id). */ +const keyOf = (w: Record) => + `${w.type}|${w.name}|${w.organization_id ?? '__env__'}|${w.state ?? 'active'}|${w.package_id ?? '__nopkg__'}`; + +function matchesWhere(r: Row, where: Record): boolean { + for (const [k, v] of Object.entries(where)) { + if (k === '$or') { + const clauses = v as Array>; + if (!clauses.some((c) => matchesWhere(r, c))) return false; + continue; + } + if (v === undefined) continue; + if ((r as unknown as Record)[k] !== v) return false; + } + return true; +} + +/** The SHARED database — what every replica of the deployment reads. */ +function makeSharedStore() { + return { + rows: new Map(), + historyRows: [] as HistoryRow[], + nextId: { value: 0 }, + }; +} + +/** + * One "replica": a stub engine over the SHARED store with its OWN registry, + * wrapped by its own protocol instance. Registry verbs record their calls — + * the observation channel every case below reads. + */ +function makeReplica(store: ReturnType) { + const { rows, historyRows, nextId } = store; + const registeredItems: Array<{ type: string; name: unknown }> = []; + const registeredObjects: string[] = []; + const unregisteredObjects: string[] = []; + const removedObjectOverlays: string[] = []; + const removedEntries: string[] = []; + + const findRow = (w: Record): { key: string; row: Row } | null => { + if (w.id !== undefined) { + for (const [k, r] of rows) if (r.id === w.id) return { key: k, row: r }; + return null; + } + if (w.package_id !== undefined) { + const k = keyOf(w); + const r = rows.get(k); + if (r) return { key: k, row: r }; + } + for (const [k, r] of rows) if (matchesWhere(r, w)) return { key: k, row: r }; + return null; + }; + + const matchesHistory = (h: HistoryRow, w: Record): boolean => { + if (w.organization_id !== undefined && h.organization_id !== w.organization_id) return false; + if (w.type !== undefined && h.type !== w.type) return false; + if (w.name !== undefined && h.name !== w.name) return false; + if (w.version !== undefined && h.version !== w.version) return false; + if (w.operation_type !== undefined && h.operation_type !== w.operation_type) return false; + return true; + }; + + const engine: any = { + async findOne(table: string, opts: { where: Record }) { + assertEngineFindOnePredicate(table, opts); + if (table === 'sys_metadata_history') { + return historyRows.find((h) => matchesHistory(h, opts.where)) ?? null; + } + if (table !== 'sys_metadata') return null; + return findRow(opts.where)?.row ?? null; + }, + async find(table: string, opts?: { where?: Record; limit?: number }) { + // Hold the caller's bound AFTER the filter, by PRESENCE — the + // objectql-double-limit contract: a bound the double ignores makes + // a pagination bug invisible to every test built on it. + const bound = (all: T[]): T[] => + typeof opts?.limit === 'number' ? all.slice(0, opts.limit) : all; + if (table === 'sys_metadata_history') { + return bound(historyRows.filter((h) => matchesHistory(h, opts?.where ?? {}))); + } + if (table !== 'sys_metadata') return []; + return bound(Array.from(rows.values()).filter((r) => matchesWhere(r, opts?.where ?? {}))); + }, + async insert(table: string, data: Record) { + if (table === 'sys_metadata_audit') return { id: 'audit_skip' }; + if (table === 'sys_metadata_history') { + nextId.value += 1; + const h = { ...(data as unknown as HistoryRow), id: `h_${nextId.value}` }; + historyRows.push(h); + return { id: h.id }; + } + if (table !== 'sys_metadata') return { id: 'side_effect_skip' }; + nextId.value += 1; + const row = { ...(data as unknown as Row), id: `r_${nextId.value}` }; + rows.set(keyOf(data), row); + return { id: row.id }; + }, + async update(_t: string, data: Record, opts: { where: Record }) { + assertEngineUpdateDispatch(data, opts); + const found = findRow(opts.where); + if (!found) return { id: null }; + const merged = { ...found.row, ...(data as unknown as Row) }; + rows.delete(found.key); + rows.set(keyOf(merged), merged); + return { id: found.row.id }; + }, + async delete(_t: string, opts: { where: Record }) { + assertEngineDeleteDispatch(opts); + const found = findRow(opts.where); + if (!found) return { deleted: 0 }; + rows.delete(found.key); + return { deleted: 1 }; + }, + async count() { return 0; }, + async transaction(cb: (ctx: unknown, info: { owned: boolean }) => Promise): Promise { + return cb(undefined, { owned: true }); + }, + async syncObjectSchema() { return true; }, + async dropObjectSchema() { return true; }, + registry: { + registerItem: (type: string, item: any) => { + registeredItems.push({ type, name: item?.name }); + }, + registerObject: (body: any) => { registeredObjects.push(body?.name); }, + unregisterObject: (name: string) => { unregisteredObjects.push(name); return true; }, + removeObjectOverlay: (name: string) => { removedObjectOverlays.push(name); }, + removeRuntimeShadow: () => false, + removeOverlayEntry: (type: string, name: string) => { + removedEntries.push(`${type}|${name}`); + return true; + }, + listItems: () => [], + getItem: () => undefined, + getObject: () => undefined, + getPackage: () => undefined, + getArtifactItem: () => undefined, + }, + }; + + const protocol = new ObjectStackProtocolImplementation( + engine, () => new Map(), 'env_prod', + ) as any; + + return { + protocol, + registeredItems, + registeredObjects, + unregisteredObjects, + removedObjectOverlays, + removedEntries, + }; +} + +/** + * A remote-driver-shaped bus: one transport object, every publish delivered + * synchronously to EVERY subscription — the publisher's own node included. + */ +function makeBus() { + const subs: Array<{ channel: string; handler: (msg: { channel: string; payload: unknown; publishedAt: number }) => void }> = []; + const published: Array<{ channel: string; payload: ClusterMetadataMutationPayload }> = []; + const bus = { + async publish(channel: string, payload: unknown) { + published.push({ channel, payload: payload as ClusterMetadataMutationPayload }); + for (const s of [...subs]) { + if (s.channel === channel) s.handler({ channel, payload, publishedAt: Date.now() }); + } + }, + subscribe(channel: string, handler: (msg: never) => void) { + const sub = { channel, handler: handler as (msg: { channel: string; payload: unknown; publishedAt: number }) => void }; + subs.push(sub); + return () => { + const i = subs.indexOf(sub); + if (i >= 0) subs.splice(i, 1); + }; + }, + async close() {}, + }; + return { bus, published, subscriptionCount: () => subs.length }; +} + +/** Clears the #8308 authoring gates — authored OWD plus at least one field. */ +const objectBody = (name: string, label = 'Gadget') => ({ + name, + label, + sharingModel: 'private', + fields: { title: { type: 'text', label: 'Title' } }, +}); + +/** Both replicas over one store, joined (or not) by the bus. */ +function makeCluster(opts: { attach: boolean } = { attach: true }) { + const store = makeSharedStore(); + const writer = makeReplica(store); + const peer = makeReplica(store); + const { bus, published, subscriptionCount } = makeBus(); + if (opts.attach) { + writer.protocol.attachMetadataMutationPubSub(bus, 'node-a'); + peer.protocol.attachMetadataMutationPubSub(bus, 'node-b'); + } + return { store, writer, peer, bus, published, subscriptionCount }; +} + +const settle = () => new Promise((r) => setTimeout(r, 0)); + +describe('[#13331] ⭐ two-arm: peer registry convergence is the bridge’s doing', () => { + it('Arm B — a runtime-authored object registers on the PEER after the writer’s save', async () => { + const { writer, peer } = makeCluster({ attach: true }); + + const res = await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + expect(res.success).toBe(true); + + // The writer registered synchronously at its own door… + expect(writer.registeredObjects).toEqual(['gadget']); + // …and the peer converges from its OWN read of the shared store — + // this line is the fix: pre-fix it was `[]`, forever. + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget'])); + // Under the canonical singular key, like every other write-through + // route (route 5 in the spelling file's trace). + expect(peer.registeredItems).toEqual([{ type: 'object', name: 'gadget' }]); + }); + + it('Arm A — CONTROL: the identical write with no bridge leaves the peer empty', async () => { + const { writer, peer } = makeCluster({ attach: false }); + + const res = await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + expect(res.success).toBe(true); + await settle(); + + // The pre-fix production shape: writer registered, peer never hears. + // This arm is what makes Arm B a measurement — the harness does not + // register anywhere on its own. + expect(writer.registeredObjects).toEqual(['gadget']); + expect(peer.registeredObjects).toEqual([]); + expect(peer.registeredItems).toEqual([]); + }); +}); + +describe('[#13331] the payload is a signal, never content', () => { + it('publishes the row ADDRESS only — no body rides the channel', async () => { + const { writer, published } = makeCluster({ attach: true }); + + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await settle(); + + expect(published).toHaveLength(1); + expect(published[0].channel).toBe(METADATA_MUTATION_CLUSTER_CHANNEL); + const payload = published[0].payload; + expect(payload.originNode).toBe('node-a'); + // Address-only: the ruled contract (2026-09-01) — a peer must re-read + // its own DB, so the wire must not offer it anything else to trust. + expect(Object.keys(payload.event).sort()).toEqual( + ['name', 'organizationId', 'state', 'type'], + ); + expect(payload.event).toEqual({ + type: 'object', name: 'gadget', state: 'active', organizationId: null, + }); + }); + + it('a DRAFT save publishes nothing — drafts never enter any registry', async () => { + const { writer, peer, published } = makeCluster({ attach: true }); + + const res = await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'draft', + }); + expect(res.success).toBe(true); + await settle(); + + expect(published).toHaveLength(0); + expect(peer.registeredObjects).toEqual([]); + }); +}); + +describe('[#13331] loopback and idempotency', () => { + it('the writer never re-applies its OWN publish (originNode suppression)', async () => { + const { writer } = makeCluster({ attach: true }); + + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await settle(); + + // Exactly the local write-through's registration — a loopback apply + // would make this 2 (the bus delivers to the publisher too). + expect(writer.registeredObjects).toEqual(['gadget']); + }); + + it('duplicate delivery converges to the same state (at-least-once is harmless)', async () => { + const { writer, peer, bus, published } = makeCluster({ attach: true }); + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget'])); + + // Replay the captured message verbatim — a second delivery of the + // same signal. + await bus.publish(published[0].channel, published[0].payload); + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget', 'gadget'])); + + // Same read, same registration, same key — nothing diverged. + expect(peer.registeredItems).toEqual([ + { type: 'object', name: 'gadget' }, + { type: 'object', name: 'gadget' }, + ]); + }); + + it('re-attaching the same (pubsub, nodeId) pair does not double-subscribe', () => { + const { writer, bus, subscriptionCount } = makeCluster({ attach: true }); + const before = subscriptionCount(); + writer.protocol.attachMetadataMutationPubSub(bus, 'node-a'); + expect(subscriptionCount()).toBe(before); + }); + + it('after detach, deliveries are no longer applied', async () => { + const { writer, peer, bus, published } = makeCluster({ attach: true }); + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget'])); + + peer.protocol.detachMetadataMutationPubSub(); + await bus.publish(published[0].channel, published[0].payload); + await settle(); + + expect(peer.registeredObjects).toEqual(['gadget']); + }); +}); + +describe('[#13331] delete fan-out — the peer heals from its own read', () => { + it('a delete on the writer retires the peer’s registry entry', async () => { + const { writer, peer } = makeCluster({ attach: true }); + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget'])); + + const res = await writer.protocol.deleteMetaItem({ type: 'object', name: 'gadget' }); + expect(res.success).toBe(true); + + // No active row remains in the shared store, so the peer runs the + // same heal walk the writer ran locally (#6808's two-place removal). + await vi.waitFor(() => expect(peer.unregisteredObjects).toEqual(['gadget'])); + expect(peer.removedObjectOverlays).toEqual(['gadget']); + expect(peer.removedEntries).toContain('object|gadget'); + }); + + it('a draft DISCARD leaves the peer’s active registration standing', async () => { + const { writer, peer } = makeCluster({ attach: true }); + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget', 'Draft rename'), mode: 'draft', + }); + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget'])); + + const res = await writer.protocol.deleteMetaItem({ type: 'object', name: 'gadget', state: 'draft' }); + expect(res.success).toBe(true); + + // The ACTIVE row survives the discard, so the peer's DB read finds it + // and re-registers (idempotent) rather than healing it away — the + // event name does not decide, the read does. + await vi.waitFor(() => expect(peer.registeredObjects).toEqual(['gadget', 'gadget'])); + expect(peer.unregisteredObjects).toEqual([]); + }); +}); + +describe('[#13331] listener replay — boot-cached consumers hear remote mutations', () => { + it('the peer’s onMetadataMutation listeners receive the remote event, once, after convergence', async () => { + const { writer, peer } = makeCluster({ attach: true }); + const writerSeen: MetadataMutationEvent[] = []; + const peerSeen: MetadataMutationEvent[] = []; + writer.protocol.onMetadataMutation((evt: MetadataMutationEvent) => { writerSeen.push(evt); }); + peer.protocol.onMetadataMutation((evt: MetadataMutationEvent) => { + // #5109 invalidate-before-notify, cross-node edition: by the time + // a listener hears it, the registry must already serve the row. + peerSeen.push({ ...evt }); + expect(peer.registeredObjects).toEqual(['gadget']); + }); + + await writer.protocol.saveMetaItem({ + type: 'object', name: 'gadget', item: objectBody('gadget'), mode: 'publish', + }); + await vi.waitFor(() => expect(peerSeen).toHaveLength(1)); + + expect(peerSeen[0]).toEqual({ + type: 'object', name: 'gadget', state: 'active', organizationId: null, + }); + // The writer's listeners heard the LOCAL emit exactly once — the + // remote replay stays local to the receiving node (no echo). + expect(writerSeen).toHaveLength(1); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.object-registry-write-through-spelling.test.ts b/packages/metadata-protocol/src/protocol.object-registry-write-through-spelling.test.ts index eaaaab3991..c4f7d4e6af 100644 --- a/packages/metadata-protocol/src/protocol.object-registry-write-through-spelling.test.ts +++ b/packages/metadata-protocol/src/protocol.object-registry-write-through-spelling.test.ts @@ -38,7 +38,7 @@ // The trace, measured on `origin/main` — all four routes fold at the producer // --------------------------------------------------------------------------- // `applyObjectRegistryMutation` has exactly ONE caller -// (`applyRegistryWriteThrough`), which has exactly FOUR: +// (`applyRegistryWriteThrough`), which has exactly FIVE: // // 1. `saveMetaItem` passes `singularTypeForRepo`, and the method // already ran `canonicalizeMetaRequestType`. @@ -53,6 +53,13 @@ // over a value read from the stored row. // 4. `rollbackMetaItem` binds `singularType = request.type` AFTER // `canonicalizeMetaRequestType` (#8819). +// 5. `applyRemoteMetadataMutation` (#13331) folds at the call site through +// `canonicalMetaType` (the complete map, +// #9161) — the event crossed a PROCESS +// boundary (a peer replica's cluster publish), +// so this caller folds rather than trusting +// the wire, even though every in-tree +// publisher emits the already-folded singular. // // Both fold maps resolve the plural — `PLURAL_TO_SINGULAR.objects === 'object'` // and `canonicalMetaUrlType('objects') === 'object'` — so no route can deliver @@ -126,7 +133,7 @@ // 6. Comment stripper neutered to return '' -> RED in §4 // (non-vacuity: the expected count is ONE, not zero) -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { PLURAL_TO_SINGULAR, canonicalMetaUrlType } from '@objectstack/spec/shared'; @@ -462,6 +469,52 @@ describe('[#8862] every object registry write-through registers under the singul expect(res.success).toBe(true); expect(objectKeys(registeredItems)).toEqual(['object']); }); + + // ── route 5: applyRemoteMetadataMutation, plural over the WIRE ────────── + // + // The one route whose input crosses a process boundary (#13331): a peer's + // cluster publish. Every in-tree publisher emits the folded singular, so + // a plural here means a foreign or future producer — the call-site fold + // through `canonicalMetaType` is what keeps such a message from minting a + // plural registry key. Driven through the real subscribe path, not by + // reaching into the private method. + it('a remote mutation addressed `objects` registers `object`', async () => { + const { protocol, registeredItems } = makeProtocol(); + // The row a converging peer would find in its own (shared) DB. + await protocol.saveMetaItem({ + type: 'object', name: 'ticket', item: objectBody('ticket'), + packageId: PKG, mode: 'publish', + }); + registeredItems.length = 0; + + let deliver: ((msg: { channel: string; payload: unknown; publishedAt: number }) => void) | undefined; + const pubsub = { + publish: async () => {}, + subscribe: (_channel: string, handler: (msg: never) => void) => { + deliver = handler as typeof deliver; + return () => {}; + }, + close: async () => {}, + }; + protocol.attachMetadataMutationPubSub(pubsub, 'node-local'); + + deliver!({ + channel: 'metadata.mutated', + payload: { + originNode: 'node-peer', + event: { type: 'objects', name: 'ticket', state: 'active', organizationId: null }, + }, + publishedAt: Date.now(), + }); + // The applier is fire-and-forget on the subscribe path; its work is a + // short chain of already-resolved fakes, so settle it via the queue. + await vi.waitFor(() => expect(registeredItems.length).toBeGreaterThan(0)); + + expect(objectKeys(registeredItems)).toEqual(['object']); + expect(registeredItems).not.toContainEqual( + expect.objectContaining({ type: 'objects' }), + ); + }); }); // ═══════════════════════════════════════════════════════════════════════════ @@ -490,9 +543,9 @@ describe('[#8862] the `applyRegistryWriteThrough` caller set stays closed', () = 'utf8', ); - it('has exactly four call sites, all traced in this file’s header', () => { + it('has exactly five call sites, all traced in this file’s header', () => { const callSites = source.match(/this\.applyRegistryWriteThrough\(/g) ?? []; - expect(callSites).toHaveLength(4); + expect(callSites).toHaveLength(5); }); it('`applyObjectRegistryMutation` is reached only through the write-through', () => { diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 5e75b9816b..d7ba022562 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -78,6 +78,10 @@ import { type DroppedFieldsEvent, type QueryAST, type EngineQueryOptionsParsed, } from '@objectstack/spec/data'; import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL, canonicalMetaUrlType, metaUrlSpellingRefusal, unrecognisedMetaTypeRefusal, METADATA_ITEM_NAME_PATTERN } from '@objectstack/spec/shared'; +// [#13331] The cluster fan-out transport type only — the protocol never +// depends on `@objectstack/service-cluster`; a bridge plugin there hands the +// live transport in through `attachMetadataMutationPubSub`. +import type { IPubSub } from '@objectstack/spec/contracts'; import { applyConversionsToStoredItem, type ConversionNotice } from '@objectstack/spec'; import { type FormView, isAggregatedViewContainer, expandViewContainer } from '@objectstack/spec/ui'; // [#11350] Emitted-specifier pin. This module's inferred public declarations @@ -3985,6 +3989,39 @@ export interface MetadataMutationEvent { organizationId?: string | null; } +/** + * [#13331] Cluster channel for post-persistence metadata mutations — the + * cross-node half of {@link MetadataMutationEvent}. + * + * Published by the protocol that performed the write (see + * `attachMetadataMutationPubSub`) and consumed by every peer replica to + * CONVERGE its in-memory registry from its OWN `sys_metadata` read. Distinct + * from `MetadataManager`'s `metadata.changed` channel on purpose: that one + * replays watch events into the metadata SERVICE's caches, while this one + * repairs the ObjectQL engine registry behind the data plane — the state the + * runtime authoring path (`saveMetaItem` → `applyRegistryWriteThrough`) + * mutates without ever touching the metadata service. + */ +export const METADATA_MUTATION_CLUSTER_CHANNEL = 'metadata.mutated'; + +/** + * [#13331] Payload for {@link METADATA_MUTATION_CLUSTER_CHANNEL}. + * + * ⭐ A SIGNAL, never trusted content (ruled 2026-09-01): the receiving replica + * re-reads the row from its own `sys_metadata` and re-runs the registry + * write-through from that read, so the shared database stays the single + * source of truth. `event` carries only the row's ADDRESS (type/name/state/ + * org scope); no body ever rides this channel, which is also what makes + * at-least-once delivery and duplicates harmless by construction — replaying + * the same address converges to the same registry state. + */ +export interface ClusterMetadataMutationPayload { + /** Origin nodeId — used for loopback suppression. */ + originNode?: string; + /** The post-persistence mutation notification, address-only. */ + event: MetadataMutationEvent; +} + /** * [#10219] A single item reached `active` through the per-item publish door * (`publishMetaItem`, i.e. `POST /api/v1/meta/:type/:name/publish`). `type` is @@ -4908,8 +4945,30 @@ export class ObjectStackProtocolImplementation implements * Notify mutation listeners (best-effort, synchronous fan-out). A * listener failure must never fail the write it observes — the row is * already persisted — so each listener is isolated in its own try/catch. + * + * [#13331] Also the cluster publish point: this method is called at + * exactly the post-persistence sites (`saveMetaItem`, + * `runPublishSideEffects`, `deleteMetaItem`) — the ONE choke point the + * listener bus above documents — so publishing here is what makes every + * authoring transport inherit the cross-node signal instead of each HTTP + * surface hand-announcing. Local listeners first, then the bus, matching + * `MetadataManager.notifyWatchers`' order. */ private emitMetadataMutation(evt: MetadataMutationEvent): void { + this.notifyMutationListenersLocal(evt); + this.publishMetadataMutation(evt); + } + + /** + * The LOCAL half of {@link emitMetadataMutation} — split out (#13331, + * mirroring `MetadataManager.notifyWatchersLocal`) so a mutation received + * FROM the cluster can be replayed to this replica's listeners without + * re-publishing it: replaying through `emitMetadataMutation` would + * publish again from this node, every peer would apply-and-republish in + * turn, and the loopback guard (which only suppresses a node's OWN + * messages) could not stop the storm. + */ + private notifyMutationListenersLocal(evt: MetadataMutationEvent): void { for (const listener of this.metadataMutationListeners) { try { listener(evt); @@ -4922,6 +4981,192 @@ export class ObjectStackProtocolImplementation implements } } + // ── [#13331] Cross-node registry convergence ──────────────────────────── + // + // The gap this closes, measured on a live 3-replica EE deployment: a + // runtime-authored object persists to the SHARED `sys_metadata` (so + // `/api/v1/meta/*` answers 200 fleet-wide) but registers with the ObjectQL + // engine registry of the WRITING replica only — `assertObjectRegistered` + // fails closed on the other replicas and `/api/v1/data/` answers + // OBJECT_NOT_FOUND on (N-1)/N of the fleet, indefinitely (200 concurrent + // creates through the LB: 67×201 / 133×404; boot-loaded control object: + // 0 errors). The authoring path lives entirely in this protocol and never + // touches the metadata service, so `MetadataManager`'s `metadata.changed` + // bridge — even when attached — never hears these writes. The protocol is + // the state owner here, and owns the fan-out (ruled 2026-09-01, Option A). + + /** The cluster transport, when a bridge attached one. [#13331] */ + private clusterPubSub?: IPubSub; + /** This node's cluster id — stamps `originNode` for loopback suppression. */ + private clusterNodeId?: string; + /** Disposer for the cluster subscription, when attached. */ + private clusterUnsubscribe?: () => void; + + /** + * [#13331] Attach a cluster pub/sub transport so this protocol's + * post-persistence mutations fan out on + * {@link METADATA_MUTATION_CLUSTER_CHANNEL} and peer mutations converge + * this replica's registry. Mirrors `MetadataManager.attachClusterPubSub()` + * and the engine's `attachAuthzInvalidationPubSub()` — including their + * idempotency on the `(pubsub, nodeId)` pair — and is called the same + * way: by `MetadataClusterBridgePlugin` in `@objectstack/service-cluster`, + * once per kernel boot at `kernel:ready`, after both services exist. + * + * ⭐ Receipt runs CONVERGENCE, not trust: the payload is an address, and + * {@link applyRemoteMetadataMutation} re-reads the row from this + * replica's own `sys_metadata` before re-running the registry + * write-through. Duplicates are harmless (same read, same registration); + * loss is bounded the way it always was — by the next boot's full reload + * — because no shipped driver exceeds at-most-once delivery + * (`IPubSub`'s own contract). This channel narrows the staleness window + * from "until restart" to "one network hop"; it does not promise more. + * + * @returns a disposer that detaches the bridge. + */ + attachMetadataMutationPubSub(pubsub: IPubSub, nodeId: string): () => void { + if (this.clusterPubSub === pubsub && this.clusterNodeId === nodeId) { + return () => this.detachMetadataMutationPubSub(); + } + this.detachMetadataMutationPubSub(); + this.clusterPubSub = pubsub; + this.clusterNodeId = nodeId; + this.clusterUnsubscribe = pubsub.subscribe( + METADATA_MUTATION_CLUSTER_CHANNEL, + (msg) => { + const p = msg.payload; + // Loopback guard — never re-apply what this node just wrote: + // the write-through already ran here, synchronously, at the + // door that persisted the row. + if (p?.originNode && p.originNode === this.clusterNodeId) return; + if (!p?.event?.type || !p.event.name) return; + // Drafts are a staging buffer and never hydrate into any + // registry — the publisher skips them (matching + // `applyRegistryWriteThrough`'s `mode === 'publish'` gate), + // and this guard keeps a peer honest about the same rule. + if (p.event.state === 'draft') return; + void this.applyRemoteMetadataMutation(p.event).catch((err) => { + console.warn( + `[Protocol] cluster metadata-mutation apply failed for ` + + `${p.event.type}/${p.event.name}: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + }, + ); + console.info( + `[Protocol] attached to the ${METADATA_MUTATION_CLUSTER_CHANNEL} cluster channel (node=${nodeId})`, + ); + return () => this.detachMetadataMutationPubSub(); + } + + /** Tear down the cluster wiring. Safe to call multiple times. [#13331] */ + detachMetadataMutationPubSub(): void { + if (this.clusterUnsubscribe) { + try { this.clusterUnsubscribe(); } catch { /* idempotent */ } + this.clusterUnsubscribe = undefined; + } + this.clusterPubSub = undefined; + this.clusterNodeId = undefined; + } + + /** + * [#13331] The cluster half of {@link emitMetadataMutation}: publish the + * mutation's ADDRESS to peers. Best-effort and fire-and-forget — a + * publish failure must never fail the write it announces (the row is + * already persisted), matching `MetadataManager.notifyWatchers`' cluster + * leg verbatim. No-op until a bridge attaches a transport. + * + * Drafts are not published: they never enter any replica's registry + * (including this one's — `saveMetaItem` gates its write-through on + * `mode === 'publish'`), so a draft signal would wake every peer to + * converge on a no-op. + */ + private publishMetadataMutation(evt: MetadataMutationEvent): void { + if (!this.clusterPubSub) return; + if (evt.state === 'draft') return; + const payload: ClusterMetadataMutationPayload = { + originNode: this.clusterNodeId, + event: evt, + }; + void this.clusterPubSub + .publish(METADATA_MUTATION_CLUSTER_CHANNEL, payload, { + partitionKey: `${evt.type}:${evt.name}`, + }) + .catch((err) => { + console.warn( + `[Protocol] cluster metadata-mutation publish failed for ` + + `${evt.type}/${evt.name}: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + } + + /** + * [#13331] Converge this replica's registry after a PEER's metadata + * mutation — the ruled receipt semantics: re-run the registry + * write-through from THIS replica's own `sys_metadata` read, never from + * the payload. + * + * One applier for both `active` and `deleted` signals, because the DB + * read decides, not the event name: an active row present means the + * registry must serve it (a save/publish — or a draft DISCARD whose + * active overlay survives, which re-registers the same body and is + * idempotent); no active row means the heal walk must run + * ({@link restoreArtifactRegistryView}, the same walk the local delete + * runs). That makes out-of-order and duplicate delivery converge to the + * row's current state by construction. + * + * The write-through call makes this the FIFTH caller of + * `applyRegistryWriteThrough` — traced, with the fold discipline the + * other four follow, in + * `protocol.object-registry-write-through-spelling.test.ts`. The fold + * here mirrors `revertCommit`'s call-site fold: the event crossed a + * process boundary, so this caller folds rather than trusting the wire — + * through {@link canonicalMetaType} (the complete map, #9161), not the + * manifest map. + * + * Scope parity is INHERITED, not re-decided: `applyRegistryWriteThrough` + * carries the org/environment gates (#6602: an org-scoped overlay never + * enters the shared registry; the non-object branch stays + * control-plane-only) and `restoreArtifactRegistryView` refuses org + * scopes on its own — so a peer applies exactly what the writer's own + * kernel shape would have applied locally. + * + * After convergence the event replays into this replica's LOCAL mutation + * listeners (never re-published — see + * {@link notifyMutationListenersLocal}), so boot-cached consumers wired + * to `onMetadataMutation` (authored hook/action re-bind) re-sync from + * their own reads exactly as they would after a local write. Registry + * first, listeners second — the #5109 invalidate-before-notify rule: a + * listener that re-reads must not observe the event and the pre-event + * registry at the same time. + */ + private async applyRemoteMetadataMutation(evt: MetadataMutationEvent): Promise { + const type = canonicalMetaType(evt.type); + const orgId = evt.organizationId ?? null; + const repo = this.getOverlayRepo(orgId); + const ref = { + type, + name: evt.name, + org: orgId ?? 'env', + } as Parameters[0]; + const current = await repo.get(ref, { state: 'active' }); + if (current) { + // The row's OWN package binding, read from this replica's DB the + // way the recovery callers derive it (#4867 / #4636) — the wire + // carries no binding, and must not. + const packageId = await this.resolveOverlayPackageBinding(type, evt.name, orgId); + this.applyRegistryWriteThrough({ + type, + name: evt.name, + item: current.body, + packageId, + organizationId: orgId, + }); + } else { + await this.restoreArtifactRegistryView(type, evt.name, orgId); + } + this.notifyMutationListenersLocal({ ...evt, type }); + } + /** * [#10219] Per-item publish listeners — the producer-side half of the * publish→re-bind signal. diff --git a/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.test.ts b/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.test.ts new file mode 100644 index 0000000000..9a5bb0b48e --- /dev/null +++ b/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.test.ts @@ -0,0 +1,214 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#13331] `MetadataClusterBridgePlugin` — first test file for this plugin, + * written with the protocol lane it gains here. + * + * The composition case that matters most is the shipped EE shape (⭐ below): + * a TS-config host boot fills the `metadata` slot with the kernel's in-memory + * core fallback (no `attachClusterPubSub`) while the `protocol` service is the + * real `ObjectStackProtocolImplementation`. Pre-fix, that boot warned + * "cross-node cache invalidation disabled" and attached NOTHING — and every + * replica but the writer answered OBJECT_NOT_FOUND for runtime-authored + * objects, indefinitely. The mutation lane must attach exactly there, without + * the metadata-service lane's absence taking it down. + * + * ⚠️ Lane 1's in-process-driver behaviour (it attaches and logs "bridged" on + * the memory driver that fans out to nobody) is #14021's card, NOT pinned + * here — these cases drive lane 1 only through its warn/absence paths so that + * card stays free to fix it. Lane 2 carries the `isInProcessClusterDriver` + * guard from birth, and that IS pinned here. + */ + +import { describe, it, expect, vi } from 'vitest'; +import type { PluginContext } from '@objectstack/core'; +import { MetadataClusterBridgePlugin } from './metadata-cluster-bridge-plugin.js'; + +interface HarnessOptions { + /** + * Cluster driver name, or `null` for "no cluster service registered" — a + * SENTINEL, not `undefined`, because an explicit `undefined` re-applies + * the destructuring default (#6621's shape, warned about in + * `protocol.delete-object-registry-unregister.test.ts`). + */ + driver?: string | null; + /** + * The `metadata` slot: `'none'` (getService throws), `'fallback'` (present, + * no attachClusterPubSub — the host-config boot's core fallback), or + * `'manager'` (exposes attachClusterPubSub). + */ + metadata?: 'none' | 'fallback' | 'manager'; + /** + * The `protocol` slot: `'none'`, `'bare'` (present, no + * attachMetadataMutationPubSub — an older or foreign implementation), or + * `'real'` (exposes attachMetadataMutationPubSub). + */ + protocol?: 'none' | 'bare' | 'real'; +} + +function makeHarness(opts: HarnessOptions = {}) { + const { driver = 'redis', metadata = 'fallback', protocol = 'real' } = opts; + + const logger = { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const detachMetadata = vi.fn(); + const attachMetadata = vi.fn((_pubsub: unknown, _nodeId: string) => detachMetadata); + const detachMutation = vi.fn(); + const attachMutation = vi.fn((_pubsub: unknown, _nodeId: string) => detachMutation); + + const pubsub = { publish: vi.fn(), subscribe: vi.fn(), close: vi.fn() }; + const cluster = + driver === null + ? undefined + : { nodeId: 'node-a', driver, pubsub, lock: {}, kv: {}, counter: {}, close: vi.fn() }; + + const metadataService = + metadata === 'none' ? undefined + : metadata === 'fallback' ? { get: vi.fn(), list: vi.fn() } + : { attachClusterPubSub: attachMetadata }; + const protocolService = + protocol === 'none' ? undefined + : protocol === 'bare' ? { saveMetaItem: vi.fn() } + : { attachMetadataMutationPubSub: attachMutation }; + + const hooks = new Map Promise | void>>(); + const ctx = { + logger, + hook(name: string, handler: () => Promise | void) { + const list = hooks.get(name) ?? []; + list.push(handler); + hooks.set(name, list); + }, + getService(name: string) { + if (name === 'cluster') { + if (!cluster) throw new Error('service not found: cluster'); + return cluster; + } + if (name === 'metadata') { + if (!metadataService) throw new Error('service not found: metadata'); + return metadataService; + } + if (name === 'protocol') { + if (!protocolService) throw new Error('service not found: protocol'); + return protocolService; + } + throw new Error(`service not found: ${name}`); + }, + } as unknown as PluginContext; + + const fire = async (name: string) => { + for (const h of hooks.get(name) ?? []) await h(); + }; + + return { + ctx, logger, fire, pubsub, + attachMetadata, detachMetadata, attachMutation, detachMutation, + }; +} + +const infoLines = (h: ReturnType) => + h.logger.info.mock.calls.map((c) => String(c[0])); +const warnLines = (h: ReturnType) => + h.logger.warn.mock.calls.map((c) => String(c[0])); + +describe('[#13331] ⭐ the shipped EE shape — fallback metadata slot, real protocol', () => { + it('warns for lane 1 AND attaches lane 2 in the same boot', async () => { + const h = makeHarness({ driver: 'redis', metadata: 'fallback', protocol: 'real' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + // Lane 1's statement stays true and VERBATIM on this boot: the + // metadata SERVICE has no cluster seam there. (The card's original + // boot symptom — the line other measurements matched byte-for-byte.) + expect(warnLines(h)).toContain( + 'MetadataClusterBridgePlugin: metadata service does not expose attachClusterPubSub(); cross-node cache invalidation disabled', + ); + // …and pre-fix that warn was the END of the story. Now the mutation + // lane attaches regardless of lane 1's outcome. + expect(h.attachMutation).toHaveBeenCalledTimes(1); + expect(h.attachMutation).toHaveBeenCalledWith(h.pubsub, 'node-a'); + expect(infoLines(h).some((l) => l.includes('bridged metadata.mutated'))).toBe(true); + }); + + it('a MISSING metadata service does not take the mutation lane down either', async () => { + const h = makeHarness({ driver: 'redis', metadata: 'none', protocol: 'real' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + expect(h.attachMutation).toHaveBeenCalledTimes(1); + }); +}); + +describe('[#13331] the mutation lane’s own guards', () => { + it('skips attach on the in-process memory driver — no peers to reach', async () => { + const h = makeHarness({ driver: 'memory', protocol: 'real' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + // The guard `AuthzClusterBridgePlugin` uses, applied from birth: an + // in-process bus fans out to nobody, and logging "bridged" over it is + // the misreading #14021 records for lane 1. + expect(h.attachMutation).not.toHaveBeenCalled(); + expect(infoLines(h).some((l) => l.includes('metadata.mutated'))).toBe(false); + }); + + it('skips quietly when no protocol service is registered', async () => { + const h = makeHarness({ driver: 'redis', protocol: 'none' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + expect(h.attachMutation).not.toHaveBeenCalled(); + expect(h.logger.error).not.toHaveBeenCalled(); + }); + + it('skips quietly when the protocol does not expose the seam', async () => { + const h = makeHarness({ driver: 'redis', protocol: 'bare' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + expect(h.attachMutation).not.toHaveBeenCalled(); + expect(h.logger.error).not.toHaveBeenCalled(); + }); + + it('no cluster service at all skips BOTH lanes', async () => { + const h = makeHarness({ driver: null, metadata: 'manager', protocol: 'real' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + expect(h.attachMetadata).not.toHaveBeenCalled(); + expect(h.attachMutation).not.toHaveBeenCalled(); + }); +}); + +describe('[#13331] both lanes present, and both released on shutdown', () => { + it('a manager metadata service and a real protocol both attach', async () => { + const h = makeHarness({ driver: 'redis', metadata: 'manager', protocol: 'real' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + + expect(h.attachMetadata).toHaveBeenCalledWith(h.pubsub, 'node-a'); + expect(h.attachMutation).toHaveBeenCalledWith(h.pubsub, 'node-a'); + expect(warnLines(h)).toEqual([]); + }); + + it('kernel:shutdown detaches both lanes', async () => { + const h = makeHarness({ driver: 'redis', metadata: 'manager', protocol: 'real' }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + await h.fire('kernel:shutdown'); + + expect(h.detachMetadata).toHaveBeenCalledTimes(1); + expect(h.detachMutation).toHaveBeenCalledTimes(1); + }); + + it('a throwing lane-1 detach does not strand lane 2’s', async () => { + const h = makeHarness({ driver: 'redis', metadata: 'manager', protocol: 'real' }); + h.detachMetadata.mockImplementation(() => { throw new Error('detach exploded'); }); + await new MetadataClusterBridgePlugin().init(h.ctx); + await h.fire('kernel:ready'); + await h.fire('kernel:shutdown'); + + expect(h.detachMutation).toHaveBeenCalledTimes(1); + expect(h.logger.error).toHaveBeenCalled(); + }); +}); diff --git a/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.ts b/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.ts index 4e427c1c32..45652e3097 100644 --- a/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.ts +++ b/packages/services/service-cluster/src/metadata-cluster-bridge-plugin.ts @@ -2,23 +2,47 @@ import type { Plugin, PluginContext } from '@objectstack/core'; import type { IClusterService } from '@objectstack/spec/contracts'; +import { isInProcessClusterDriver } from './split-brain-guard.js'; /** - * Bridges the cluster pub/sub bus to the metadata service so that + * Bridges the cluster pub/sub bus to the metadata state owners so that * metadata mutations on one node invalidate registry caches on peer * nodes. Implements the "first real consumer" of the cluster API. * * Implementation detail: this plugin lives in `@objectstack/service-cluster` - * (not in `@objectstack/metadata`) to avoid forcing every metadata - * consumer to pull the cluster service. The metadata package only needs - * the `IPubSub` interface, which lives in `@objectstack/spec/contracts`. + * (not in `@objectstack/metadata` / `@objectstack/metadata-protocol`) to + * avoid forcing every metadata consumer to pull the cluster service. The + * state-owner packages only need the `IPubSub` interface, which lives in + * `@objectstack/spec/contracts`. * - * Activates only when both services are present and the metadata service - * exposes `attachClusterPubSub()`. Late binding is achieved via the + * TWO lanes, late-bound independently at `kernel:ready`, because the state + * that goes stale lives in two different owners (#13331): + * + * 1. **Metadata service** (`attachClusterPubSub()` — `metadata.changed`): + * replays watch events into peer `MetadataManager` caches + * (registry/list-cache invalidation, #5109). Only a real manager exposes + * the seam; the host-config boot's in-memory core fallback does not, and + * the warn below says so. + * 2. **Metadata protocol** (`attachMetadataMutationPubSub()` — + * `metadata.mutated`): fans the protocol's post-persistence mutation + * signal out to peers, which re-run the ObjectQL registry write-through + * from their OWN `sys_metadata` read. This is the lane behind the data + * plane: without it, an object authored at runtime through + * `PUT /api/v1/meta/*` answers OBJECT_NOT_FOUND on every replica that + * did not perform the write, until restart (#13331 — measured on a + * 3-replica EE deployment: 200 concurrent creates through the LB gave + * 67×201 / 133×404). The lanes are independent on purpose: the boot + * shape that lacks lane 1 (host-config, fallback metadata slot) is + * exactly the shipped EE shape that needs lane 2. + * + * Activates each lane only when the cluster service and that lane's state + * owner are present and expose the seam. Late binding is achieved via the * `kernel:ready` lifecycle hook. * - * Channel: `metadata.changed` — payload shape defined by - * `ClusterMetadataChangedPayload` in `@objectstack/metadata`. + * Channels: `metadata.changed` — payload shape defined by + * `ClusterMetadataChangedPayload` in `@objectstack/metadata`; + * `metadata.mutated` — payload shape defined by + * `ClusterMetadataMutationPayload` in `@objectstack/metadata-protocol`. * * See `content/docs/kernel/cluster.mdx` §5. */ @@ -28,11 +52,11 @@ export class MetadataClusterBridgePlugin implements Plugin { type = 'standard'; private detach?: () => void; + private detachMutation?: () => void; async init(ctx: PluginContext): Promise { ctx.hook('kernel:ready', async () => { let cluster: IClusterService | undefined; - let md: unknown; try { cluster = ctx.getService('cluster'); } catch { @@ -41,50 +65,128 @@ export class MetadataClusterBridgePlugin implements Plugin { ); return; } - try { - md = ctx.getService('metadata'); - } catch { - ctx.logger.debug( - 'MetadataClusterBridgePlugin: no "metadata" service registered, skipping', - ); - return; - } - - const attach = (md as { attachClusterPubSub?: unknown }) - .attachClusterPubSub; - if (typeof attach !== 'function') { - ctx.logger.warn( - 'MetadataClusterBridgePlugin: metadata service does not expose attachClusterPubSub(); cross-node cache invalidation disabled', - ); - return; - } + this.attachMetadataServiceLane(ctx, cluster); + this.attachProtocolLane(ctx, cluster); + }); + ctx.hook('kernel:shutdown', async () => { try { - this.detach = (attach as ( - pubsub: IClusterService['pubsub'], - nodeId: string, - ) => () => void).call(md, cluster.pubsub, cluster.nodeId); - ctx.logger.info( - `MetadataClusterBridgePlugin: bridged metadata.changed → cluster.pubsub (node=${cluster.nodeId})`, - ); + this.detach?.(); } catch (err) { ctx.logger.error( - 'MetadataClusterBridgePlugin: attach failed', + 'MetadataClusterBridgePlugin: detach error', err as Error, ); } - }); - - ctx.hook('kernel:shutdown', async () => { + this.detach = undefined; try { - this.detach?.(); + this.detachMutation?.(); } catch (err) { ctx.logger.error( - 'MetadataClusterBridgePlugin: detach error', + 'MetadataClusterBridgePlugin: mutation-lane detach error', err as Error, ); } - this.detach = undefined; + this.detachMutation = undefined; }); } + + /** + * Lane 1 — the metadata SERVICE's `metadata.changed` bridge, exactly as + * it has always behaved (its log lines are measured facts other cards + * lean on — the warn below is #13331's original boot symptom, and it + * remains TRUE on the host-config boot: the fallback metadata slot has + * no cluster seam, so metadata-SERVICE cache invalidation stays off + * there. The data-plane registry gap that warn used to imply is what + * lane 2 closes.) + */ + private attachMetadataServiceLane(ctx: PluginContext, cluster: IClusterService): void { + let md: unknown; + try { + md = ctx.getService('metadata'); + } catch { + ctx.logger.debug( + 'MetadataClusterBridgePlugin: no "metadata" service registered, skipping', + ); + return; + } + + const attach = (md as { attachClusterPubSub?: unknown }) + .attachClusterPubSub; + if (typeof attach !== 'function') { + ctx.logger.warn( + 'MetadataClusterBridgePlugin: metadata service does not expose attachClusterPubSub(); cross-node cache invalidation disabled', + ); + return; + } + + try { + this.detach = (attach as ( + pubsub: IClusterService['pubsub'], + nodeId: string, + ) => () => void).call(md, cluster.pubsub, cluster.nodeId); + ctx.logger.info( + `MetadataClusterBridgePlugin: bridged metadata.changed → cluster.pubsub (node=${cluster.nodeId})`, + ); + } catch (err) { + ctx.logger.error( + 'MetadataClusterBridgePlugin: attach failed', + err as Error, + ); + } + } + + /** + * Lane 2 — the metadata PROTOCOL's `metadata.mutated` fan-out (#13331). + * + * Duck-typed exactly like lane 1 feature-detects `attachClusterPubSub()`: + * this package must not depend on `@objectstack/metadata-protocol`. + * + * Guarded on {@link isInProcessClusterDriver} from birth (the shape + * `AuthzClusterBridgePlugin` uses): the in-process memory driver fans out + * to nobody, so "attached" there would be the misreading #14021 records + * for lane 1's info line — a new lane does not inherit a known defect. + */ + private attachProtocolLane(ctx: PluginContext, cluster: IClusterService): void { + let protocol: unknown; + try { + protocol = ctx.getService('protocol'); + } catch { + ctx.logger.debug( + 'MetadataClusterBridgePlugin: no "protocol" service registered, skipping mutation fan-out', + ); + return; + } + + const attach = (protocol as { attachMetadataMutationPubSub?: unknown }) + .attachMetadataMutationPubSub; + if (typeof attach !== 'function') { + ctx.logger.debug( + 'MetadataClusterBridgePlugin: protocol service does not expose attachMetadataMutationPubSub(), skipping mutation fan-out', + ); + return; + } + + if (isInProcessClusterDriver(cluster.driver)) { + ctx.logger.debug( + `MetadataClusterBridgePlugin: cluster driver "${cluster.driver}" is in-process; mutation fan-out has no peers to reach, skipping`, + ); + return; + } + + try { + this.detachMutation = (attach as ( + pubsub: IClusterService['pubsub'], + nodeId: string, + ) => () => void).call(protocol, cluster.pubsub, cluster.nodeId); + ctx.logger.info( + `MetadataClusterBridgePlugin: bridged metadata.mutated → cluster.pubsub (node=${cluster.nodeId})`, + ); + } catch (err) { + ctx.logger.error( + 'MetadataClusterBridgePlugin: mutation-lane attach failed', + err as Error, + ); + } + } } diff --git a/scripts/engine-double-contract.pinned.json b/scripts/engine-double-contract.pinned.json index c1d6016491..62347b940e 100644 --- a/scripts/engine-double-contract.pinned.json +++ b/scripts/engine-double-contract.pinned.json @@ -351,6 +351,21 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/protocol.cluster-mutation-fanout.test.ts", + "verb": "delete", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/protocol.cluster-mutation-fanout.test.ts", + "verb": "findOne", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/protocol.cluster-mutation-fanout.test.ts", + "verb": "update", + "pinned": 1 + }, { "file": "packages/metadata-protocol/src/protocol.code-only-types.test.ts", "verb": "delete",