test(metadata): pin default memory cluster driver's cross-process isolation (#13609 measurement)

packages/metadata/src/metadata-manager-cluster.test.ts

@@ -399,3 +399,115 @@ describe('MetadataManager — a cluster peer write invalidates local caches (#51
         expect(cachedTypes(a)).toEqual(['view']);
     });
 });
+
+/**
+ * #13609 — the seam behind "delete keeps being served cluster-wide, longer
+ * than any TTL", measured statically (no live multi-node deployment
+ * available; see the issue for the full write-up).
+ *
+ * Every test above in this file shares ONE `TestPubSub` bus between A and
+ * B — the correct model of a WORKING cross-node transport (a real `redis` /
+ * `postgres` cluster driver), and it is why `unregister()`'s cluster fan-out
+ * (`notifyWatchers` → `CLUSTER_CHANNEL` → the peer's `invalidateForForeignWrite`)
+ * checks out as correct: register() and unregister() call it identically, so
+ * the #13405 reading ("create broadcasts, delete does not mirror") is false
+ * as a claim about this mechanism.
+ *
+ * But `Runtime`'s shipped DEFAULT (`cluster` option omitted) is
+ * `defineCluster({})` → `driver: 'memory'`, and `MemoryPubSub`'s own
+ * doc-comment is explicit: "No cross-process delivery — use the redis /
+ * postgres / nats driver for real multi-node setups." Each of N replica
+ * PROCESSES constructs its OWN `MemoryPubSub` instance; nothing wires them
+ * together. The split-brain guard (`assertClusterDriverSafeForTopology`,
+ * `split-brain-guard.ts`) only fires when the operator has *declared*
+ * multi-node via `OS_EXPECT_MULTI_NODE` / `OS_CLUSTER_REPLICAS>1` — silent
+ * otherwise, by design (ADR-0010, path A).
+ *
+ * So the below gives A and B *separate* `TestPubSub` instances instead of
+ * `makeCluster()`'s shared one — the faithful in-process stand-in for two
+ * real OS processes each on the shipped default driver. No live cluster is
+ * needed to observe the isolation: it is a property of the transport object
+ * (no socket, no IPC — see `MemoryPubSub`'s full implementation), identical
+ * whether the two instances live in one test process or two real ones.
+ *
+ * Both replicas locally `register()` the row before the delete, matching
+ * what `restoreRuntimeDatasources` does on every replica's own boot: each
+ * reads the same durable `sys_metadata` row and calls `metadata.register()`
+ * LOCALLY — not via a broadcast. That is also the reconciling condition for
+ * the three-way tension: `readListUncached()` merges the in-memory registry
+ * BEFORE the loader and never overwrites a registry hit with a loader
+ * answer (see `metadata-manager.ts`). A **new** name a peer's registry has
+ * never seen falls through to the (shared, authoritative) loader on every
+ * read, so a create looks like it "reaches" peers within one `list()` call —
+ * no propagation required. A name a peer's registry already holds POSITIVE
+ * is never re-checked against the loader at all, so a stale positive
+ * survives every list-cache TTL window forever, not for ~30s. That asymmetry
+ * — not "delete doesn't broadcast" — is what #13405 actually observed.
+ */
+describe('MetadataManager — default `memory` cluster driver does not cross OS processes (#13609)', () => {
+    const dsRow = (name: string) => ({ name, driver: 'postgres', origin: 'runtime' as const });
+
+    it('control: WITH a working cross-node transport, unregister() on A evicts B immediately', async () => {
+        // `makeCluster()` shares one bus — models a real redis/postgres driver.
+        const { store, a, b } = makeCluster();
+        await store.save('datasource', 'ds_doomed', dsRow('ds_doomed'));
+        // Both replicas locally register at "boot", exactly like
+        // `restoreRuntimeDatasources` reading the same `sys_metadata` row.
+        await a.register('datasource', 'ds_doomed', dsRow('ds_doomed'));
+        await b.register('datasource', 'ds_doomed', dsRow('ds_doomed'));
+        expect(await b.get('datasource', 'ds_doomed')).toBeTruthy();
+
+        await a.unregister('datasource', 'ds_doomed');
+
+        // Positive control: the probe below (get + list, same door
+        // `/api/v1/meta/datasource` reads through) WOULD have seen this.
+        expect(await b.get('datasource', 'ds_doomed')).toBeUndefined();
+        expect(viewNames(await b.list('datasource'))).toEqual([]);
+    });
+
+    it('WITHOUT it (the shipped default across real replicas): B keeps serving the deleted row past 10 list-cache TTL windows', async () => {
+        vi.useFakeTimers();
+        try {
+            const store = new SharedStoreLoader();
+            const a = new MetadataManager({ formats: ['json'], loaders: [store] });
+            const b = new MetadataManager({ formats: ['json'], loaders: [store] });
+            // Two INDEPENDENT pubsub instances, not shared — the isolation
+            // `defineCluster({ driver: 'memory' })` (the omitted-config
+            // default) actually ships with across two real processes.
+            a.attachClusterPubSub(new TestPubSub(), 'node-A');
+            b.attachClusterPubSub(new TestPubSub(), 'node-B');
+
+            await store.save('datasource', 'ds_doomed', dsRow('ds_doomed'));
+            await a.register('datasource', 'ds_doomed', dsRow('ds_doomed'));
+            await b.register('datasource', 'ds_doomed', dsRow('ds_doomed'));
+            expect(await b.get('datasource', 'ds_doomed')).toBeTruthy();
+
+            // DELETE lands on A (e.g. the admin REST door). Storage-first,
+            // in-memory second (#5259): A is correct immediately, and the
+            // shared DB row is gone too.
+            await a.unregister('datasource', 'ds_doomed');
+            expect(await a.get('datasource', 'ds_doomed')).toBeUndefined();
+            expect(store.storage.get('datasource')?.has('ds_doomed')).toBe(false);
+
+            // B's broadcast never arrives (separate pubsub instance). Advance
+            // WAY past the list-cache TTL this manager applies to `list()` —
+            // if this were TTL-bound (the #5109 / A2.3 shape) it would have
+            // cleared by now. It has not: the stale entry lives in the
+            // REGISTRY, which carries no TTL at all, and every fresh
+            // `readListUncached()` re-derives the same answer because the
+            // registry hit is never checked against the (correct) loader.
+            const ttl = (MetadataManager as unknown as { LIST_CACHE_TTL_MS: number })
+                .LIST_CACHE_TTL_MS;
+            vi.advanceTimersByTime(ttl * 10);
+
+            // This is exactly what `/api/v1/meta/datasource` reads through
+            // (`MetadataProtocol` → `metadataService.list('datasource')`) and
+            // what the admin `listDatasourceRecords` reads for the
+            // detail/admin registry (`metadataOf()?.list('datasource')`).
+            expect(await b.get('datasource', 'ds_doomed')).toBeTruthy();
+            expect(viewNames(await b.list('datasource'))).toEqual(['ds_doomed']);
+        } finally {
+            vi.useRealTimers();
+        }
+    });
+});