From 083caaecebec8605badd5b40e2fe9557cd2a27dc Mon Sep 17 00:00:00 2001 From: Matteo Date: Fri, 11 Sep 2026 12:32:21 +0200 Subject: [PATCH 1/2] fix(mcp): page the boot-time tool load instead of holding two copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cloud instance crash-looped nine times on 10 Sep between 05:51 and 06:13 UTC, each time with "FATAL ERROR: Ineffective mark-compacts near heap limit", taking sign-in, sign-up and every tenant's MCP endpoint down with it. loadAllTools pulled every active connector with all of its enabled tools in one findMany. On cloud that is 578 connectors and 22,215 tools — about 118 MB of raw JSON before V8 object overhead — and the whole result set stayed live while the registry it fed was being built, so peak heap at boot was roughly twice the steady state. Steady state alone is ~1.6 GB against a 2048 MB old space, which left nothing for the transient copy. Read a page of connectors at a time so the raw rows for a page are collectable once registered. The registry itself is unchanged and still holds every tenant's tools; this only removes the duplicate. The registry growing linearly with tenants is the real ceiling and wants a bounded, per-organization cache — that is a larger change and is not attempted here. While extracting the per-connector registration, reloadConnectorTools turns out to have been a verbatim copy of the same 45 lines. Both now call registerConnectorTools, so they cannot drift. --- .../src/mcp-server/mcp-server.service.spec.ts | 76 +++++++ .../src/mcp-server/mcp-server.service.ts | 195 +++++++++--------- 2 files changed, 172 insertions(+), 99 deletions(-) diff --git a/packages/backend/src/mcp-server/mcp-server.service.spec.ts b/packages/backend/src/mcp-server/mcp-server.service.spec.ts index e42a5515..e653cd46 100644 --- a/packages/backend/src/mcp-server/mcp-server.service.spec.ts +++ b/packages/backend/src/mcp-server/mcp-server.service.spec.ts @@ -108,3 +108,79 @@ describe('McpServerService.jsonSchemaToZod', () => { expect(schema.parse({ q: 'Domoferm' })).toEqual({ q: 'Domoferm' }); }); }); + +/** + * `loadAllTools` pages through the connector table instead of pulling every + * tenant's connectors in one query — the single query materialised the whole + * result set alongside the registry it was filling, roughly doubling peak heap + * at boot. These lock in that the paging actually terminates, covers every + * connector exactly once, and never holds more than one page. + */ +describe('McpServerService.loadAllTools paging', () => { + const PAGE = (McpServerService as any).LOAD_PAGE_SIZE as number; + + /** A service with just enough wired up to run the paging loop. */ + function makeService(connectorIds: string[]) { + const svc: any = Object.create(McpServerService.prototype); + const calls: Array> = []; + const registered: string[] = []; + + svc.prisma = { + connector: { + findMany: jest.fn(async (args: Record) => { + calls.push(args); + const start = args.cursor + ? connectorIds.indexOf(args.cursor.id) + (args.skip ?? 0) + : 0; + return connectorIds + .slice(start, start + args.take) + .map((id) => ({ id, tools: [] })); + }), + }, + }; + svc.registerConnectorTools = (c: { id: string }) => registered.push(c.id); + + return { svc, calls, registered }; + } + + it('visits every connector exactly once, in order', async () => { + const ids = Array.from({ length: PAGE * 2 + 7 }, (_, i) => `c${i}`); + const { svc, registered } = makeService(ids); + + await svc.loadAllTools(); + + expect(registered).toEqual(ids); + }); + + it('never asks for more than one page at a time', async () => { + const ids = Array.from({ length: PAGE * 3 }, (_, i) => `c${i}`); + const { svc, calls } = makeService(ids); + + await svc.loadAllTools(); + + expect(calls.every((c) => c.take === PAGE)).toBe(true); + // First page has no cursor; every later one skips past the previous last id. + expect(calls[0].cursor).toBeUndefined(); + expect(calls.slice(1).every((c) => c.skip === 1 && c.cursor)).toBe(true); + }); + + it('stops on an exact multiple of the page size instead of looping forever', async () => { + const ids = Array.from({ length: PAGE * 2 }, (_, i) => `c${i}`); + const { svc, calls, registered } = makeService(ids); + + await svc.loadAllTools(); + + expect(registered).toHaveLength(PAGE * 2); + // Two full pages, then one more that comes back empty and ends the loop. + expect(calls).toHaveLength(3); + }); + + it('does nothing when there are no active connectors', async () => { + const { svc, calls, registered } = makeService([]); + + await svc.loadAllTools(); + + expect(registered).toEqual([]); + expect(calls).toHaveLength(1); + }); +}); diff --git a/packages/backend/src/mcp-server/mcp-server.service.ts b/packages/backend/src/mcp-server/mcp-server.service.ts index 54509a60..53708588 100644 --- a/packages/backend/src/mcp-server/mcp-server.service.ts +++ b/packages/backend/src/mcp-server/mcp-server.service.ts @@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config'; import { ModuleRef } from '@nestjs/core'; import { z } from 'zod'; import { McpStrategy, MCP_STRATEGY } from '@rekog/mcp-nest'; +import type { Connector, McpTool } from '../generated/prisma/client'; import { PrismaService } from '../common/prisma.service'; import { decrypt } from '../common/crypto/encryption.util'; import { getRequiredSecret } from '../common/secrets.util'; @@ -65,64 +66,105 @@ export class McpServerService implements OnModuleInit { ); } + /** + * How many connectors to pull from the database at a time in + * {@link loadAllTools}. Small enough that the raw Prisma rows for a page are + * garbage in between pages; large enough that a full boot is a few dozen + * round trips, not hundreds. + */ + private static readonly LOAD_PAGE_SIZE = 25; + + /** + * Register every enabled tool of every active connector, across all tenants. + * + * Read in pages rather than as one `findMany`. The registry itself is + * unavoidably large — on the cloud instance it holds ~22k tools, ~118 MB of + * raw JSON before V8 object overhead — but loading every connector in a + * single query ALSO materialised the whole result set at once, so peak heap + * at boot was roughly twice the steady state. That is what pushed the + * process past --max-old-space-size and crash-looped it nine times on + * 10 Sep. Paging keeps the transient half bounded to one page. + */ async loadAllTools(): Promise { - const connectors = await this.prisma.connector.findMany({ - where: { isActive: true }, - include: { tools: { where: { isEnabled: true } } }, - }); + let cursor: string | undefined; - for (const connector of connectors) { - for (const tool of connector.tools) { - const toolDef = { - id: tool.id, - connectorId: connector.id, - organizationId: connector.organizationId, - name: tool.name, - description: tool.description, - parameters: tool.parameters as Record, - connectorType: connector.type, - useProxy: tool.useProxy, - connectorConfig: { - baseUrl: connector.baseUrl, - authType: connector.authType, - authConfig: this.decryptAuthConfig(connector.authConfig), - headers: connector.headers as Record | undefined, - envVars: connector.envVars as Record | undefined, - specUrl: connector.specUrl ?? undefined, - config: connector.config as Record | undefined, - }, - endpointMapping: tool.endpointMapping as any, - responseMapping: tool.responseMapping as - | Record - | undefined, - outputSchema: tool.outputSchema as unknown, - annotations: tool.annotations as unknown, - }; + for (;;) { + const page = await this.prisma.connector.findMany({ + where: { isActive: true }, + include: { tools: { where: { isEnabled: true } } }, + orderBy: { id: 'asc' }, + take: McpServerService.LOAD_PAGE_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }); - // Register in our internal registry (for execution lookup) - this.toolRegistry.registerTool(toolDef); + if (page.length === 0) break; - // Strip params covered by env vars so the AI doesn't need to provide them - const envVars = connector.envVars as Record | undefined; - const effectiveSchema = this.stripEnvVarParams( - tool.parameters as Record, - envVars, - ); + for (const connector of page) { + this.registerConnectorTools(connector); + } - // Register as a native MCP tool so it appears directly in tools/list, - // but only the first time we see this name. The upstream library's - // McpRegistryService is single-tenant (one tool per name); our - // ToolRegistry resolves cross-org collisions at handler-dispatch - // time via getToolForOrg/getTool, so the second+ registration with - // the same name would just overwrite and emit a warning. - if (this.toolRegistry.countByName(tool.name) === 1) { - this.registerMcpTool( - tool.name, - tool.description, - effectiveSchema, - deriveToolAnnotations(toolDef), - ); - } + cursor = page[page.length - 1].id; + if (page.length < McpServerService.LOAD_PAGE_SIZE) break; + } + } + + /** + * Register one connector's enabled tools in both registries. Shared by the + * boot-time load and by {@link reloadConnectorTools} so the two can't drift. + */ + private registerConnectorTools( + connector: Connector & { tools: McpTool[] }, + ): void { + for (const tool of connector.tools) { + const toolDef = { + id: tool.id, + connectorId: connector.id, + organizationId: connector.organizationId, + name: tool.name, + description: tool.description, + parameters: tool.parameters as Record, + connectorType: connector.type, + useProxy: tool.useProxy, + connectorConfig: { + baseUrl: connector.baseUrl, + authType: connector.authType, + authConfig: this.decryptAuthConfig(connector.authConfig), + headers: connector.headers as Record | undefined, + envVars: connector.envVars as Record | undefined, + specUrl: connector.specUrl ?? undefined, + config: connector.config as Record | undefined, + }, + endpointMapping: tool.endpointMapping as any, + responseMapping: tool.responseMapping as + | Record + | undefined, + outputSchema: tool.outputSchema as unknown, + annotations: tool.annotations as unknown, + }; + + // Register in our internal registry (for execution lookup) + this.toolRegistry.registerTool(toolDef); + + // Strip params covered by env vars so the AI doesn't need to provide them + const envVars = connector.envVars as Record | undefined; + const effectiveSchema = this.stripEnvVarParams( + tool.parameters as Record, + envVars, + ); + + // Register as a native MCP tool so it appears directly in tools/list, + // but only the first time we see this name. The upstream library's + // McpRegistryService is single-tenant (one tool per name); our + // ToolRegistry resolves cross-org collisions at handler-dispatch + // time via getToolForOrg/getTool, so the second+ registration with + // the same name would just overwrite and emit a warning. + if (this.toolRegistry.countByName(tool.name) === 1) { + this.registerMcpTool( + tool.name, + tool.description, + effectiveSchema, + deriveToolAnnotations(toolDef), + ); } } } @@ -149,52 +191,7 @@ export class McpServerService implements OnModuleInit { }); if (connector && connector.isActive) { - for (const tool of connector.tools) { - const toolDef = { - id: tool.id, - connectorId: connector.id, - organizationId: connector.organizationId, - name: tool.name, - description: tool.description, - parameters: tool.parameters as Record, - connectorType: connector.type, - useProxy: tool.useProxy, - connectorConfig: { - baseUrl: connector.baseUrl, - authType: connector.authType, - authConfig: this.decryptAuthConfig(connector.authConfig), - headers: connector.headers as Record | undefined, - envVars: connector.envVars as Record | undefined, - specUrl: connector.specUrl ?? undefined, - config: connector.config as Record | undefined, - }, - endpointMapping: tool.endpointMapping as any, - responseMapping: tool.responseMapping as - | Record - | undefined, - outputSchema: tool.outputSchema as unknown, - annotations: tool.annotations as unknown, - }; - - this.toolRegistry.registerTool(toolDef); - - const envVars = connector.envVars as Record | undefined; - const effectiveSchema = this.stripEnvVarParams( - tool.parameters as Record, - envVars, - ); - // Same dedup rule as loadAllTools — register on the upstream - // single-tenant MCP registry only when this is the first tool - // with this name across all orgs/connectors. - if (this.toolRegistry.countByName(tool.name) === 1) { - this.registerMcpTool( - tool.name, - tool.description, - effectiveSchema, - deriveToolAnnotations(toolDef), - ); - } - } + this.registerConnectorTools(connector); } this.logger.log( From 9f87067e52770bdc11ce784184b4ecc9436a8af4 Mon Sep 17 00:00:00 2001 From: Matteo Date: Fri, 11 Sep 2026 13:15:17 +0200 Subject: [PATCH 2/2] fix(cloud): let the server .env raise the backend heap cap start.sh already honours NODE_MAX_OLD_SPACE_MB, but the cloud compose never passed it through, so the only way to give the backend more heap than the 2048 MB default was to rebuild the image. Plumb it, defaulting to the same 2048 so nothing changes for anyone who does not set it. --- docker-compose.cloud.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml index 79beb4a4..b5bb8b25 100644 --- a/docker-compose.cloud.yml +++ b/docker-compose.cloud.yml @@ -37,6 +37,12 @@ services: environment: - NODE_ENV=production - PORT=4000 + # V8 old-space cap for the backend, read by start.sh. Left unset it + # defaults to 2048, which suits a 4 GB host; the tool registry outgrew + # that and crash-looped the instance on 10 Sep. Raise it in the server + # .env when the host has the RAM to back it — roughly half of total is a + # safe ceiling, since postgres, redis, db-rest and caddy share the box. + - NODE_MAX_OLD_SPACE_MB=${NODE_MAX_OLD_SPACE_MB:-2048} - DEPLOYMENT_MODE=cloud - NEXT_PUBLIC_API_URL=https://${DOMAIN} - DATABASE_URL=postgresql://amcp:${POSTGRES_PASSWORD}@postgres:5432/anythingmcp