From 14fa30af6854327d02815cb63615e48be3efa949 Mon Sep 17 00:00:00 2001 From: mmarabel <166927047+mmarabel@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:35:05 +0200 Subject: [PATCH] Classify Cloudflare bot challenges as upstream_bot_challenge instead of rejected credentials A response carrying cf-mitigated: challenge was served by Cloudflare's edge before the request reached the API, so the credential was never checked. OpenAPI and GraphQL invocations reported it as connection_rejected with a re-authenticate prompt and the challenge page's HTML; OpenAPI health probes marked the connection expired. Detect the documented header in the SDK and fail such calls as upstream_bot_challenge with the Ray ID. OpenAPI health probes report the challenge as degraded. Fixes #2126 --- .changeset/upstream-bot-challenge.md | 7 ++ packages/core/sdk/src/index.ts | 6 + .../sdk/src/upstream-bot-challenge.test.ts | 64 ++++++++++ .../core/sdk/src/upstream-bot-challenge.ts | 91 ++++++++++++++ .../plugins/graphql/src/sdk/plugin.test.ts | 54 ++++++++ packages/plugins/graphql/src/sdk/plugin.ts | 17 +++ packages/plugins/openapi/src/sdk/backing.ts | 33 +++++ .../openapi/src/sdk/upstream-failures.test.ts | 116 ++++++++++++++++++ 8 files changed, 388 insertions(+) create mode 100644 .changeset/upstream-bot-challenge.md create mode 100644 packages/core/sdk/src/upstream-bot-challenge.test.ts create mode 100644 packages/core/sdk/src/upstream-bot-challenge.ts diff --git a/.changeset/upstream-bot-challenge.md b/.changeset/upstream-bot-challenge.md new file mode 100644 index 0000000000..0ba8584fa0 --- /dev/null +++ b/.changeset/upstream-bot-challenge.md @@ -0,0 +1,7 @@ +--- +"@executor-js/sdk": patch +"@executor-js/plugin-openapi": patch +"@executor-js/plugin-graphql": patch +--- + +A request that Cloudflare bot protection challenges before it reaches the API (`cf-mitigated: challenge`) now fails as `upstream_bot_challenge` with the Ray ID, instead of `connection_rejected` with a prompt to re-authenticate and the challenge page's HTML. OpenAPI health checks report such a probe as degraded rather than expired, since the credential was never checked. diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index 627d3de1de..cce0abaf7b 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -529,6 +529,12 @@ export { insufficientScopeFromEmbeddedJson, type InsufficientScopeDetection, } from "./insufficient-scope"; +export { + botChallengeMessage, + botChallengeToolFailure, + detectBotChallenge, + type BotChallengeDetection, +} from "./upstream-bot-challenge"; // Endpoint sanitization for span attributes — plugins stamping a user-supplied // endpoint must strip its credential-bearing parts first. diff --git a/packages/core/sdk/src/upstream-bot-challenge.test.ts b/packages/core/sdk/src/upstream-bot-challenge.test.ts new file mode 100644 index 0000000000..702e543b38 --- /dev/null +++ b/packages/core/sdk/src/upstream-bot-challenge.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { botChallengeToolFailure, detectBotChallenge } from "./upstream-bot-challenge"; + +describe("detectBotChallenge", () => { + it("detects a Cloudflare challenge and carries its Ray ID", () => { + expect( + detectBotChallenge({ + headers: { "cf-mitigated": "challenge", "cf-ray": "8f1a2b3c4d5e6f70-SJC" }, + }), + ).toEqual({ provider: "cloudflare", rayId: "8f1a2b3c4d5e6f70-SJC" }); + }); + + it("matches header names and the value case-insensitively", () => { + expect(detectBotChallenge({ headers: { "CF-Mitigated": " Challenge " } })).toEqual({ + provider: "cloudflare", + }); + }); + + it("does not classify without the cf-mitigated challenge header", () => { + // A Cloudflare-fronted origin's own 403 carries cf-ray but no mitigation. + expect( + detectBotChallenge({ headers: { server: "cloudflare", "cf-ray": "8f1a2b3c4d5e6f70-SJC" } }), + ).toBeNull(); + expect(detectBotChallenge({ headers: { "cf-mitigated": "block" } })).toBeNull(); + expect(detectBotChallenge({})).toBeNull(); + }); +}); + +describe("botChallengeToolFailure", () => { + it("is a non-authentication failure that tells the agent the credential was not checked", () => { + const result = botChallengeToolFailure({ + integration: { id: "example_api", scope: "user" }, + status: 403, + detection: { provider: "cloudflare", rayId: "8f1a2b3c4d5e6f70-SJC" }, + }); + + expect(result).toMatchObject({ + ok: false, + error: { + code: "upstream_bot_challenge", + status: 403, + retryable: false, + message: expect.stringMatching(/Ray ID 8f1a2b3c4d5e6f70-SJC.*credential was not checked/), + details: { + category: "upstream_protection", + integration: { id: "example_api", scope: "user" }, + upstream: { + status: 403, + provider: "cloudflare", + mitigation: "challenge", + rayId: "8f1a2b3c4d5e6f70-SJC", + }, + }, + }, + }); + const recovery = (result as { error: { details: { recovery: Record } } }).error + .details.recovery; + expect( + recovery.createConnectionTool, + "no reconnect hint: a new credential meets the same challenge", + ).toBeUndefined(); + }); +}); diff --git a/packages/core/sdk/src/upstream-bot-challenge.ts b/packages/core/sdk/src/upstream-bot-challenge.ts new file mode 100644 index 0000000000..3c24789e80 --- /dev/null +++ b/packages/core/sdk/src/upstream-bot-challenge.ts @@ -0,0 +1,91 @@ +// Detecting an upstream bot-protection challenge. A site behind Cloudflare +// can answer an API request with a challenge page ("Just a moment...") +// instead of forwarding it: a Managed Challenge, JS challenge, or Bot Fight +// Mode verdict against the caller's network. Hosted executors send from +// datacenter or Workers egress, which is exactly the traffic those rules +// target, so a key that works from a laptop can be challenged here. +// +// The challenge is served by the edge BEFORE the request reaches the API, so +// the credential was never evaluated. Folding it into connection_rejected +// (whose recovery tells the agent to re-authenticate) sends the user off to +// rotate a key that is fine; the fix is on the API operator's side. +// +// Detection is deliberately strict: it keys off the `cf-mitigated: challenge` +// response header, which Cloudflare documents as the signal for a challenged +// request and which an origin cannot produce by accident. The HTML body is +// not inspected: a page that merely mentions Cloudflare never classifies. A +// miss is benign: the failure stays on its existing classification. + +import { ToolResult } from "./tool-result"; + +export type BotChallengeDetection = { + readonly provider: "cloudflare"; + /** Cloudflare's per-request Ray ID (`cf-ray`), which the site operator can + * look up in their Security Events to see which rule challenged it. */ + readonly rayId?: string; +}; + +const headerValue = ( + headers: Record | undefined, + name: string, +): string | undefined => { + for (const [key, value] of Object.entries(headers ?? {})) { + if (key.toLowerCase() === name) return value; + } + return undefined; +}; + +/** Inspect an upstream response's headers for a bot-protection challenge. + * Returns `null` when nothing matches, so callers fall through to their + * existing classification. */ +export const detectBotChallenge = (input: { + readonly headers?: Record; +}): BotChallengeDetection | null => { + const mitigated = headerValue(input.headers, "cf-mitigated"); + if (mitigated?.trim().toLowerCase() !== "challenge") return null; + const rayId = headerValue(input.headers, "cf-ray")?.trim(); + return { provider: "cloudflare", ...(rayId ? { rayId } : {}) }; +}; + +/** One-line explanation of a challenged request, shared by tool failures and + * health-check details so both surfaces say the same thing. */ +export const botChallengeMessage = (input: { + readonly integration: string; + readonly status: number; + readonly detection: BotChallengeDetection; +}): string => + `Cloudflare bot protection in front of "${input.integration}" answered HTTP ${input.status} with a challenge (cf-mitigated: challenge${input.detection.rayId ? `, Ray ID ${input.detection.rayId}` : ""}), so the request never reached the API and the connection's credential was not checked. Re-authenticating will not help; the API operator must exempt API traffic from the challenge.`; + +/** The tool result for a challenged request: a typed, non-authentication + * failure that carries the Ray ID instead of the challenge page's HTML. */ +export const botChallengeToolFailure = (input: { + readonly integration: { readonly id: string; readonly scope?: string }; + readonly status: number; + readonly detection: BotChallengeDetection; +}): ToolResult => + ToolResult.fail({ + code: "upstream_bot_challenge", + status: input.status, + message: botChallengeMessage({ + integration: input.integration.id, + status: input.status, + detection: input.detection, + }), + // A challenge expects a browser to solve it; replaying the same request + // from the same egress gets the same verdict. + retryable: false, + details: { + category: "upstream_protection", + integration: input.integration, + upstream: { + status: input.status, + provider: input.detection.provider, + mitigation: "challenge", + ...(input.detection.rayId ? { rayId: input.detection.rayId } : {}), + }, + recovery: { + instructions: + "The upstream's bot protection blocked this request before authentication, so the connection's credential is not the problem. Do not ask the user to re-enter or rotate it. Tell them the API operator has to allow API traffic past the challenge, and pass on the Ray ID so the operator can find the event.", + }, + }, + }); diff --git a/packages/plugins/graphql/src/sdk/plugin.test.ts b/packages/plugins/graphql/src/sdk/plugin.test.ts index 45290fbfec..2e80acea4c 100644 --- a/packages/plugins/graphql/src/sdk/plugin.test.ts +++ b/packages/plugins/graphql/src/sdk/plugin.test.ts @@ -685,6 +685,60 @@ describe("graphqlPlugin real protocol server", () => { }), ); + it.effect( + "classifies a Cloudflare challenge as upstream_bot_challenge, not an auth failure", + () => + Effect.gen(function* () { + const server = yield* serveTestHttpApp((request) => + Effect.gen(function* () { + const webRequest = yield* HttpServerRequest.toWeb(request); + const body = yield* Effect.promise(() => webRequest.text()); + if (body.includes("__schema")) { + return HttpServerResponse.jsonUnsafe({ data: introspectionResult }); + } + return HttpServerResponse.text("Just a moment...", { + status: 403, + headers: { + "content-type": "text/html", + "cf-mitigated": "challenge", + "cf-ray": "8f1a2b3c4d5e6f70-SJC", + }, + }); + }), + ); + const executor = yield* makeExecutor(); + + yield* executor.graphql.addIntegration({ + endpoint: server.url("/graphql"), + slug: "challenged_graph", + }); + yield* createOrgConnection(executor, { + integration: "challenged_graph", + name: "main", + template: "none", + }); + + const result = yield* executor.execute( + toolAddr("challenged_graph", "main", "query.hello"), + { + name: "Ada", + }, + ); + + expect(result).toMatchObject({ + ok: false, + error: { + code: "upstream_bot_challenge", + status: 403, + details: { + category: "upstream_protection", + upstream: { rayId: "8f1a2b3c4d5e6f70-SJC" }, + }, + }, + }); + }), + ); + it.effect("invokes OAuth-backed integrations with a rendered bearer token", () => Effect.gen(function* () { const server = yield* serveGraphqlTestServer({ diff --git a/packages/plugins/graphql/src/sdk/plugin.ts b/packages/plugins/graphql/src/sdk/plugin.ts index 176e6d7721..f236854767 100644 --- a/packages/plugins/graphql/src/sdk/plugin.ts +++ b/packages/plugins/graphql/src/sdk/plugin.ts @@ -4,6 +4,8 @@ import { HttpClient } from "effect/unstable/http"; import { authToolFailure, + botChallengeToolFailure, + detectBotChallenge, detectInsufficientScope, AuthTemplateSlug, definePlugin, @@ -1434,6 +1436,21 @@ export const graphqlPlugin = definePlugin((options?: GraphqlPluginOptions) => { // gateway's OAuth error object), and even when it is, the transport // status is the authoritative signal — labelling it graphql_errors // would hide the credential problem from the agent entirely. + // + // A bot-protection challenge comes first of all: the edge answered + // before the request reached the endpoint, so neither the credential + // nor the GraphQL layer was involved. + const botChallenge = + result.status < 200 || result.status >= 300 + ? detectBotChallenge({ headers: result.headers }) + : null; + if (botChallenge) { + return botChallengeToolFailure({ + integration: { id: integration, scope: credential.owner }, + status: result.status, + detection: botChallenge, + }); + } if (result.status === 401 || result.status === 403) { // A scope-insufficient 403 is not fixable by re-authenticating // the same grant; give it its own code so the agent stops looping diff --git a/packages/plugins/openapi/src/sdk/backing.ts b/packages/plugins/openapi/src/sdk/backing.ts index 82cd6dcdba..1bd0d8fcfa 100644 --- a/packages/plugins/openapi/src/sdk/backing.ts +++ b/packages/plugins/openapi/src/sdk/backing.ts @@ -7,7 +7,10 @@ import { ToolName, ToolResult, authToolFailure, + botChallengeMessage, + botChallengeToolFailure, classifyProbeResponse, + detectBotChallenge, detectInsufficientScope, sortHealthCheckCandidatesByIdentity, extractIdentity, @@ -774,6 +777,18 @@ export const invokeOpenApiBackedTool = (input: { const result = invocation.result; const ok = result.status >= 200 && result.status < 300; if (!ok) { + // A bot-protection challenge is served by the edge before the request + // reaches the API, whatever status it carries (403 for a managed + // challenge, 503 for older JS challenges), so it is classified ahead + // of the credential branch: the key was never evaluated. + const botChallenge = detectBotChallenge({ headers: result.headers }); + if (botChallenge) { + return botChallengeToolFailure({ + integration: { id: integration, scope: input.credential.owner }, + status: result.status, + detection: botChallenge, + }); + } if (result.status === 401 || result.status === 403) { // A 403 naming a scope shortfall (RFC 6750 insufficient_scope, // Google's ACCESS_TOKEN_SCOPE_INSUFFICIENT) cannot be fixed by @@ -1012,6 +1027,24 @@ export const checkHealthOpenApi = (input: { } satisfies HealthCheckResult; } + // A bot-protection challenge never reached the API, so it says nothing + // about the credential: degraded, not expired. + const probeOk = probe.result.status >= 200 && probe.result.status < 300; + const botChallenge = probeOk ? null : detectBotChallenge({ headers: probe.result.headers }); + if (botChallenge) { + return { + status: "degraded", + httpStatus: probe.result.status, + checkedAt, + detail: botChallengeMessage({ + integration: String(input.integration.slug), + status: probe.result.status, + detection: botChallenge, + }), + reason: "upstream_status", + } satisfies HealthCheckResult; + } + // Body-aware: a configuration 403 (Google accessNotConfigured / // SERVICE_DISABLED) reads misconfigured, not expired. const status = classifyProbeResponse(probe.result.status, probe.result.error); diff --git a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts index 1f52b30c50..8462ace362 100644 --- a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts +++ b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts @@ -39,6 +39,7 @@ import { ToolAddress, } from "@executor-js/sdk"; import { makeTestConfig, memoryCredentialsPlugin } from "@executor-js/sdk/testing"; +import { variable } from "@executor-js/sdk/http-auth"; import { addOpenApiTestConnection, makeOpenApiHttpApiTestIntegrationConfig, @@ -402,6 +403,121 @@ describe("OpenAPI upstream failure modes", () => { }), ); + // A Cloudflare challenge page is the edge answering before the request + // reaches the API: the key was never evaluated, so it must not read as a + // rejected credential (and the challenge HTML must not be the payload). + const CHALLENGE_HTML = + "Just a moment..."; + + it.effect("Cloudflare challenge 403 is classified as upstream_bot_challenge", () => + Effect.gen(function* () { + const server = yield* startScriptedServer(() => ({ + status: 403, + headers: { + "content-type": "text/html; charset=UTF-8", + "cf-mitigated": "challenge", + "cf-ray": "8f1a2b3c4d5e6f70-SJC", + server: "cloudflare", + }, + body: CHALLENGE_HTML, + })); + const { executor, address } = yield* buildExecutorForOpenApiServer(server); + + const result = yield* executor.execute(address, {}); + + expect(result).toMatchObject({ + ok: false, + error: { + code: "upstream_bot_challenge", + status: 403, + retryable: false, + message: expect.stringContaining("Ray ID 8f1a2b3c4d5e6f70-SJC"), + details: { + category: "upstream_protection", + upstream: { status: 403, provider: "cloudflare", rayId: "8f1a2b3c4d5e6f70-SJC" }, + }, + }, + }); + expect(JSON.stringify(result)).not.toContain("_cf_chl_opt"); + }), + ); + + it.effect("Cloudflare challenge on a non-auth status is classified the same way", () => + Effect.gen(function* () { + const server = yield* startScriptedServer(() => ({ + status: 503, + headers: { "content-type": "text/html", "cf-mitigated": "challenge" }, + body: CHALLENGE_HTML, + })); + const { executor, address } = yield* buildExecutorForOpenApiServer(server); + + const result = yield* executor.execute(address, {}); + + expect(result).toMatchObject({ + ok: false, + error: { code: "upstream_bot_challenge", status: 503 }, + }); + }), + ); + + it.effect( + "challenge-looking HTML without the cf-mitigated header stays connection_rejected", + () => + Effect.gen(function* () { + const server = yield* startScriptedServer(() => ({ + status: 403, + headers: { "content-type": "text/html", server: "cloudflare" }, + body: CHALLENGE_HTML, + })); + const { executor, address } = yield* buildExecutorForOpenApiServer(server); + + const result = yield* executor.execute(address, {}); + + expect(result).toMatchObject({ + ok: false, + error: { code: "connection_rejected", status: 403 }, + }); + }), + ); + + it.effect("a challenged health probe reads degraded, not expired", () => + Effect.gen(function* () { + const server = yield* startScriptedServer(() => ({ + status: 403, + headers: { + "content-type": "text/html", + "cf-mitigated": "challenge", + "cf-ray": "8f1a2b3c4d5e6f70-SJC", + }, + body: CHALLENGE_HTML, + })); + const executor = yield* createExecutor(makeTestConfig({ plugins: testPlugins() })); + yield* executor.openapi.addSpec({ + spec: { kind: "blob", value: server.specJson }, + slug: "f", + baseUrl: server.baseUrl, + healthCheck: { operation: LIST_THINGS }, + authenticationTemplate: [ + { slug: "apiKey", type: "apiKey", headers: { "x-api-key": [variable("token")] } }, + ], + }); + + const health = yield* executor.connections.validate({ + owner: "org", + integration: IntegrationSlug.make("f"), + template: AuthTemplateSlug.make("apiKey"), + value: "token", + }); + + expect(health).toMatchObject({ + status: "degraded", + httpStatus: 403, + reason: "upstream_status", + detail: expect.stringContaining("credential was not checked"), + }); + }), + ); + it.effect("upstream returns malformed JSON despite Content-Type: application/json", () => Effect.gen(function* () { const server = yield* startScriptedServer(() => ({