Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions apps/cloud/src/auth/access-token-options.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import type { JWTVerifyOptions } from "jose";

/**
* Require expiring WorkOS tokens. Token lifetime is controlled by WorkOS via
* `exp`; do not cap the age locally, since AuthKit issues MCP access tokens
* that live for several days.
*/
/** WorkOS credentials may authorize a request for at most 24 hours after issuance. */
export const WORKOS_ACCESS_TOKEN_MAX_AGE_SECONDS = 24 * 60 * 60;

/** Require signed, expiring tokens and cap their effective lifetime even if the issuer sets a later exp. */
export const workosAccessTokenOptions: JWTVerifyOptions = {
requiredClaims: ["exp", "iat"],
maxTokenAge: WORKOS_ACCESS_TOKEN_MAX_AGE_SECONDS,
};
2 changes: 1 addition & 1 deletion apps/cloud/src/auth/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ export const withServiceLogging = <A, E, R>(
effect: Effect.Effect<A, unknown, R>,
): Effect.Effect<A, E, R> =>
effect.pipe(
Effect.tapCause((cause) => Effect.logError(`${name} failed`, cause)),
Effect.tapCause(() => Effect.logError(`${name} failed`)),
Effect.mapError(publicError),
Effect.withSpan(name),
) as Effect.Effect<A, E, R>;
10 changes: 5 additions & 5 deletions apps/cloud/src/auth/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,7 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group(
Effect.gen(function* () {
yield* Effect.logWarning(
"createOrganization: could not provision the Autumn customer",
{ organizationId: org.id, error },
{ organizationId: org.id },
);
yield* captureCauseEffect(error);
}),
Expand Down Expand Up @@ -620,10 +620,10 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group(
{ organizationId },
),
),
Effect.tapError((error) =>
Effect.tapError(() =>
Effect.logError(
"deleteOrganization: org marked deleted but the Autumn customer could not be deleted; retry the deletion",
{ organizationId, error },
{ organizationId },
),
),
Effect.mapError(() => new OrganizationDeletionIncomplete({ step: "billing" })),
Expand Down Expand Up @@ -663,10 +663,10 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group(
s.deleteOrganizationCascade(organizationId, deletedAt),
)
.pipe(
Effect.tapError((error) =>
Effect.tapError(() =>
Effect.logError(
"deleteOrganization: org marked deleted, removed from WorkOS and Autumn, but local purge failed, tenant data and secrets orphaned; retry the deletion",
{ organizationId, error },
{ organizationId },
),
),
);
Expand Down
2 changes: 1 addition & 1 deletion apps/cloud/src/auth/workos-events-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ export const runWorkOsEventsSync = (): Promise<void> =>
Effect.scoped,
Effect.catchCause((cause) =>
Effect.gen(function* () {
yield* Effect.logError("workos_events: sync run failed", cause);
yield* Effect.logError("workos_events: sync run failed");
yield* captureCauseEffect(cause);
}),
),
Expand Down
2 changes: 1 addition & 1 deletion apps/cloud/src/engine/execution-gate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ export const makeExecutionLimitGate = (checkBalance: ExecutionBalanceCheck) => {
Effect.catch((error: unknown) =>
Effect.gen(function* () {
yield* Effect.sync(() => {
console.warn("[billing] execution balance check failed open:", error);
console.warn("[billing] execution balance check failed open");
});
yield* captureCauseEffect(error);
return { blocked: false } as const satisfies GateDecision;
Expand Down
3 changes: 1 addition & 2 deletions apps/cloud/src/engine/execution-rate-limit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ const failOpen = (
"rate_limit.check.error_tag": outcome.errorTag,
});
yield* Effect.sync(() => {
console.warn("[rate-limit] execution rate limit check failed open:", error);
console.warn("[rate-limit] execution rate limit check failed open");
});
if (!outcome.timedOut) yield* captureCauseEffect(error);
return { blocked: false } as const satisfies GateDecision;
Expand Down Expand Up @@ -303,7 +303,6 @@ export const makeExecutionRateLimiter = (
`[rate-limit] exemption lookup failed for ${organizationId}; treating as ${
cached ? "last known" : "not exempt"
}:`,
error,
);
});
yield* captureCauseEffect(error);
Expand Down
3 changes: 1 addition & 2 deletions apps/cloud/src/extensions/billing/member-seats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,9 @@ export const forkReportMemberSeats = (
waitUntil(Effect.runPromise(autumn.setMemberSeats(organizationId, seats)));
});
}).pipe(
Effect.catch((error) =>
Effect.catch(() =>
Effect.logWarning("reportMemberSeats: seat recount failed", {
organizationId,
error,
}),
),
Effect.withSpan("billing.reportMemberSeats"),
Expand Down
6 changes: 3 additions & 3 deletions apps/cloud/src/extensions/billing/route.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "cloudflare:workers";
import { Cause, Effect } from "effect";
import { Effect } from "effect";
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import { autumnHandler } from "autumn-js/backend";

Expand Down Expand Up @@ -138,7 +138,7 @@ const handler = Effect.gen(function* () {
);

if (statusCode >= 400) {
console.error("[autumn] upstream error:", statusCode, response);
console.error("[autumn] upstream error", { status: statusCode });
return yield* new HttpResponseError({
status: statusCode,
code: "billing_request_failed",
Expand All @@ -150,7 +150,7 @@ const handler = Effect.gen(function* () {
}).pipe(
Effect.catchCause((err) => {
if (isServerError(err)) {
console.error("[autumn] request failed:", Cause.pretty(err));
console.error("[autumn] request failed", { status: 500 });
}
return toErrorServerResponseEffect(err);
}),
Expand Down
4 changes: 2 additions & 2 deletions apps/cloud/src/extensions/billing/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ const make = Effect.sync(() => {
// Silent billing data loss is worth paging on: autumn.trackExecution
// is fire-and-forget so the caller doesn't handle it themselves.
yield* Effect.sync(() => {
console.error("[billing] track failed:", error);
console.error("[billing] track failed");
});
yield* captureCauseEffect(error);
yield* Effect.annotateCurrentSpan({ "autumn.track.failed": true });
Expand Down Expand Up @@ -217,7 +217,7 @@ const make = Effect.sync(() => {
// Silent seat drift means wrong invoices, so failures page just
// like a lost execution track.
yield* Effect.sync(() => {
console.error("[billing] seat sync failed:", error);
console.error("[billing] seat sync failed");
});
yield* captureCauseEffect(error);
yield* Effect.annotateCurrentSpan({ "autumn.members.failed": true });
Expand Down
14 changes: 2 additions & 12 deletions apps/cloud/src/observability/error-logging.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,6 @@
import { Cause, Effect, Option, Predicate, Result } from "effect";
import { HttpRouter, HttpServerRequest } from "effect/unstable/http";

const MAX_LOGGED_CAUSE_CHARS = 4_000;

const truncate = (value: string): string =>
value.length <= MAX_LOGGED_CAUSE_CHARS
? value
: `${value.slice(0, MAX_LOGGED_CAUSE_CHARS)}\n...[truncated ${
value.length - MAX_LOGGED_CAUSE_CHARS
} chars]`;

const loggedCause = (cause: Cause.Cause<unknown>): string => truncate(Cause.pretty(cause));

const objectValue = (value: unknown, key: string): unknown =>
Predicate.hasProperty(value, key) ? value[key] : undefined;

Expand Down Expand Up @@ -65,7 +54,8 @@ export const logApiErrorCause = (
path: requestPath(request),
status: httpStatus(error),
errorTag: errorTag(error),
cause: loggedCause(cause),
// Error causes can contain provider responses, request bodies, and secrets.
// Keep only the classification; never serialize the exception or its stack.
});
};

Expand Down
23 changes: 7 additions & 16 deletions apps/cloud/src/observability/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,20 @@
// `withObservability` (in @executor-js/api) wraps every handler effect; when
// it sees an unmapped cause it asks `ErrorCapture.captureException` for a
// trace id and fails with `InternalError({ traceId })`. The client gets
// the opaque id, we get the full cause + stack in Sentry.
// the opaque id; Sentry receives a minimized diagnostic event.
// ---------------------------------------------------------------------------

import * as Sentry from "@sentry/cloudflare";
import type { ErrorEvent, Scope } from "@sentry/cloudflare";
import { Cause, Effect, Layer, Predicate } from "effect";
import type * as Tracer from "effect/Tracer";

import { minimizeSentryEvent } from "./sentry-privacy";

import { ErrorCapture } from "@executor-js/api";
import { classifyDurableObjectError } from "@executor-js/cloudflare/mcp/durable-object-errors";
import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";

// Drizzle/postgres-js include the failing SQL (params + bound values) in
// their error message. For OpenAPI source inserts that's 1MB+ of spec
// text which blows past terminal scrollback and hides the actual pg
// error. Sentry still receives the full, untruncated cause via
// `setExtra`; only the dev-console mirror is capped.
const MAX_CONSOLE_CAUSE_CHARS = 4_000;
const OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
const OTEL_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/;

Expand Down Expand Up @@ -52,11 +48,6 @@ export type OtelCorrelationContext = {
readonly spanId: string;
};

const truncate = (s: string): string =>
s.length <= MAX_CONSOLE_CAUSE_CHARS
? s
: `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n…[truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`;

const validOtelContext = (context: OtelCorrelationContext): boolean =>
OTEL_TRACE_ID_PATTERN.test(context.traceId) && OTEL_SPAN_ID_PATTERN.test(context.spanId);

Expand Down Expand Up @@ -212,7 +203,7 @@ export const beforeSendCloudEvent = (
options?: { readonly logPayload?: boolean },
): ErrorEvent | null => {
const reported = beforeSendWithOtelCorrelation(event, options);
return reported === null ? null : withStableGroupingFingerprint(reported);
return reported === null ? null : minimizeSentryEvent(withStableGroupingFingerprint(reported));
};

/**
Expand All @@ -230,8 +221,8 @@ export const beforeSendCloudEvent = (
export const cloudSentryOptions = (env: Env) => ({
dsn: env.SENTRY_DSN,
tracesSampleRate: 0,
enableLogs: true,
sendDefaultPii: true,
enableLogs: false,
sendDefaultPii: false,
skipOpenTelemetrySetup: true,
beforeSend: (event: ErrorEvent) =>
beforeSendCloudEvent(event, {
Expand Down Expand Up @@ -365,7 +356,7 @@ export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(
ErrorCapture.of({
captureException: (cause) =>
Effect.gen(function* () {
console.error("[api] unhandled cause:", truncate(Cause.pretty(cause)));
console.error("[api] unhandled cause", classificationTagsOf(cause));
return (yield* captureCauseEffect(cause)) ?? "";
}),
}),
Expand Down
7 changes: 5 additions & 2 deletions apps/cloud/src/observability/redact-span-urls.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ export class UrlRedactingSpanProcessor implements SpanProcessor {
if (name !== event.name) event.name = name;
if (event.attributes === undefined) continue;
for (const [key, value] of Object.entries(event.attributes)) {
if (key === "exception.message" || key === "exception.stacktrace") {
event.attributes[key] = "[REDACTED]";
continue;
}
// Event attributes permit string[] exactly as span attributes do, so
// array elements get the same free-text scrub, in place.
if (Array.isArray(value)) {
Expand All @@ -124,8 +128,7 @@ export class UrlRedactingSpanProcessor implements SpanProcessor {

const message = span.status.message;
if (typeof message === "string") {
const redacted = redactUrlsInText(message);
if (redacted !== message) span.status.message = redacted;
span.status.message = "[REDACTED]";
}
}
}
58 changes: 58 additions & 0 deletions apps/cloud/src/observability/sentry-privacy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import type { ErrorEvent } from "@sentry/cloudflare";

const SAFE_TAGS = new Set([
"otel_trace_id",
"otel_span_id",
"operation",
"reason",
"status",
"mcp.do.cause_owner",
]);

const identifier = (value: string | undefined): string | undefined =>
value !== undefined && /^[\w.$<>:/@ -]{1,160}$/.test(value) ? value : undefined;

const sourceFile = (value: string | undefined): string | undefined => {
if (!value) return undefined;
const path = URL.canParse(value) ? new URL(value).pathname : value.split(/[?#]/)[0];
return path && /^\/?(?:assets\/)?[\w./-]+\.(?:js|mjs|ts|tsx)$/.test(path) ? path : undefined;
};

/** Keep error classification, source positions and correlation; omit all raw payloads. */
export const minimizeSentryEvent = (event: ErrorEvent): ErrorEvent => ({
type: undefined,
event_id: event.event_id,
timestamp: event.timestamp,
platform: event.platform,
level: event.level,
release: event.release,
environment: event.environment,
fingerprint: event.fingerprint?.map((part) => identifier(part) ?? "Error"),
tags: Object.fromEntries(
Object.entries(event.tags ?? {}).filter(
([key, value]) => SAFE_TAGS.has(key) && identifier(String(value)) !== undefined,
),
),
exception: {
values: event.exception?.values?.map((exception) => ({
type: identifier(exception.type) ?? "Error",
value: "Details omitted to protect request data",
mechanism: exception.mechanism
? {
type: identifier(exception.mechanism.type) ?? "generic",
handled: exception.mechanism.handled,
}
: undefined,
stacktrace: {
frames: exception.stacktrace?.frames?.map((frame) => ({
filename: sourceFile(frame.filename),
function: identifier(frame.function),
module: identifier(frame.module),
lineno: frame.lineno,
colno: frame.colno,
in_app: frame.in_app,
})),
},
})),
},
});
1 change: 1 addition & 0 deletions apps/cloud/wrangler.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
},
"observability": {
"enabled": true,
"redact_query_string": true,
},
"ratelimits": [
{
Expand Down
Loading
Loading