diff --git a/.agents/skills/databuddy-internal/SKILL.md b/.agents/skills/databuddy-internal/SKILL.md index 48709518b..63bdc861e 100644 --- a/.agents/skills/databuddy-internal/SKILL.md +++ b/.agents/skills/databuddy-internal/SKILL.md @@ -57,9 +57,11 @@ Keep additions **minimal**: one bullet, a new `rg` hint, or a routing note—eno - Slack-reachable shared packages (`@databuddy/ai`, `@databuddy/rpc`) must not import `evlog/elysia`; use host-injected request logger providers from the API and plain evlog fallbacks elsewhere. - AI link tools must assign link folders by existing folder `id` or `slug` only; folder names are display text and must not be used for routing or dedupe. - `apps/basket`: ingest and LLM tracking service, Elysia app on port `4000` +- A legacy Basket dedupe value of `"1"` is an admission reservation, not delivery confirmation; claim or retry it rather than suppressing the event. - `apps/docs`: Next.js + Fumadocs docs app on port `3005` - When a user drops a prototype, remove only prototype-specific wiring and preserve the existing product surfaces it temporarily reused. - `apps/links`: redirect/link service +- `apps/links` tests share Bun `mock.module` state; mocks of `../lib/logging` must include every export used by later test files, including `setAttributes` for producer imports. - `apps/uptime`: uptime monitoring service - `apps/uptime` BullMQ worker concurrency defaults high for Bun async I/O; do not lower it just because `10_000` looks large. Verify downstream saturation or lock/timeout evidence first. - Public status pages render from `apps/status`; `apps/dashboard` owns status-page management/config UI only. When cleaning public status UX, update shared `@databuddy/ui/uptime` pieces or `apps/status` wrappers instead of redesigning dashboard-only route remnants. diff --git a/apps/basket/package.json b/apps/basket/package.json index 419e2ab67..6df1e552a 100644 --- a/apps/basket/package.json +++ b/apps/basket/package.json @@ -24,7 +24,6 @@ "@maxmind/geoip2-node": "^6.3.4", "@types/ua-parser-js": "^0.7.39", "async-mutex": "^0.5.0", - "effect": "^4.0.0-beta.90", "elysia": "catalog:", "evlog": "catalog:", "kafkajs": "^2.2.4", diff --git a/apps/basket/src/index.ts b/apps/basket/src/index.ts index 2de440da5..b1f5f2855 100644 --- a/apps/basket/src/index.ts +++ b/apps/basket/src/index.ts @@ -8,7 +8,7 @@ import { import { shutdownPostgres } from "@databuddy/db"; import { clickHouse } from "@databuddy/db/clickhouse"; import { getRedisCache } from "@databuddy/redis/redis"; -import { disconnect, disposeRuntime, runPromise } from "@lib/producer"; +import { disconnect, ShutdownDrainError } from "@lib/producer"; import { Kafka } from "kafkajs"; import { databuddyEvlogRedaction } from "@databuddy/shared/evlog-redaction"; import { @@ -79,13 +79,28 @@ async function gracefulShutdown(signal: string, exitCode = 0) { error_message: error instanceof Error ? error.message : String(error), }); const { shutdownRedis } = await import("@databuddy/redis"); + // Wait for acknowledged delivery before tearing down its dependencies. + try { + await disconnect(); + } catch (error) { + finalExitCode = 1; + if (error instanceof ShutdownDrainError) { + log.error({ + lifecycle: "producerDrain", + error_message: + "Basket producer drain timed out waiting for in-flight delivery", + in_flight: error.inFlight, + drain_timeout_ms: error.deadlineMs, + }); + } else { + logErr("producerDrain")(error); + } + } await Promise.all([ shutdownRedis().catch(logErr("redisShutdown")), shutdownPostgres().catch(logErr("postgresShutdown")), - flushBatchedAxiomDrain().catch(logErr("drainFlush")), - runPromise(disconnect).catch(logErr("shutdown")), - disposeRuntime().catch(logErr("runtimeDispose")), ]); + await flushBatchedAxiomDrain().catch(logErr("drainFlush")); closeGeoIPReader(); } catch (error) { finalExitCode = 1; diff --git a/apps/basket/src/lib/event-service.delivery.test.ts b/apps/basket/src/lib/event-service.delivery.test.ts new file mode 100644 index 000000000..a8631817b --- /dev/null +++ b/apps/basket/src/lib/event-service.delivery.test.ts @@ -0,0 +1,367 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const { + mockApplyVisitorIdPrivacy, + mockGetDailySalt, + mockGetGeo, + mockLoggerSet, + mockMarkDuplicateReservationDelivered, + mockParseUserAgent, + mockReleaseDuplicateReservation, + mockReserveDuplicate, + mockSend, + mockSendBatch, + mockShouldAnonymizeVisitorIds, +} = vi.hoisted(() => ({ + mockApplyVisitorIdPrivacy: vi.fn((value: unknown) => String(value ?? "")), + mockGetDailySalt: vi.fn(() => Promise.resolve("daily-salt")), + mockGetGeo: vi.fn(() => + Promise.resolve({ + anonymizedIP: "anonymous-ip", + city: "San Francisco", + country: "US", + region: "CA", + }) + ), + mockLoggerSet: vi.fn(), + mockMarkDuplicateReservationDelivered: vi.fn(() => Promise.resolve()), + mockParseUserAgent: vi.fn(() => Promise.resolve({ browserName: "Chrome" })), + mockReleaseDuplicateReservation: vi.fn(() => Promise.resolve()), + mockReserveDuplicate: vi.fn(() => + Promise.resolve({ + duplicate: false, + key: "dedup:track:evt_1", + token: "reservation-token", + }) + ), + mockSend: vi.fn(() => Promise.resolve()), + mockSendBatch: vi.fn(() => Promise.resolve()), + mockShouldAnonymizeVisitorIds: vi.fn(() => false), +})); + +vi.mock("@lib/producer", () => ({ + send: mockSend, + sendBatch: mockSendBatch, +})); + +vi.mock("@lib/security", () => ({ + applyVisitorIdPrivacy: mockApplyVisitorIdPrivacy, + getDailySalt: mockGetDailySalt, + markDuplicateReservationDelivered: mockMarkDuplicateReservationDelivered, + releaseDuplicateReservation: mockReleaseDuplicateReservation, + reserveDuplicate: mockReserveDuplicate, + shouldAnonymizeVisitorIds: mockShouldAnonymizeVisitorIds, +})); + +vi.mock("@lib/tracing", () => ({ + record: (_name: string, operation: () => Promise) => operation(), +})); + +vi.mock("@utils/ip-geo", () => ({ + extractTrustedClientIp: vi.fn(() => undefined), + getGeo: mockGetGeo, +})); + +vi.mock("@utils/user-agent", () => ({ + parseUserAgent: mockParseUserAgent, +})); + +vi.mock("evlog/elysia", () => ({ + useLogger: () => ({ set: mockLoggerSet }), +})); + +const { + buildTrackEvent, + insertOutgoingLink, + insertTrackEvent, + insertTrackEventsBatch, + stableAnalyticsEventId, +} = await import("./event-service"); + +const request = new Request("http://localhost/ingest"); +const trackReservation = { + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:reservation-token", + ttl: 86_400, +}; + +function trackBatchItem(eventId: string) { + return { + event: buildTrackEvent( + { anonymousId: "anon_1", eventId, name: "pageview" }, + { + anonymousId: "anon_1", + clientId: "ws_test", + eventId, + geo: { anonymizedIP: "anonymous-ip" }, + now: 1, + ua: {}, + } + ), + sourceEventId: eventId, + }; +} + +describe("core event delivery", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetGeo.mockResolvedValue({ + anonymizedIP: "anonymous-ip", + city: "San Francisco", + country: "US", + region: "CA", + }); + mockParseUserAgent.mockResolvedValue({ browserName: "Chrome" }); + mockReserveDuplicate.mockResolvedValue(trackReservation); + mockSend.mockResolvedValue(undefined); + mockSendBatch.mockResolvedValue(undefined); + mockShouldAnonymizeVisitorIds.mockReturnValue(false); + }); + + test("awaits durable delivery for core track events", async () => { + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(mockSend).toHaveBeenCalledWith( + "analytics-events", + expect.objectContaining({ client_id: "ws_test", event_name: "pageview" }) + ); + expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( + trackReservation + ); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("releases a track reservation and returns retryable 503 when delivery fails", async () => { + const deliveryFailure = new Error("ClickHouse unavailable"); + mockSend.mockRejectedValueOnce(deliveryFailure); + + await expect( + insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ) + ).rejects.toMatchObject({ + message: "Analytics delivery temporarily unavailable", + status: 503, + }); + + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("reuses the same ClickHouse UUID when a client retries a failed track event", async () => { + const deliveredEvents: Array<{ id: string }> = []; + let attempts = 0; + mockSend.mockImplementation((_topic, event) => { + deliveredEvents.push(event as { id: string }); + attempts += 1; + return attempts === 1 + ? Promise.reject(new Error("ambiguous delivery")) + : Promise.resolve(); + }); + + await expect( + insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_retry", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_retry", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(deliveredEvents.map((event) => event.id)).toEqual([ + stableAnalyticsEventId("ws_test", "track", "evt_retry"), + stableAnalyticsEventId("ws_test", "track", "evt_retry"), + ]); + expect(mockReserveDuplicate).toHaveBeenCalledWith( + stableAnalyticsEventId("ws_test", "track", "evt_retry"), + "track", + "evt_retry" + ); + }); + + test("awaits outgoing-link delivery and releases its reservation on failure", async () => { + const reservation = { + duplicate: false, + key: "dedup:outgoing_link:evt_link_1", + ttl: 86_400, + }; + mockReserveDuplicate.mockResolvedValueOnce(reservation); + mockSend.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + + await expect( + insertOutgoingLink( + { + anonymousId: "anon_1", + eventId: "evt_link_1", + href: "https://example.com/pricing", + }, + "ws_test", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSend).toHaveBeenCalledWith( + "analytics-outgoing-links", + expect.objectContaining({ client_id: "ws_test" }) + ); + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith(reservation); + }); + + test("does not send a duplicate core event", async () => { + mockReserveDuplicate.mockResolvedValueOnce({ duplicate: true }); + + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(mockSend).not.toHaveBeenCalled(); + }); + + test("returns retryable 503 without publishing a track event owned by another request", async () => { + mockReserveDuplicate.mockResolvedValueOnce({ + duplicate: false, + retryable: true, + }); + + await expect( + insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSend).not.toHaveBeenCalled(); + expect(mockMarkDuplicateReservationDelivered).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("returns retryable 503 without publishing an outgoing link owned by another request", async () => { + mockReserveDuplicate.mockResolvedValueOnce({ + duplicate: false, + retryable: true, + }); + + await expect( + insertOutgoingLink( + { + anonymousId: "anon_1", + eventId: "evt_link_1", + href: "https://example.com/pricing", + }, + "ws_test", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSend).not.toHaveBeenCalled(); + expect(mockMarkDuplicateReservationDelivered).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("continues delivery when Redis is unavailable and no reservation was acquired", async () => { + const unavailableReservation = { duplicate: false }; + mockReserveDuplicate.mockResolvedValueOnce(unavailableReservation); + + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(mockSend).toHaveBeenCalledOnce(); + expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( + unavailableReservation + ); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("delivers one logical batch event and promotes its reservation after acknowledgement", async () => { + const first = trackBatchItem("evt_1"); + const duplicate = trackBatchItem("evt_1"); + + await insertTrackEventsBatch([first, duplicate]); + + expect(mockReserveDuplicate).toHaveBeenCalledOnce(); + expect(mockReserveDuplicate).toHaveBeenCalledWith( + first.event.id, + "track", + "evt_1" + ); + expect(mockSendBatch).toHaveBeenCalledWith("analytics-events", [first.event]); + expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("releases acquired batch reservations when another request owns an item", async () => { + mockReserveDuplicate + .mockResolvedValueOnce(trackReservation) + .mockResolvedValueOnce({ duplicate: false, retryable: true }); + + await expect( + insertTrackEventsBatch([trackBatchItem("evt_1"), trackBatchItem("evt_2")]) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSendBatch).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("releases acquired batch reservations when reservation lookup fails", async () => { + mockReserveDuplicate + .mockResolvedValueOnce(trackReservation) + .mockRejectedValueOnce(new Error("Redis lookup failed")); + + await expect( + insertTrackEventsBatch([trackBatchItem("evt_1"), trackBatchItem("evt_2")]) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSendBatch).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("releases batch reservations when durable delivery fails", async () => { + mockSendBatch.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + + await expect(insertTrackEventsBatch([trackBatchItem("evt_1")])).rejects.toMatchObject({ + status: 503, + }); + + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); +}); diff --git a/apps/basket/src/lib/event-service.test.ts b/apps/basket/src/lib/event-service.test.ts index 353fe536c..7cdec9c70 100644 --- a/apps/basket/src/lib/event-service.test.ts +++ b/apps/basket/src/lib/event-service.test.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from "vitest"; import { CONTROL_CHARS, longString, XSS_PAYLOADS } from "../test-helpers"; -import { buildTrackEvent, type TrackEventContext } from "./event-service"; +import { + buildTrackEvent, + stableAnalyticsEventId, + type TrackEventContext, +} from "./event-service"; // ── Fixtures ── @@ -72,7 +76,7 @@ describe("buildTrackEvent — field mapping", () => { const result = buildTrackEvent(fullTrackData, fullCtx); // Identity - expect(result.id).toBeTruthy(); // randomUUIDv7 + expect(result.id).toBe(stableAnalyticsEventId("ws_test", "track", "evt_123")); expect(result.client_id).toBe("ws_test"); // Names & content @@ -135,6 +139,21 @@ describe("buildTrackEvent — field mapping", () => { expect(result.properties).toBe('{"plan":"pro","color":"blue"}'); }); + test("uses a stable UUID for the same client retry", () => { + const first = buildTrackEvent(fullTrackData, fullCtx); + const retry = buildTrackEvent(fullTrackData, fullCtx); + const otherClient = buildTrackEvent(fullTrackData, { + ...fullCtx, + clientId: "ws_other", + }); + + expect(first.id).toBe(retry.id); + expect(first.id).not.toBe(otherClient.id); + expect(first.id).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ); + }); + test("minimal input → defaults applied", () => { const result = buildTrackEvent({ name: "click" }, fullCtx); diff --git a/apps/basket/src/lib/event-service.ts b/apps/basket/src/lib/event-service.ts index d4da14945..fd5e66870 100644 --- a/apps/basket/src/lib/event-service.ts +++ b/apps/basket/src/lib/event-service.ts @@ -5,11 +5,13 @@ import type { WebVitalsSpansInsert, } from "@databuddy/db/clickhouse/tables"; import type { ErrorSpan, IndividualVital } from "@databuddy/validation"; -import { runFork, runPromise, send, sendBatch } from "@lib/producer"; +import { send, sendBatch } from "@lib/producer"; import { - checkDuplicate, getDailySalt, applyVisitorIdPrivacy, + markDuplicateReservationDelivered, + releaseDuplicateReservation, + reserveDuplicate, shouldAnonymizeVisitorIds, } from "@lib/security"; import { record } from "@lib/tracing"; @@ -23,7 +25,9 @@ import { validateSessionId, } from "@utils/validation"; import { randomUUIDv7 } from "bun"; +import { createError } from "evlog"; import { useLogger } from "evlog/elysia"; +import { createHash } from "node:crypto"; export interface TrackEventContext { anonymousId: string; @@ -47,6 +51,41 @@ export interface TrackEventContext { }; } +export interface BatchEvent { + event: T; + sourceEventId: string; +} + +function deliveryUnavailable(cause: unknown) { + return createError({ + code: "basket.DELIVERY_UNAVAILABLE", + message: "Analytics delivery temporarily unavailable", + status: 503, + why: "Databuddy could not durably accept the event.", + fix: "Retry the same event after a short delay.", + cause: cause instanceof Error ? cause : new Error(String(cause)), + }); +} + +/** + * ClickHouse stores analytics ids as UUIDs, while public client event ids can + * be arbitrary strings. Derive a valid, stable UUID so the same client retry + * preserves its physical identity without accepting arbitrary input as UUID. + */ +export function stableAnalyticsEventId( + clientId: string, + eventType: "outgoing_link" | "track", + eventId: string +): string { + const digest = createHash("sha256") + .update(JSON.stringify([clientId, eventType, eventId])) + .digest("hex"); + const variant = "89ab".charAt(Number.parseInt(digest.charAt(16), 16) % 4); + const uuid = `${digest.slice(0, 12)}5${digest.slice(13, 16)}${variant}${digest.slice(17, 32)}`; + + return `${uuid.slice(0, 8)}-${uuid.slice(8, 12)}-${uuid.slice(12, 16)}-${uuid.slice(16, 20)}-${uuid.slice(20)}`; +} + export function buildTrackEvent( trackData: any, ctx: TrackEventContext @@ -55,7 +94,7 @@ export function buildTrackEvent( typeof trackData.timestamp === "number" ? trackData.timestamp : ctx.now; return { - id: randomUUIDv7(), + id: stableAnalyticsEventId(ctx.clientId, "track", ctx.eventId), client_id: ctx.clientId, event_name: sanitizeString( trackData.name, @@ -113,6 +152,36 @@ export function buildTrackEvent( }; } +async function deliverReserved( + deliveryId: string, + eventType: "outgoing_link" | "track", + sourceEventId: string, + deliver: () => Promise +): Promise { + const reservation = await reserveDuplicate( + deliveryId, + eventType, + sourceEventId + ); + if (reservation.duplicate) { + return; + } + if (reservation.retryable) { + throw deliveryUnavailable( + new Error("A concurrent attempt owns this analytics event") + ); + } + + try { + await deliver(); + } catch (error) { + await releaseDuplicateReservation(reservation); + throw deliveryUnavailable(error); + } + + await markDuplicateReservationDelivered(reservation); +} + export function insertTrackEvent( trackData: any, clientId: string, @@ -122,62 +191,55 @@ export function insertTrackEvent( ): Promise { return record("insertTrackEvent", async () => { const log = useLogger(); - let eventId = sanitizeString( - trackData.eventId, - VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH - ); - if (!eventId) { - eventId = randomUUIDv7(); - } - - const [isDuplicate, geoData] = await Promise.all([ - checkDuplicate(eventId, "track"), - getGeo(ip, request), - ]); - - if (isDuplicate) { - return; - } + const eventId = + sanitizeString( + trackData.eventId, + VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH + ) || randomUUIDv7(); - const trustedCountry = extractTrustedClientIp(request) - ? geoData.country - : undefined; - const anonymizeVisitorIds = shouldAnonymizeVisitorIds( - trackData.anonymizeVisitorIds, - trustedCountry - ); - const [salt, ua] = await Promise.all([ - anonymizeVisitorIds ? getDailySalt() : Promise.resolve(undefined), - parseUserAgent(userAgent), - ]); - - log.set({ - event: { id: eventId, name: trackData.name, path: trackData.path }, - geo: { - country: geoData.country, - region: geoData.region, - city: geoData.city, - }, - }); + const deliveryId = stableAnalyticsEventId(clientId, "track", eventId); + await deliverReserved(deliveryId, "track", eventId, async () => { + const geoData = await getGeo(ip, request); + const trustedCountry = extractTrustedClientIp(request) + ? geoData.country + : undefined; + const anonymizeVisitorIds = shouldAnonymizeVisitorIds( + trackData.anonymizeVisitorIds, + trustedCountry + ); + const [salt, ua] = await Promise.all([ + anonymizeVisitorIds ? getDailySalt() : Promise.resolve(undefined), + parseUserAgent(userAgent), + ]); + + log.set({ + event: { id: eventId, name: trackData.name, path: trackData.path }, + geo: { + country: geoData.country, + region: geoData.region, + city: geoData.city, + }, + }); + + const anonymousId = applyVisitorIdPrivacy( + trackData.anonymousId, + anonymizeVisitorIds, + salt + ); - const anonymousId = applyVisitorIdPrivacy( - trackData.anonymousId, - anonymizeVisitorIds, - salt - ); + const now = Date.now(); - const now = Date.now(); + const trackEvent = buildTrackEvent(trackData, { + clientId, + eventId, + anonymousId, + geo: geoData, + ua, + now, + }); - const trackEvent = buildTrackEvent(trackData, { - clientId, - eventId, - anonymousId, - geo: geoData, - ua, - now, + await send("analytics-events", trackEvent); }); - - runFork(send("analytics-events", trackEvent)); }); } @@ -188,66 +250,135 @@ export function insertOutgoingLink( ): Promise { return record("insertOutgoingLink", async () => { const log = useLogger(); - let eventId = sanitizeString( - linkData.eventId, - VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH - ); - - if (!eventId) { - eventId = randomUUIDv7(); - } - - if (await checkDuplicate(eventId, "outgoing_link")) { - return; - } - - log.set({ - event: { id: eventId, type: "outgoing_link", href: linkData.href }, - }); - - const now = Date.now(); + const eventId = + sanitizeString( + linkData.eventId, + VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH + ) || randomUUIDv7(); - const trustedIp = extractTrustedClientIp(request); - const visitorCountry = - linkData.anonymizeVisitorIds === "auto" && trustedIp - ? (await getGeo(trustedIp, request)).country - : undefined; - const anonymizeVisitorIds = shouldAnonymizeVisitorIds( - linkData.anonymizeVisitorIds, - visitorCountry + const deliveryId = stableAnalyticsEventId( + clientId, + "outgoing_link", + eventId ); - const salt = anonymizeVisitorIds ? await getDailySalt() : undefined; - - const outgoingLinkEvent: OutgoingLinksInsert = { - id: randomUUIDv7(), - client_id: clientId, - anonymous_id: applyVisitorIdPrivacy( - linkData.anonymousId, - anonymizeVisitorIds, - salt - ), - session_id: validateSessionId(linkData.sessionId), - href: sanitizeUrl(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), - text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), - properties: linkData.properties - ? JSON.stringify(linkData.properties) - : "{}", - timestamp: - typeof linkData.timestamp === "number" ? linkData.timestamp : now, - }; - - runFork(send("analytics-outgoing-links", outgoingLinkEvent)); + await deliverReserved(deliveryId, "outgoing_link", eventId, async () => { + log.set({ + event: { id: eventId, type: "outgoing_link", href: linkData.href }, + }); + + const now = Date.now(); + + const trustedIp = extractTrustedClientIp(request); + const visitorCountry = + linkData.anonymizeVisitorIds === "auto" && trustedIp + ? (await getGeo(trustedIp, request)).country + : undefined; + const anonymizeVisitorIds = shouldAnonymizeVisitorIds( + linkData.anonymizeVisitorIds, + visitorCountry + ); + const salt = anonymizeVisitorIds ? await getDailySalt() : undefined; + + const outgoingLinkEvent: OutgoingLinksInsert = { + id: deliveryId, + client_id: clientId, + anonymous_id: applyVisitorIdPrivacy( + linkData.anonymousId, + anonymizeVisitorIds, + salt + ), + session_id: validateSessionId(linkData.sessionId), + href: sanitizeUrl(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), + text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), + properties: linkData.properties + ? JSON.stringify(linkData.properties) + : "{}", + timestamp: + typeof linkData.timestamp === "number" ? linkData.timestamp : now, + }; + + await send("analytics-outgoing-links", outgoingLinkEvent); + }); }); } -export function insertTrackEventsBatch(events: EventsInsert[]): Promise { - return record("insertTrackEventsBatch", async () => { - if (events.length === 0) { - return; - } +async function deliverBatch( + eventType: "outgoing_link" | "track", + topic: string, + items: BatchEvent[] +): Promise { + const uniqueItems = Array.from( + new Map(items.map((item) => [item.event.id, item])).values() + ); + if (uniqueItems.length === 0) { + return; + } + + const reservationResults = await Promise.allSettled( + uniqueItems.map(async (item) => ({ + item, + reservation: await reserveDuplicate( + item.event.id, + eventType, + item.sourceEventId + ), + })) + ); + const reservations = reservationResults.flatMap((result) => + result.status === "fulfilled" ? [result.value] : [] + ); + const reservationFailure = reservationResults.find( + (result): result is PromiseRejectedResult => result.status === "rejected" + ); + const retryableReservation = reservations.find( + ({ reservation }) => reservation.retryable + ); + if (reservationFailure || retryableReservation) { + await Promise.allSettled( + reservations.map(({ reservation }) => + releaseDuplicateReservation(reservation) + ) + ); + throw deliveryUnavailable( + reservationFailure?.reason ?? + new Error("A concurrent attempt owns this analytics event") + ); + } + + const accepted = reservations.filter( + ({ reservation }) => !reservation.duplicate + ); + if (accepted.length === 0) { + return; + } + + try { + await sendBatch( + topic, + accepted.map(({ item }) => item.event) + ); + } catch (error) { + await Promise.allSettled( + accepted.map(({ reservation }) => + releaseDuplicateReservation(reservation) + ) + ); + throw deliveryUnavailable(error); + } + + await Promise.allSettled( + accepted.map(({ reservation }) => + markDuplicateReservationDelivered(reservation) + ) + ); +} - await runPromise(sendBatch("analytics-events", events)); - }); +export function insertTrackEventsBatch( + events: BatchEvent[] +): Promise { + return record("insertTrackEventsBatch", () => + deliverBatch("track", "analytics-events", events) + ); } export function insertErrorSpans( @@ -295,7 +426,7 @@ export function insertErrorSpans( ) || "Error", })); - await runPromise(sendBatch("analytics-error-spans", spans)); + await sendBatch("analytics-error-spans", spans); }); } @@ -330,20 +461,16 @@ export function insertIndividualVitals( metric_value: vital.metricValue, })); - await runPromise(sendBatch("analytics-vitals-spans", spans)); + await sendBatch("analytics-vitals-spans", spans); }); } export function insertOutgoingLinksBatch( - events: OutgoingLinksInsert[] + events: BatchEvent[] ): Promise { - return record("insertOutgoingLinksBatch", async () => { - if (events.length === 0) { - return; - } - - await runPromise(sendBatch("analytics-outgoing-links", events)); - }); + return record("insertOutgoingLinksBatch", () => + deliverBatch("outgoing_link", "analytics-outgoing-links", events) + ); } export function insertCustomEvents( @@ -414,6 +541,6 @@ export function insertCustomEvents( : undefined, })); - await runPromise(sendBatch("analytics-custom-events", spans)); + await sendBatch("analytics-custom-events", spans); }); } diff --git a/apps/basket/src/lib/producer.delivery.test.ts b/apps/basket/src/lib/producer.delivery.test.ts new file mode 100644 index 000000000..9c75b877c --- /dev/null +++ b/apps/basket/src/lib/producer.delivery.test.ts @@ -0,0 +1,193 @@ +import type { ClickHouseClient } from "@clickhouse/client"; +import type { Producer } from "kafkajs"; +import { describe, expect, test, vi } from "vitest"; +import type { ProducerConfig } from "./producer"; + +const { mockCaptureError } = vi.hoisted(() => ({ + mockCaptureError: vi.fn(), +})); + +vi.mock("@databuddy/db/clickhouse", () => ({ + clickHouse: {}, + TABLE_NAMES: { + ai_traffic_spans: "analytics.ai_traffic_spans", + blocked_traffic: "analytics.blocked_traffic", + custom_events: "analytics.custom_events", + error_spans: "analytics.error_spans", + events: "analytics.events", + link_visits: "analytics.link_visits", + outgoing_links: "analytics.outgoing_links", + web_vitals_spans: "analytics.web_vitals_spans", + }, +})); + +vi.mock("@lib/tracing", () => ({ + captureError: mockCaptureError, + record: (_name: string, fn: () => Promise) => fn(), +})); + +const { createProducer } = await import("./producer"); + +const topicMap = { "analytics-events": "analytics.events" }; + +const baseConfig: ProducerConfig = { + broker: undefined, + chunkSize: 100, + kafkaTimeout: 1_000, + maxProducerRetries: 0, + password: undefined, + producerRetryDelay: 1, + reconnectCooldown: 1, + selfHost: true, + shutdownDrainTimeout: 50, + username: undefined, +}; + +const event = (id: string) => ({ + client_id: "ws_1", + event_id: id, + timestamp: 1, +}); + +function makeProducer( + insert: (input: unknown) => Promise, + config: Partial = {} +) { + return createProducer( + { ...baseConfig, ...config }, + null, + { insert } as unknown as ClickHouseClient, + topicMap + ); +} + +describe("producer delivery guarantees", () => { + test("resolves a core send only after direct ClickHouse fallback succeeds", async () => { + const insert = vi.fn(() => Promise.resolve()); + const producer = makeProducer(insert); + + await producer.sendOne("analytics-events", event("event_1")); + + expect(insert).toHaveBeenCalledWith( + expect.objectContaining({ + format: "JSONEachRow", + table: "analytics.events", + values: [event("event_1")], + }) + ); + }); + + test("returns a retryable error instead of acknowledging a failed direct fallback", async () => { + const producer = makeProducer(() => Promise.reject(new Error("offline"))); + + await expect( + producer.sendOne("analytics-events", event("event_1")) + ).rejects.toMatchObject({ + _tag: "ClickHouseFallbackError", + retryable: true, + table: "analytics.events", + topic: "analytics-events", + }); + }); + + test("does not admit a send after shutdown starts while delivery is active", async () => { + let releaseInsert: (() => void) | undefined; + const producer = makeProducer( + () => + new Promise((resolve) => { + releaseInsert = resolve; + }), + { shutdownDrainTimeout: 500 } + ); + + const activeDelivery = producer.sendOne("analytics-events", event("event_1")); + await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); + + const shutdown = producer.shutDown(); + await expect( + producer.sendOne("analytics-events", event("event_2")) + ).rejects.toMatchObject({ + _tag: "ProducerShuttingDownError", + retryable: true, + }); + + releaseInsert?.(); + await activeDelivery; + await shutdown; + }); + + test("reports an in-flight direct delivery when shutdown reaches its deadline", async () => { + let releaseInsert: (() => void) | undefined; + const producer = makeProducer( + () => + new Promise((resolve) => { + releaseInsert = resolve; + }), + { shutdownDrainTimeout: 20 } + ); + + const activeDelivery = producer.sendOne("analytics-events", event("event_1")); + await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); + await expect(producer.shutDown()).rejects.toMatchObject({ + _tag: "ShutdownDrainError", + inFlight: 1, + retryable: true, + }); + + releaseInsert?.(); + await activeDelivery; + }); + + test("shares a cold Kafka connection across concurrent sends", async () => { + let resolveConnect: (() => void) | undefined; + const connect = vi.fn( + () => + new Promise((resolve) => { + resolveConnect = resolve; + }) + ); + const send = vi.fn(() => Promise.resolve()); + const kafka = { + connect, + disconnect: vi.fn(() => Promise.resolve()), + send, + } as unknown as Producer; + const producer = createProducer( + { ...baseConfig, broker: "redpanda.test:9092", selfHost: false }, + kafka, + { insert: vi.fn(() => Promise.resolve()) } as unknown as ClickHouseClient, + topicMap + ); + + const first = producer.sendOne("analytics-events", event("event_1")); + const second = producer.sendOne("analytics-events", event("event_2")); + await vi.waitFor(() => expect(connect).toHaveBeenCalledOnce()); + resolveConnect?.(); + + await Promise.all([first, second]); + expect(send).toHaveBeenCalledTimes(2); + }); + + test("falls back when Kafka connect throws synchronously", async () => { + const insert = vi.fn(() => Promise.resolve()); + const kafka = { + connect: vi.fn(() => { + throw new Error("Kafka client is not ready"); + }), + disconnect: vi.fn(() => Promise.resolve()), + send: vi.fn(() => Promise.resolve()), + } as unknown as Producer; + const producer = createProducer( + { ...baseConfig, broker: "redpanda.test:9092", selfHost: false }, + kafka, + { insert } as unknown as ClickHouseClient, + topicMap + ); + + await producer.sendOne("analytics-events", event("event_1")); + + expect(insert).toHaveBeenCalledWith( + expect.objectContaining({ values: [event("event_1")] }) + ); + }); +}); diff --git a/apps/basket/src/lib/producer.kafka.test.ts b/apps/basket/src/lib/producer.kafka.test.ts index 2509b93ff..f424db02c 100644 --- a/apps/basket/src/lib/producer.kafka.test.ts +++ b/apps/basket/src/lib/producer.kafka.test.ts @@ -72,9 +72,7 @@ vi.mock("@lib/tracing", () => ({ record: (_name: string, fn: Function) => Promise.resolve().then(() => fn()), })); -const { disconnect, disposeRuntime, getStats, runPromise, send } = await import( - "./producer" -); +const { disconnect, send } = await import("./producer"); beforeEach(() => { mockCaptureError.mockClear(); @@ -82,7 +80,6 @@ beforeEach(() => { }); afterAll(async () => { - await disposeRuntime().catch(() => {}); for (const [key, value] of Object.entries(originalEnv)) { if (value === undefined) { delete process.env[key]; @@ -93,35 +90,26 @@ afterAll(async () => { }); describe("producer Kafka send failure handling", () => { - test("backs off after a send failure and still disconnects on shutdown", async () => { - await runPromise( - send("analytics-events", { - client_id: "ws_1", - event_id: "event_1", - timestamp: Date.now(), - }) - ); - - await runPromise( - send("analytics-events", { - client_id: "ws_1", - event_id: "event_2", - timestamp: Date.now(), - }) - ); - - const stats = await runPromise(getStats); + test("falls back to direct ClickHouse writes and still disconnects on shutdown", async () => { + await send("analytics-events", { + client_id: "ws_1", + event_id: "event_1", + timestamp: Date.now(), + }); + + await send("analytics-events", { + client_id: "ws_1", + event_id: "event_2", + timestamp: Date.now(), + }); expect(mockKafka).toHaveBeenCalledTimes(1); expect(mockProducer).toHaveBeenCalledTimes(1); expect(mockConnect).toHaveBeenCalledTimes(1); expect(mockSend).toHaveBeenCalledTimes(1); - expect(stats?.bufferSize).toBe(2); - expect(stats?.connected).toBe(false); - expect(stats?.failed).toBe(true); - expect(stats?.failedCount).toBe(1); + expect(mockClickHouseInsert).toHaveBeenCalledTimes(2); - await runPromise(disconnect); + await disconnect(); expect(mockDisconnect).toHaveBeenCalledTimes(1); }); diff --git a/apps/basket/src/lib/producer.test.ts b/apps/basket/src/lib/producer.test.ts index 12358ea99..cce3ffd1a 100644 --- a/apps/basket/src/lib/producer.test.ts +++ b/apps/basket/src/lib/producer.test.ts @@ -29,7 +29,7 @@ vi.mock("@lib/tracing", () => ({ record: (_name: string, fn: Function) => Promise.resolve().then(() => fn()), })); -const { disposeRuntime, getStats, runPromise, send } = await import("./producer"); +const { send } = await import("./producer"); beforeEach(async () => { mockCaptureError.mockClear(); @@ -37,7 +37,6 @@ beforeEach(async () => { }); afterAll(async () => { - await disposeRuntime().catch(() => {}); if (originalSelfHost === undefined) { delete process.env.SELFHOST; } else { @@ -45,32 +44,37 @@ afterAll(async () => { } }); -describe("producer fallback topics", () => { - test("blocked traffic is buffered for ClickHouse fallback", async () => { - await runPromise( - send("analytics-blocked-traffic", { - id: "blocked_1", - client_id: "ws_1", - timestamp: Date.now(), + describe("producer fallback topics", () => { + test("blocked traffic is delivered through the direct ClickHouse fallback", async () => { + await send("analytics-blocked-traffic", { + id: "blocked_1", + client_id: "ws_1", + timestamp: Date.now(), + }); + + expect(mockClickHouseInsert).toHaveBeenCalledWith( + expect.objectContaining({ + table: "analytics.blocked_traffic", + values: [ + expect.objectContaining({ id: "blocked_1", client_id: "ws_1" }), + ], }) ); - - const stats = await runPromise(getStats); - expect(stats?.errors).toBe(0); - expect(stats?.bufferSize).toBe(1); }); test("unknown topics include the missing topic in error context", async () => { - await runPromise( + await expect( send("analytics-unmapped-topic", { id: "evt_1", client_id: "ws_1", timestamp: Date.now(), }) - ); + ).rejects.toMatchObject({ + _tag: "UnknownKafkaTopicError", + retryable: false, + topic: "analytics-unmapped-topic", + }); - const stats = await runPromise(getStats); - expect(stats?.errors).toBe(1); expect(mockCaptureError).toHaveBeenCalledWith( expect.objectContaining({ message: "Unknown Kafka topic" }), { topic: "analytics-unmapped-topic" } diff --git a/apps/basket/src/lib/producer.ts b/apps/basket/src/lib/producer.ts index 5605dc065..2f7ae6406 100644 --- a/apps/basket/src/lib/producer.ts +++ b/apps/basket/src/lib/producer.ts @@ -2,7 +2,6 @@ import type { ClickHouseClient } from "@clickhouse/client"; import { clickHouse, TABLE_NAMES } from "@databuddy/db/clickhouse"; import { readBooleanEnv } from "@databuddy/env/boolean"; import { captureError, record } from "@lib/tracing"; -import { Data, Effect, Layer, ManagedRuntime, Ref, Schedule } from "effect"; import { createError } from "evlog"; import { CompressionTypes, Kafka, type Producer } from "kafkajs"; @@ -12,54 +11,80 @@ function stringifyEvent(event: unknown): string { ); } -export class KafkaConnectionError extends Data.TaggedError( - "KafkaConnectionError" -)<{ readonly cause?: Error }> {} -export class KafkaSendError extends Data.TaggedError("KafkaSendError")<{ +function toError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +export class ProducerShuttingDownError extends Error { + readonly _tag = "ProducerShuttingDownError"; + readonly eventCount: number; + readonly retryable = true; + + constructor({ eventCount }: { eventCount: number }) { + super("Basket producer is shutting down"); + this.name = this._tag; + this.eventCount = eventCount; + } +} + +export class UnknownKafkaTopicError extends Error { + readonly _tag = "UnknownKafkaTopicError"; + readonly retryable = false; readonly topic: string; - readonly cause?: Error; -}> {} -export class BufferOverflowError extends Data.TaggedError( - "BufferOverflowError" -)<{ readonly bufferLength: number }> {} -export class FlushError extends Data.TaggedError("FlushError")<{ + + constructor({ topic }: { topic: string }) { + super(`Unknown Kafka topic: ${topic}`); + this.name = this._tag; + this.topic = topic; + } +} + +export class ClickHouseFallbackError extends Error { + readonly _tag = "ClickHouseFallbackError"; + override readonly cause?: Error; + readonly retryable = true; readonly table: string; - readonly cause?: Error; -}> {} - -export type ProducerError = - | KafkaConnectionError - | KafkaSendError - | BufferOverflowError - | FlushError; - -interface BufferedEvent { - event: unknown; - table: string; + readonly topic: string; + + constructor({ + cause, + table, + topic, + }: { + cause?: Error; + table: string; + topic: string; + }) { + super(`Direct ClickHouse fallback insert failed for ${topic}`); + this.name = this._tag; + this.cause = cause; + this.table = table; + this.topic = topic; + } } -interface ProducerState { - buffer: BufferedEvent[]; - buffered: number; - connected: boolean; - connectionFailed: boolean; - dropped: number; - errors: number; - failedCount: number; - flushed: number; - flushing: boolean; - lastErrorTime: number | null; - lastRetry: number; - producerInitialized: boolean; - sent: number; - shuttingDown: boolean; +export class ShutdownDrainError extends Error { + readonly _tag = "ShutdownDrainError"; + readonly deadlineMs: number; + readonly inFlight: number; + readonly retryable = true; + + constructor({ + deadlineMs, + inFlight, + }: { + deadlineMs: number; + inFlight: number; + }) { + super("Basket producer did not drain before shutdown"); + this.name = this._tag; + this.deadlineMs = deadlineMs; + this.inFlight = inFlight; + } } -interface ProducerConfig { +export interface ProducerConfig { broker?: string; - bufferHardMax: number; - bufferInterval: number; - bufferMax: number; chunkSize: number; kafkaTimeout: number; maxProducerRetries: number; @@ -67,43 +92,14 @@ interface ProducerConfig { producerRetryDelay: number; reconnectCooldown: number; selfHost: boolean; + shutdownDrainTimeout: number; username?: string; } -const INITIAL_STATE: ProducerState = { - buffer: [], - sent: 0, - failedCount: 0, - buffered: 0, - flushed: 0, - dropped: 0, - errors: 0, - lastErrorTime: null, - connected: false, - connectionFailed: false, - lastRetry: 0, - producerInitialized: false, - shuttingDown: false, - flushing: false, -}; - -function toError(err: unknown): Error { - return err instanceof Error ? err : new Error(String(err)); -} - -function groupBufferedEvents( - items: BufferedEvent[] -): Map { - const grouped = new Map(); - for (const item of items) { - const tableEvents = grouped.get(item.table); - if (tableEvents) { - tableEvents.push(item); - continue; - } - grouped.set(item.table, [item]); - } - return grouped; +export interface BasketProducer { + sendMany: (topic: string, events: unknown[]) => Promise; + sendOne: (topic: string, event: unknown, key?: string) => Promise; + shutDown: () => Promise; } async function insertClickHouseChunks( @@ -111,361 +107,229 @@ async function insertClickHouseChunks( table: string, events: unknown[], chunkSize: number -) { - for (let i = 0; i < events.length; i += chunkSize) { +): Promise { + for (let index = 0; index < events.length; index += chunkSize) { await ch.insert({ table, - values: events.slice(i, i + chunkSize), + values: events.slice(index, index + chunkSize), format: "JSONEachRow", }); } } -function rebufferOrDropEvents({ - bufferHardMax, - events, - inc, - ref, - table, - error, -}: { - bufferHardMax: number; - error: FlushError; - events: unknown[]; - inc: (field: keyof ProducerState, n?: number) => Effect.Effect; - ref: Ref.Ref; - table: string; -}) { - return Ref.get(ref).pipe( - Effect.flatMap((state) => { - if (state.buffer.length + events.length <= bufferHardMax) { - return Ref.update(ref, (current) => ({ - ...current, - buffer: [ - ...current.buffer, - ...events.map((event) => ({ table, event })), - ], - errors: current.errors + 1, - })); - } - - return inc("dropped", events.length).pipe( - Effect.tap(() => inc("errors", 1)), - Effect.tap(() => - Effect.sync(() => - captureError(error.cause, { - message: `Dropped ${String(events.length)} events - buffer full`, - }) - ) - ) - ); - }) - ); -} - -function makeProducerEffects( +export function createProducer( config: ProducerConfig, kafka: Producer | null, ch: ClickHouseClient, - topicMap: Record, - ref: Ref.Ref -) { - const enabled = !config.selfHost && Boolean(config.broker); - - const inc = (field: keyof ProducerState, n = 1) => - Ref.update(ref, (s) => ({ ...s, [field]: (s[field] as number) + n })); - - const connect: Effect.Effect = Effect.gen(function* () { - if (!(enabled && kafka)) { - return (yield* Ref.get(ref)).connected; + topicMap: Record +): BasketProducer { + const kafkaEnabled = + !config.selfHost && Boolean(config.broker) && kafka !== null; + let connected = false; + let connectionFailed = false; + let connecting: Promise | null = null; + let inFlight = 0; + let lastRetry = 0; + let producerInitialized = false; + let shuttingDown = false; + + function connect(): Promise { + if (!(kafkaEnabled && kafka)) { + return Promise.resolve(false); } - const s = yield* Ref.get(ref); - if (s.connected) { - return true; + if (connected) { + return Promise.resolve(true); } - if ( - s.connectionFailed && - Date.now() - s.lastRetry < config.reconnectCooldown - ) { - return false; + if (connectionFailed && Date.now() - lastRetry < config.reconnectCooldown) { + return Promise.resolve(false); + } + if (connecting) { + return connecting; } - return yield* Effect.tryPromise({ - try: () => kafka.connect(), - catch: (e) => new KafkaConnectionError({ cause: toError(e) }), - }).pipe( - Effect.tap(() => - Ref.update(ref, (st) => ({ - ...st, - connected: true, - connectionFailed: false, - lastRetry: 0, - producerInitialized: true, - })) - ), - Effect.as(true), - Effect.catchTag("KafkaConnectionError", (err) => - Ref.update(ref, (st) => ({ - ...st, - connectionFailed: true, - lastRetry: Date.now(), - errors: st.errors + 1, - lastErrorTime: Date.now(), - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(err.cause, { - message: - "Redpanda connection failed, using ClickHouse fallback", - }) - ) - ), - Effect.as(false) - ) - ) - ); - }); + connecting = Promise.resolve() + .then(() => kafka.connect()) + .then(() => { + connected = true; + connectionFailed = false; + lastRetry = 0; + producerInitialized = true; + return true; + }) + .catch((error) => { + connectionFailed = true; + lastRetry = Date.now(); + captureError(toError(error), { + message: "Redpanda connection failed, using ClickHouse fallback", + }); + return false; + }) + .finally(() => { + connecting = null; + }); - const flush: Effect.Effect = Effect.gen(function* () { - const pre = yield* Ref.get(ref); - if (pre.buffer.length === 0 || pre.flushing) { - return; + return connecting; + } + + function resolveTopic(topic: string): string { + const table = topicMap[topic]; + if (table) { + return table; } - const batchSize = Math.min(pre.buffer.length, config.bufferMax); - const items = yield* Ref.modify(ref, (s) => [ - s.buffer.slice(0, batchSize), - { ...s, buffer: s.buffer.slice(batchSize), flushing: true }, - ]); - - const grouped = groupBufferedEvents(items); - - yield* Effect.forEach( - grouped.entries(), - ([table, entries]: [string, BufferedEvent[]]) => { - const events = entries.map((entry) => entry.event); - return Effect.tryPromise({ - try: () => - record("clickhouseFallbackInsert", () => - insertClickHouseChunks(ch, table, events, config.chunkSize) - ), - catch: (e) => new FlushError({ table, cause: toError(e) }), - }).pipe( - Effect.tap(() => inc("flushed", events.length)), - Effect.catchTag("FlushError", (error) => - rebufferOrDropEvents({ - bufferHardMax: config.bufferHardMax, - error, - events, - inc, - ref, - table, - }) - ) - ); - }, - { concurrency: "unbounded" } + captureError( + createError({ + code: "basket.UNKNOWN_KAFKA_TOPIC", + message: "Unknown Kafka topic", + status: 500, + why: `Topic "${topic}" is not mapped to a ClickHouse table.`, + fix: "Check topicMap configuration.", + }), + { topic } ); + throw new UnknownKafkaTopicError({ topic }); + } - yield* Ref.update(ref, (s) => ({ ...s, flushing: false })); - }); - - const toBuffer = ( + async function persistDirectly( topic: string, - event: unknown - ): Effect.Effect => - Effect.gen(function* () { - const table = topicMap[topic]; - if (!table) { - yield* inc("errors", 1); - yield* Effect.sync(() => - captureError( - createError({ - code: "basket.UNKNOWN_KAFKA_TOPIC", - message: "Unknown Kafka topic", - status: 500, - why: `Topic "${topic}" is not mapped to a ClickHouse table.`, - fix: "Check topicMap configuration.", - }), - { topic } - ) - ); - return; - } - - const result = yield* Ref.modify(ref, (s) => { - if (s.shuttingDown) { - return ["shutdown" as const, { ...s, dropped: s.dropped + 1 }]; - } - if (s.buffer.length >= config.bufferHardMax) { - return ["overflow" as const, { ...s, dropped: s.dropped + 1 }]; - } - return [ - "ok" as const, - { - ...s, - buffer: [...s.buffer, { table, event }], - buffered: s.buffered + 1, - }, - ]; + events: unknown[] + ): Promise { + const table = resolveTopic(topic); + try { + await record("clickhouseDirectFallbackInsert", () => + insertClickHouseChunks(ch, table, events, config.chunkSize) + ); + } catch (error) { + const fallbackError = new ClickHouseFallbackError({ + cause: toError(error), + table, + topic, }); + captureError(fallbackError.cause, { + message: "Direct ClickHouse fallback insert failed; rejecting delivery", + table, + topic, + }); + throw fallbackError; + } + } - if (result === "overflow") { - return yield* Effect.fail( - new BufferOverflowError({ - bufferLength: (yield* Ref.get(ref)).buffer.length, - }) - ); - } - }); + function acquireSendSlot(eventCount: number): void { + if (shuttingDown) { + throw new ProducerShuttingDownError({ eventCount }); + } + inFlight += eventCount; + } - const bufferAll = (topic: string, events: unknown[]) => - Effect.forEach(events, (e) => toBuffer(topic, e), { - discard: true, - }); + function releaseSendSlot(eventCount: number): void { + inFlight = Math.max(0, inFlight - eventCount); + } - const sendViaKafka = ( + async function sendViaKafka( topic: string, messages: Array<{ value: string; key?: string }>, fallbackEvents: unknown[] - ): Effect.Effect => - Effect.gen(function* () { - const s = yield* Ref.get(ref); - if (s.shuttingDown) { - yield* bufferAll(topic, fallbackEvents); - return; - } - - if (enabled && kafka) { - const isConnected = yield* connect; - if (isConnected) { - const sent = yield* Effect.tryPromise({ - try: () => - kafka.send({ - topic, - messages, - timeout: config.kafkaTimeout, - compression: CompressionTypes.GZIP, - }), - catch: (e) => new KafkaSendError({ topic, cause: toError(e) }), - }).pipe( - Effect.tap(() => inc("sent", messages.length)), - Effect.as(true), - Effect.catchTag("KafkaSendError", (err) => - Ref.update(ref, (st) => ({ - ...st, - connectionFailed: true, - connected: false, - lastRetry: Date.now(), - failedCount: st.failedCount + messages.length, - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(err.cause, { - message: "Redpanda send failed, buffering to ClickHouse", - message_count: messages.length, - topic, - }) - ) - ), - Effect.as(false) - ) - ) - ); - if (sent) { - return; - } + ): Promise { + acquireSendSlot(fallbackEvents.length); + + try { + if (kafkaEnabled && kafka && (await connect())) { + try { + await kafka.send({ + topic, + messages, + timeout: config.kafkaTimeout, + compression: CompressionTypes.GZIP, + }); + return; + } catch (error) { + connectionFailed = true; + connected = false; + lastRetry = Date.now(); + captureError(toError(error), { + message: "Redpanda send failed, falling back to ClickHouse", + message_count: messages.length, + topic, + }); } } - yield* bufferAll(topic, fallbackEvents); - }); - - const sendOne = ( - topic: string, - event: unknown, - key?: string - ): Effect.Effect => - sendViaKafka( - topic, - [ - { - value: stringifyEvent(event), - key: key || (event as { client_id?: string }).client_id, - }, - ], - [event] - ); + await persistDirectly(topic, fallbackEvents); + } finally { + releaseSendSlot(fallbackEvents.length); + } + } - const sendMany = ( - topic: string, - events: unknown[] - ): Effect.Effect => { - if (events.length === 0) { - return Effect.void; + async function disconnectKafka(): Promise { + if (!(producerInitialized && kafka)) { + return; } - return sendViaKafka( - topic, - events.map((e) => ({ - value: stringifyEvent(e), - key: - (e as { client_id?: string }).client_id || - (e as { event_id?: string }).event_id, - })), - events - ); - }; - const shutDown: Effect.Effect = Effect.gen(function* () { - yield* Ref.update(ref, (s) => ({ ...s, shuttingDown: true })); - yield* Effect.sleep("1 second"); - yield* flush.pipe(Effect.catch(() => Effect.void)); - const post = yield* Ref.get(ref); - if (post.buffer.length > 0 && !post.flushing) { - yield* flush.pipe(Effect.catch(() => Effect.void)); + try { + await kafka.disconnect(); + } catch (error) { + captureError(toError(error), { + message: "Error disconnecting Redpanda producer", + }); + } finally { + connected = false; + producerInitialized = false; } - if (post.producerInitialized && kafka) { - yield* Effect.tryPromise({ - try: () => kafka.disconnect(), - catch: (e) => new KafkaConnectionError({ cause: toError(e) }), - }).pipe( - Effect.ensuring( - Ref.update(ref, (s) => ({ - ...s, - connected: false, - producerInitialized: false, - })) - ), - Effect.catch((err) => - Effect.sync(() => - captureError(err.cause, { - message: "Error disconnecting Redpanda producer", - }) - ) - ) + } + + async function waitForInFlight(): Promise { + const deadline = Date.now() + config.shutdownDrainTimeout; + while (inFlight > 0) { + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new ShutdownDrainError({ + deadlineMs: config.shutdownDrainTimeout, + inFlight, + }); + } + await new Promise((resolve) => + setTimeout(resolve, Math.min(10, remaining)) ); } - }); - - const stats: Effect.Effect = Ref.get(ref).pipe( - Effect.map( - ({ - buffer, - flushing: _f, - shuttingDown: _s, - connectionFailed, - producerInitialized: _p, - ...rest - }) => ({ - ...rest, - failed: connectionFailed, - bufferSize: buffer.length, - kafkaEnabled: enabled, - }) - ) - ); + } - return { flush, sendOne, sendMany, shutDown, stats }; + return { + async sendOne(topic, event, key): Promise { + await sendViaKafka( + topic, + [ + { + value: stringifyEvent(event), + key: key || (event as { client_id?: string }).client_id, + }, + ], + [event] + ); + }, + async sendMany(topic, events): Promise { + if (events.length === 0) { + return; + } + await sendViaKafka( + topic, + events.map((event) => ({ + value: stringifyEvent(event), + key: + (event as { client_id?: string }).client_id || + (event as { event_id?: string }).event_id, + })), + events + ); + }, + async shutDown(): Promise { + shuttingDown = true; + try { + await waitForInFlight(); + } finally { + await disconnectKafka(); + } + }, + }; } function initializeKafka(config: ProducerConfig): Producer | null { @@ -508,21 +372,6 @@ function initializeKafka(config: ProducerConfig): Producer | null { }); } -export interface ProducerStatsSnapshot { - buffered: number; - bufferSize: number; - connected: boolean; - dropped: number; - errors: number; - failed: boolean; - failedCount: number; - flushed: number; - kafkaEnabled: boolean; - lastErrorTime: number | null; - lastRetry: number; - sent: number; -} - const CONFIG: ProducerConfig = { broker: process.env.REDPANDA_BROKER, username: process.env.REDPANDA_USER, @@ -532,10 +381,8 @@ const CONFIG: ProducerConfig = { kafkaTimeout: 10_000, maxProducerRetries: 3, producerRetryDelay: 300, - bufferInterval: 5000, - bufferMax: 1000, - bufferHardMax: 10_000, chunkSize: 5000, + shutdownDrainTimeout: 8000, }; const TOPIC_MAP: Record = { @@ -549,48 +396,25 @@ const TOPIC_MAP: Record = { "analytics-link-visits": TABLE_NAMES.link_visits, }; -let fx: ReturnType | null = null; - -const ProducerLive = Layer.effectDiscard( - Effect.gen(function* () { - const ref = yield* Ref.make({ ...INITIAL_STATE }); - const effects = makeProducerEffects( - CONFIG, - initializeKafka(CONFIG), - clickHouse, - TOPIC_MAP, - ref - ); - fx = effects; - yield* effects.flush.pipe( - Effect.catch(() => Effect.void), - Effect.repeat(Schedule.spaced(CONFIG.bufferInterval)), - Effect.forkScoped - ); - }) +const producer = createProducer( + CONFIG, + initializeKafka(CONFIG), + clickHouse, + TOPIC_MAP ); -const runtime = ManagedRuntime.make(ProducerLive); - -const withFx = ( - fn: (f: NonNullable) => Effect.Effect -): Effect.Effect => - Effect.suspend(() => (fx ? fn(fx) : Effect.succeed(undefined))); - export const send = (topic: string, event: unknown, key?: string) => - withFx((f) => f.sendOne(topic, event, key)); + producer.sendOne(topic, event, key); export const sendBatch = (topic: string, events: unknown[]) => - withFx((f) => f.sendMany(topic, events)); + producer.sendMany(topic, events); -export const disconnect = withFx((f) => f.shutDown); +export const disconnect = () => producer.shutDown(); -export const getStats = withFx((f) => f.stats); - -export const runFork = (effect: Effect.Effect) => - runtime.runFork(effect); - -export const runPromise = (effect: Effect.Effect) => - runtime.runPromise(effect); - -export const disposeRuntime = () => runtime.dispose(); +export function runFork(delivery: Promise): void { + delivery.catch((error) => { + captureError(error, { + message: "Asynchronous producer delivery rejected", + }); + }); +} diff --git a/apps/basket/src/lib/security.test.ts b/apps/basket/src/lib/security.test.ts index a34f98cc4..edfcd16c5 100644 --- a/apps/basket/src/lib/security.test.ts +++ b/apps/basket/src/lib/security.test.ts @@ -1,7 +1,9 @@ import { vi, beforeEach, describe, expect, test } from "vitest"; import { - checkDuplicate, applyVisitorIdPrivacy, + markDuplicateReservationDelivered, + releaseDuplicateReservation, + reserveDuplicate, saltAnonymousId, shouldAnonymizeVisitorIds, } from "./security"; @@ -114,17 +116,17 @@ describe("visitor ID anonymization helpers", () => { }); }); -// ── checkDuplicate (needs Redis mock) ── +// ── duplicate reservations (needs Redis mock) ── -const { mockRedisSet, mockLoggerSet, mockCaptureError } = vi.hoisted(() => ({ - mockRedisSet: vi.fn(() => Promise.resolve("OK")), +const { mockRedisEval, mockLoggerSet, mockCaptureError } = vi.hoisted(() => ({ + mockRedisEval: vi.fn(), mockLoggerSet: vi.fn(() => {}), mockCaptureError: vi.fn(), })); vi.mock("@databuddy/redis/redis", () => ({ - redis: { set: mockRedisSet }, - getRedisCache: () => ({ set: mockRedisSet }), + redis: { eval: mockRedisEval }, + getRedisCache: () => ({}), })); vi.mock("@databuddy/redis/cacheable", () => ({ cacheable: (fn: () => Promise) => fn, @@ -140,106 +142,214 @@ vi.mock("@lib/tracing", () => ({ captureError: mockCaptureError, })); -describe("checkDuplicate", () => { +describe("duplicate reservations", () => { beforeEach(() => { - mockRedisSet.mockReset(); + mockRedisEval.mockReset(); mockLoggerSet.mockReset(); mockCaptureError.mockReset(); }); - test("first event (NX returns OK) → not duplicate", async () => { - mockRedisSet.mockResolvedValue("OK"); - const result = await checkDuplicate("evt_1", "track"); - expect(result).toBe(false); - expect(mockRedisSet).toHaveBeenCalledWith( + test("atomically writes a pending reservation for a new event", async () => { + mockRedisEval.mockResolvedValue("acquired"); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toMatchObject({ + duplicate: false, + key: "dedup:track:evt_1", + token: expect.stringMatching(/^pending:/), + ttl: 86_400, + }); + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1])'), + 1, "dedup:track:evt_1", - "1", - "EX", - 86_400, - "NX" + expect.stringMatching(/^pending:/), + 86_400 ); }); - test("duplicate event (NX returns null) → is duplicate", async () => { - mockRedisSet.mockResolvedValue(null); - const result = await checkDuplicate("evt_1", "track"); - expect(result).toBe(true); - }); + test("suppresses only a confirmed delivered reservation", async () => { + mockRedisEval.mockResolvedValue("delivered"); - test("exit_ prefix → uses longer TTL (172800)", async () => { - mockRedisSet.mockResolvedValue("OK"); - await checkDuplicate("exit_abc", "track"); - expect(mockRedisSet).toHaveBeenCalledWith( - "dedup:track:exit_abc", - "1", - "EX", - 172_800, - "NX" - ); + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: true }); + expect(mockLoggerSet).toHaveBeenCalledWith({ + dedup: { duplicate: true, eventType: "track" }, + }); }); - test("non-exit prefix → uses standard TTL (86400)", async () => { - mockRedisSet.mockResolvedValue("OK"); - await checkDuplicate("normal_abc", "track"); - expect(mockRedisSet).toHaveBeenCalledWith( - "dedup:track:normal_abc", - "1", - "EX", - 86_400, - "NX" + test("claims a legacy admission reservation before delivery", async () => { + mockRedisEval.mockResolvedValue("acquired"); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toMatchObject({ + duplicate: false, + key: "dedup:track:evt_1", + token: expect.stringMatching(/^pending:/), + ttl: 86_400, + }); + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('if value == "1" then'), + 1, + "dedup:track:evt_1", + expect.stringMatching(/^pending:/), + 86_400 ); }); - test("different event types → different keys", async () => { - mockRedisSet.mockResolvedValue("OK"); - await checkDuplicate("evt_1", "outgoing_link"); - expect(mockRedisSet).toHaveBeenCalledWith( - "dedup:outgoing_link:evt_1", - "1", - "EX", - 86_400, - "NX" - ); + test("requires a retry when another request owns a pending reservation", async () => { + mockRedisEval.mockResolvedValue("pending"); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: false, retryable: true }); }); - test("transient Redis error → retries once before failing open", async () => { - mockRedisSet + test("recovers ownership when an ambiguous EVAL wrote this request's token", async () => { + mockRedisEval .mockRejectedValueOnce(new Error("stale connection")) - .mockResolvedValueOnce("OK"); + .mockResolvedValueOnce("acquired"); - const result = await checkDuplicate("evt_1", "track"); + const reservation = await reserveDuplicate("evt_1", "track"); - expect(result).toBe(false); - expect(mockRedisSet).toHaveBeenCalledTimes(2); + expect(reservation).toMatchObject({ + duplicate: false, + token: expect.stringMatching(/^pending:/), + }); + expect(mockRedisEval).toHaveBeenCalledTimes(2); + expect(mockRedisEval.mock.calls[1]).toEqual(mockRedisEval.mock.calls[0]); expect(mockCaptureError).not.toHaveBeenCalled(); }); - test("ambiguous retry null after Redis error → not duplicate", async () => { - mockRedisSet - .mockRejectedValueOnce(new Error("stale connection")) - .mockResolvedValueOnce(null); + test("preserves the exit-event retention when its delivery id is hashed", async () => { + mockRedisEval.mockResolvedValue("acquired"); - const result = await checkDuplicate("evt_1", "track"); + await reserveDuplicate("stable-delivery-id", "track", "exit_abc"); - expect(result).toBe(false); - expect(mockRedisSet).toHaveBeenCalledTimes(2); - expect(mockLoggerSet).not.toHaveBeenCalled(); - expect(mockCaptureError).not.toHaveBeenCalled(); + expect(mockRedisEval).toHaveBeenCalledWith( + expect.any(String), + 1, + "dedup:track:stable-delivery-id", + expect.stringMatching(/^pending:/), + 172_800 + ); }); - test("Redis error → returns false (fail-open)", async () => { - mockRedisSet.mockRejectedValue(new Error("Redis down")); - const result = await checkDuplicate("evt_1", "track"); - expect(result).toBe(false); - expect(mockRedisSet).toHaveBeenCalledTimes(2); + test("fails open for a Redis outage without turning an event into a duplicate", async () => { + mockRedisEval.mockRejectedValue(new Error("Redis down")); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: false }); + expect(mockRedisEval).toHaveBeenCalledTimes(2); expect(mockCaptureError).toHaveBeenCalledOnce(); }); - test("duplicate event logs dedup context", async () => { - mockRedisSet.mockResolvedValue(null); - await checkDuplicate("evt_dup", "track"); - expect(mockLoggerSet).toHaveBeenCalledWith({ - dedup: { duplicate: true, eventType: "track" }, + test("does not release a reservation it does not own", async () => { + await releaseDuplicateReservation({ duplicate: false }); + + expect(mockRedisEval).not.toHaveBeenCalled(); + }); + + test("marks only an owned pending reservation as delivered", async () => { + mockRedisEval.mockResolvedValue("OK"); + await markDuplicateReservationDelivered({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:owner-attempt", + ttl: 86_400, + }); + + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + 1, + "dedup:track:evt_1", + "pending:owner-attempt", + "delivered", + 86_400 + ); + }); + + test("does not let a stale delivery promote a newer reservation", async () => { + let storedToken = "pending:newer-attempt"; + mockRedisEval.mockImplementation( + async (_script, _keys, _key, expectedToken: string, deliveredValue: string) => { + if (storedToken === expectedToken) { + storedToken = deliveredValue; + return "OK"; + } + return 0; + } + ); + + await markDuplicateReservationDelivered({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:stale-attempt", + ttl: 86_400, + }); + + expect(storedToken).toBe("pending:newer-attempt"); + }); + + test("releases only the pending reservation owned by the failed request", async () => { + mockRedisEval.mockResolvedValue(1); + + await releaseDuplicateReservation({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:owner-attempt", }); + + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + 1, + "dedup:track:evt_1", + "pending:owner-attempt" + ); + }); + + test("does not let a stale attempt release a newer reservation", async () => { + let storedToken = "pending:newer-attempt"; + mockRedisEval.mockImplementation( + async (_script, _keys, _key, expectedToken: string) => { + if (storedToken === expectedToken) { + storedToken = ""; + return 1; + } + return 0; + } + ); + + await releaseDuplicateReservation({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:stale-attempt", + }); + + expect(storedToken).toBe("pending:newer-attempt"); + }); + + test("captures release failures without hiding the original delivery failure", async () => { + mockRedisEval.mockRejectedValue(new Error("Redis down")); + + await expect( + releaseDuplicateReservation({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:owner-attempt", + }) + ).resolves.toBeUndefined(); + + expect(mockCaptureError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + message: + "Failed to release duplicate reservation after delivery failure", + }) + ); }); }); diff --git a/apps/basket/src/lib/security.ts b/apps/basket/src/lib/security.ts index 2fc24c01d..ec07c0a09 100644 --- a/apps/basket/src/lib/security.ts +++ b/apps/basket/src/lib/security.ts @@ -9,6 +9,35 @@ import { useLogger } from "evlog/elysia"; const EXIT_EVENT_TTL = 172_800; const STANDARD_EVENT_TTL = 86_400; const DEDUP_RETRY_DELAY_MS = 25; +const PENDING_DEDUP_PREFIX = "pending:"; +const DELIVERED_DEDUP_VALUE = "delivered"; +// Kept until all keys written by the pre-reservation implementation expire. +// Unlike "delivered", this value was written before delivery was acknowledged. +const LEGACY_PENDING_DEDUP_VALUE = "1"; +const RESERVE_DEDUP_RESERVATION = `local value = redis.call("GET", KEYS[1]) +if value == false then + redis.call("SET", KEYS[1], ARGV[1], "EX", ARGV[2]) + return "acquired" +end +if value == ARGV[1] then + return "acquired" +end +if value == "${DELIVERED_DEDUP_VALUE}" then + return "delivered" +end +if value == "${LEGACY_PENDING_DEDUP_VALUE}" then + redis.call("SET", KEYS[1], ARGV[1], "EX", ARGV[2]) + return "acquired" +end +return "pending"`; +const RELEASE_PENDING_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("DEL", KEYS[1]) +end +return 0`; +const MARK_PENDING_DEDUP_RESERVATION_DELIVERED = `if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("SET", KEYS[1], ARGV[2], "EX", ARGV[3]) +end +return 0`; const RAW_VISITOR_ID_COUNTRIES = ["US"]; const COUNTRY_CODES: Record = { @@ -119,50 +148,144 @@ export function applyVisitorIdPrivacy( return saltAnonymousId(sanitized, salt); } -function wait(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); +export interface DuplicateReservation { + readonly duplicate: boolean; + readonly key?: string; + /** + * Redis confirmed that this request did not acquire the reservation. The + * caller must retry instead of publishing alongside its current owner. + * Omitted when Redis is unavailable so ingestion remains fail-open. + */ + readonly retryable?: true; + /** + * Present only when this request atomically acquired the pending key. It is + * required to release the key so an older failed request cannot erase a + * newer retry's reservation. + */ + readonly token?: string; + readonly ttl?: number; } -async function setDedupKey(key: string, ttl: number): Promise { +async function reserveDedupKey( + key: string, + ttl: number, + token: string +): Promise<"acquired" | "delivered" | "pending"> { + const reserve = async () => { + const result = await redis.eval( + RESERVE_DEDUP_RESERVATION, + 1, + key, + token, + ttl + ); + return result === "acquired" || result === "delivered" ? result : "pending"; + }; + try { - return await redis.set(key, "1", "EX", ttl, "NX"); + return await reserve(); } catch (firstError) { - await wait(DEDUP_RETRY_DELAY_MS); + await new Promise((resolve) => setTimeout(resolve, DEDUP_RETRY_DELAY_MS)); try { - const retryResult = await redis.set(key, "1", "EX", ttl, "NX"); - // If the first SET succeeded but the client saw an error, retry returns null. - // Treat that ambiguous state as first delivery so ingestion fails open. - return retryResult ?? "OK"; + return await reserve(); } catch { throw firstError; } } } -export function checkDuplicate( +export function reserveDuplicate( eventId: string, - eventType: string -): Promise { - return record("checkDuplicate", async () => { + eventType: string, + sourceEventId = eventId +): Promise { + return record("reserveDuplicate", async () => { const key = `dedup:${eventType}:${eventId}`; - const ttl = eventId.startsWith("exit_") + const ttl = sourceEventId.startsWith("exit_") ? EXIT_EVENT_TTL : STANDARD_EVENT_TTL; + const token = `${PENDING_DEDUP_PREFIX}${crypto.randomUUID()}`; try { - const result = await setDedupKey(key, ttl); - const isDuplicate = result === null; - if (isDuplicate) { + const result = await reserveDedupKey(key, ttl, token); + if (result === "delivered") { useLogger().set({ dedup: { duplicate: true, eventType } }); + return { duplicate: true }; + } + + // This request did not acquire a pending key. Publishing without ownership + // can duplicate a concurrent delivery or overwrite its confirmed state. + if (result === "pending") { + return { duplicate: false, retryable: true }; } - return isDuplicate; + + return { + duplicate: false, + key, + token, + ttl, + }; } catch (error) { captureError(error, { message: "Failed to check duplicate event in Redis", eventId, eventType, }); - return false; + return { duplicate: false }; + } + }); +} + +/** + * Only a confirmed Kafka or ClickHouse acknowledgement may turn a pending + * retry guard into a duplicate suppression key. The promotion is conditional + * on the owner's token so a stale request cannot overwrite a newer reservation. + */ +export function markDuplicateReservationDelivered( + reservation: DuplicateReservation +): Promise { + return record("markDuplicateReservationDelivered", async () => { + if (!(reservation.key && reservation.token && reservation.ttl)) { + return; + } + + try { + await redis.eval( + MARK_PENDING_DEDUP_RESERVATION_DELIVERED, + 1, + reservation.key, + reservation.token, + DELIVERED_DEDUP_VALUE, + reservation.ttl + ); + } catch (error) { + captureError(error, { + message: "Failed to confirm duplicate reservation after delivery", + }); + } + }); +} + +export function releaseDuplicateReservation( + reservation: DuplicateReservation +): Promise { + return record("releaseDuplicateReservation", async () => { + if (!(reservation.key && reservation.token)) { + return; + } + + try { + await redis.eval( + RELEASE_PENDING_DEDUP_RESERVATION, + 1, + reservation.key, + reservation.token + ); + } catch (error) { + captureError(error, { + message: + "Failed to release duplicate reservation after delivery failure", + }); } }); } diff --git a/apps/basket/src/routes/basket.ts b/apps/basket/src/routes/basket.ts index 5c73b25ca..e8afd2dd2 100644 --- a/apps/basket/src/routes/basket.ts +++ b/apps/basket/src/routes/basket.ts @@ -24,6 +24,8 @@ import { insertOutgoingLinksBatch, insertTrackEvent, insertTrackEventsBatch, + stableAnalyticsEventId, + type BatchEvent, } from "@lib/event-service"; import { summarizeRejectedBody } from "@lib/rejection-summary"; import { @@ -97,14 +99,17 @@ function processTrackEventData( salt ); - return buildTrackEvent(trackData, { - clientId, - eventId, - anonymousId, - geo: geoData, - ua, - now: Date.now(), - }); + return { + event: buildTrackEvent(trackData, { + clientId, + eventId, + anonymousId, + geo: geoData, + ua, + now: Date.now(), + }), + sourceEventId: eventId, + }; }); } @@ -112,7 +117,8 @@ async function processOutgoingLinkData( linkData: OutgoingLinkInput, clientId: string, visitorCountry?: unknown -): Promise { +): Promise> { + const eventId = parseEventId(linkData.eventId, () => randomUUIDv7()); const timestamp = parseTimestamp(linkData.timestamp); const anonymizeVisitorIds = shouldAnonymizeVisitorIds( linkData.anonymizeVisitorIds, @@ -127,14 +133,17 @@ async function processOutgoingLinkData( ); return { - id: randomUUIDv7(), - client_id: clientId, - anonymous_id: anonymousId, - session_id: validateSessionId(linkData.sessionId), - href: sanitizeString(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), - text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), - properties: parseProperties(linkData.properties), - timestamp, + event: { + id: stableAnalyticsEventId(clientId, "outgoing_link", eventId), + client_id: clientId, + anonymous_id: anonymousId, + session_id: validateSessionId(linkData.sessionId), + href: sanitizeString(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), + text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), + properties: parseProperties(linkData.properties), + timestamp, + }, + sourceEventId: eventId, }; } @@ -169,9 +178,9 @@ const app = new Elysia() } if (eventType === "track") { - insertTrackEvent(eventData, clientId, userAgent, ip, request); + await insertTrackEvent(eventData, clientId, userAgent, ip, request); } else if (eventType === "outgoing_link") { - insertOutgoingLink(eventData, clientId, request); + await insertOutgoingLink(eventData, clientId, request); } else if (eventType === "web_vitals") { const vitalParse = individualVitalSchema.safeParse(eventData); if (!vitalParse.success) { @@ -182,7 +191,11 @@ const app = new Elysia() [vitalParse.data], request ); - insertIndividualVitals([vitalParse.data], clientId, visitorCountry); + await insertIndividualVitals( + [vitalParse.data], + clientId, + visitorCountry + ); } else if (eventType === "error") { const errorParse = errorSpanSchema.safeParse(eventData); if (!errorParse.success) { @@ -193,16 +206,18 @@ const app = new Elysia() [errorParse.data], request ); - insertErrorSpans([errorParse.data], clientId, visitorCountry); + await insertErrorSpans([errorParse.data], clientId, visitorCountry); } return createPixelResponse(); } catch (error) { - if (error instanceof EvlogError) { + if (error instanceof EvlogError && error.status < 500) { return createPixelResponse(); } log.error(error instanceof Error ? error : new Error(String(error))); - return createPixelResponse(); + // Do not report a successful image load before the event has durable + // storage; callers that can retry need an honest failure response. + return createPixelResponse(503); } }) .post("/vitals", async ({ body, query, request }) => { @@ -484,8 +499,8 @@ const app = new Elysia() const { clientId, userAgent, ip } = validation; log.set({ clientId }); - const trackEvents: EventsInsert[] = []; - const outgoingLinkEvents: OutgoingLinksInsert[] = []; + const trackEvents: BatchEvent[] = []; + const outgoingLinkEvents: BatchEvent[] = []; const results: Record[] = []; let batchVisitorCountry: string | undefined; let hasResolvedBatchVisitorCountry = false; @@ -614,10 +629,17 @@ const app = new Elysia() } } - await Promise.all([ + const deliveryResults = await Promise.allSettled([ insertTrackEventsBatch(trackEvents), insertOutgoingLinksBatch(outgoingLinkEvents), ]); + const deliveryFailure = deliveryResults.find( + (result): result is PromiseRejectedResult => + result.status === "rejected" + ); + if (deliveryFailure) { + throw deliveryFailure.reason; + } log.set({ processed: results.length, diff --git a/apps/basket/src/routes/integration.test.ts b/apps/basket/src/routes/integration.test.ts index 8a08c6bea..65aad76c4 100644 --- a/apps/basket/src/routes/integration.test.ts +++ b/apps/basket/src/routes/integration.test.ts @@ -1,8 +1,8 @@ +import { createError } from "evlog"; import { vi, beforeEach, describe, expect, test } from "vitest"; const { noop, - noopAsync, mockLogger, mockValidateRequest, mockCheckForBot, @@ -22,7 +22,6 @@ const { mockResolveApiKeyOwnerId, } = vi.hoisted(() => { const noop = vi.fn(() => {}); - const noopAsync = vi.fn(() => Promise.resolve()); const defaultApiKey = { id: "key_1", organizationId: "org_1", @@ -39,7 +38,6 @@ const { }; return { noop, - noopAsync, mockLogger: { set: vi.fn(() => {}), warn: vi.fn(() => {}), @@ -102,6 +100,7 @@ vi.mock("@lib/event-service", () => ({ insertIndividualVitals: mockInsertIndividualVitals, insertErrorSpans: mockInsertErrorSpans, insertCustomEvents: mockInsertCustomEvents, + stableAnalyticsEventId: vi.fn(() => "stable_id"), })); vi.mock("@lib/security", () => ({ @@ -171,9 +170,8 @@ vi.mock("@hooks/auth", () => ({ vi.mock("@lib/producer", () => ({ runFork: noop, - send: vi.fn(() => ({})), - sendBatch: vi.fn(() => ({})), - runPromise: noopAsync, + send: vi.fn(() => Promise.resolve()), + sendBatch: vi.fn(() => Promise.resolve()), })); // ── Import routes after mocks ── @@ -443,6 +441,11 @@ describe("POST /events", () => { // ── POST /batch ── describe("POST /batch", () => { + beforeEach(() => { + mockInsertOutgoingLinksBatch.mockClear(); + mockInsertTrackEventsBatch.mockClear(); + }); + test("batch of track events → 200", async () => { const res = await post(basketApp, "/batch", [ { @@ -462,6 +465,59 @@ describe("POST /batch", () => { const body = await json(res); expect(body.batch).toBe(true); expect(body.processed).toBe(2); + expect(mockInsertTrackEventsBatch).toHaveBeenCalledWith([ + { + event: expect.objectContaining({ id: "built_id" }), + sourceEventId: "evt_1", + }, + { + event: expect.objectContaining({ id: "built_id" }), + sourceEventId: "evt_2", + }, + ]); + }); + + test("waits for both delivery topics before returning a retryable failure", async () => { + let releaseOutgoingDelivery: (() => void) | undefined; + let responseSettled = false; + mockInsertTrackEventsBatch.mockRejectedValueOnce( + createError({ + message: "Analytics delivery temporarily unavailable", + status: 503, + }) + ); + mockInsertOutgoingLinksBatch.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseOutgoingDelivery = resolve; + }) + ); + + const response = post(basketApp, "/batch", [ + { + type: "track", + eventId: "evt_1", + name: "pageview", + path: "https://example.com/a", + }, + { + type: "outgoing_link", + eventId: "evt_link_1", + href: "https://external.com", + }, + ]); + void response.then(() => { + responseSettled = true; + }); + + await vi.waitFor(() => + expect(mockInsertOutgoingLinksBatch).toHaveBeenCalledOnce() + ); + expect(responseSettled).toBe(false); + + releaseOutgoingDelivery?.(); + const result = await response; + expect(result.status).toBe(503); }); test("not an array → 400", async () => { @@ -529,11 +585,13 @@ describe("GET /px.jpg", () => { expect(res.headers.get("Content-Type")).toBe("image/gif"); }); - test("always returns pixel even on error", async () => { + test("returns an empty retryable response when delivery fails", async () => { mockValidateRequest.mockRejectedValueOnce(new Error("boom")); const res = await get(basketApp, "/px.jpg?name=test"); - expect(res.status).toBe(200); - expect(res.headers.get("Content-Type")).toBe("image/gif"); + expect(res.status).toBe(503); + expect(res.headers.get("Content-Type")).toBeNull(); + expect(res.headers.get("Retry-After")).toBe("5"); + expect(new Uint8Array(await res.arrayBuffer())).toHaveLength(0); }); }); diff --git a/apps/basket/src/utils/pixel.test.ts b/apps/basket/src/utils/pixel.test.ts index 9c1856d13..cae9b1a25 100644 --- a/apps/basket/src/utils/pixel.test.ts +++ b/apps/basket/src/utils/pixel.test.ts @@ -19,6 +19,14 @@ describe("createPixelResponse", () => { new Uint8Array([0x47, 0x49, 0x46]) ); }); + + test("returns an empty retryable response so image transport can retry", async () => { + const r = createPixelResponse(503); + expect(r.status).toBe(503); + expect(r.headers.get("Content-Type")).toBeNull(); + expect(r.headers.get("Retry-After")).toBe("5"); + expect(new Uint8Array(await r.arrayBuffer())).toHaveLength(0); + }); }); // ── parsePixelQuery ── diff --git a/apps/basket/src/utils/pixel.ts b/apps/basket/src/utils/pixel.ts index f0cf9e9b1..91a5d0d25 100644 --- a/apps/basket/src/utils/pixel.ts +++ b/apps/basket/src/utils/pixel.ts @@ -13,14 +13,26 @@ const TRANSPARENT_PIXEL = Buffer.from( /** * Returns a 1x1 transparent GIF response */ -export function createPixelResponse(): Response { +export function createPixelResponse(status = 200): Response { + const headers = { + "Cache-Control": "no-cache, no-store, must-revalidate", + Pragma: "no-cache", + Expires: "0", + ...(status >= 500 ? { "Retry-After": "5" } : {}), + }; + + // Browsers report a decodable GIF as an image load even when its HTTP status + // is 5xx. Return no image body for failures so pixel transport receives an + // error event and retains the event for retry. + if (status >= 400) { + return new Response(null, { status, headers }); + } + return new Response(TRANSPARENT_PIXEL, { - status: 200, + status, headers: { "Content-Type": "image/gif", - "Cache-Control": "no-cache, no-store, must-revalidate", - Pragma: "no-cache", - Expires: "0", + ...headers, }, }); } diff --git a/apps/links/src/routes/redirect.route.test.ts b/apps/links/src/routes/redirect.route.test.ts index 8c73b101f..859c649f8 100644 --- a/apps/links/src/routes/redirect.route.test.ts +++ b/apps/links/src/routes/redirect.route.test.ts @@ -74,6 +74,7 @@ mock.module("../lib/logging", () => ({ captureError: mock(() => {}), mergeWideEvent: mock(() => {}), record: async (_name: string, run: () => Promise | T) => run(), + setAttributes: mock(() => {}), })); mock.module("../lib/producer", () => ({ diff --git a/apps/uptime/package.json b/apps/uptime/package.json index 0f9a8b53e..9fdcc9b6b 100644 --- a/apps/uptime/package.json +++ b/apps/uptime/package.json @@ -18,7 +18,8 @@ "effect": "^4.0.0-beta.59", "elysia": "catalog:", "evlog": "catalog:", - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "zod": "catalog:" }, "packageManager": "bun@1.3.14" } diff --git a/apps/uptime/src/actions.ts b/apps/uptime/src/actions.ts index 01c168fb3..d7ac9538c 100644 --- a/apps/uptime/src/actions.ts +++ b/apps/uptime/src/actions.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { connect } from "node:tls"; import { db } from "@databuddy/db"; import { @@ -429,6 +430,7 @@ const runUptimeCheck = ( site_id: siteId, url: normalizedUrl, timestamp, + event_id: randomUUID(), status: pingResult.ok ? MonitorStatus.UP : MonitorStatus.DOWN, http_code: pingResult.statusCode, ttfb_ms: pingResult.ttfb, diff --git a/apps/uptime/src/index.ts b/apps/uptime/src/index.ts index a5a87a942..67b75a1e9 100644 --- a/apps/uptime/src/index.ts +++ b/apps/uptime/src/index.ts @@ -15,7 +15,7 @@ import { import { disconnectProducer } from "./lib/producer"; import { captureError } from "./lib/tracing"; import { syncSchedulers } from "./sync-schedulers"; -import { startUptimeWorker } from "./worker"; +import { startUptimeDeliveryWorker, startUptimeWorker } from "./worker"; initLogger({ env: { @@ -29,6 +29,12 @@ initLogger({ sampling: {}, }); +let shuttingDown = false; +let shutdownExitCode = 0; +let uptimeWorker: ReturnType | null = null; +let uptimeDeliveryWorker: ReturnType | null = + null; + process.on("unhandledRejection", (reason, _promise) => { captureError(reason, { process: "unhandledRejection" }); log.error({ @@ -45,54 +51,116 @@ process.on("uncaughtException", (error) => { error_stack: error instanceof Error ? error.stack : undefined, error_source: "process", }); + // Continuing after an uncaught exception can corrupt worker state. Drain the + // durable relay within its normal bound, then let the platform restart us. + shutdown("uncaughtException", 1).catch((shutdownError) => { + captureError(shutdownError, { + process: "uncaughtException", + error_step: "fatal_shutdown", + }); + process.exit(1); + }); }); const DRAIN_TIMEOUT_MS = 10_000; -const drainAll = (worker: ReturnType | null) => - Effect.all( - [ - Effect.tryPromise({ - try: () => worker?.close() ?? Promise.resolve(), - catch: (c) => c, - }), - Effect.tryPromise({ try: () => closeUptimeQueue(), catch: (c) => c }), - Effect.tryPromise({ - try: () => flushBatchedUptimeDrain(), - catch: (c) => c, - }), - Effect.tryPromise({ - try: () => shutdownPostgres(), - catch: (c) => c, - }), - Effect.tryPromise({ try: () => disconnectProducer(), catch: (c) => c }), - ], - { concurrency: "unbounded" } - ).pipe( +const drainStep = (step: string, action: () => Promise) => + Effect.tryPromise({ + try: action, + catch: (cause) => cause, + }).pipe( + Effect.catch((cause) => + Effect.sync(() => + log.error({ + lifecycle: "shutdown", + error_step: step, + error_message: cause instanceof Error ? cause.message : String(cause), + }) + ) + ) + ); + +const drainAll = ( + worker: ReturnType | null, + deliveryWorker: ReturnType | null +) => + Effect.gen(function* () { + // Stop source admission before delivery so an acknowledged checkpoint is + // never handed to a producer that is already closing. Each cleanup is + // isolated: a failed close must never skip the queue, producer, or log drain. + yield* drainStep( + "uptime_worker_close", + () => worker?.close() ?? Promise.resolve() + ); + yield* drainStep( + "uptime_delivery_worker_close", + () => deliveryWorker?.close() ?? Promise.resolve() + ); + yield* Effect.all( + [ + drainStep("uptime_queue_close", () => closeUptimeQueue()), + drainStep("uptime_log_flush", () => flushBatchedUptimeDrain()), + drainStep("uptime_postgres_close", () => shutdownPostgres()), + drainStep("uptime_producer_disconnect", () => disconnectProducer()), + ], + { concurrency: "unbounded" } + ); + }).pipe( Effect.timeout(`${DRAIN_TIMEOUT_MS} millis`), - Effect.catch(() => + Effect.catch((cause) => Effect.sync(() => log.error({ lifecycle: "shutdown", - error_step: "drain_timeout", + error_step: + cause && + typeof cause === "object" && + "_tag" in cause && + cause._tag === "TimeoutError" + ? "drain_timeout" + : "drain_failed", drain_timeout_ms: DRAIN_TIMEOUT_MS, + error_message: cause instanceof Error ? cause.message : String(cause), }) ) ) ); -async function shutdown(signal: string) { +async function shutdown(signal: string, exitCode = 0) { + shutdownExitCode = Math.max(shutdownExitCode, exitCode); + if (shuttingDown) { + return; + } + shuttingDown = true; log.info("lifecycle", `${signal} received, shutting down gracefully`); - await Effect.runPromise(drainAll(uptimeWorker)); - process.exit(0); + try { + await Effect.runPromise(drainAll(uptimeWorker, uptimeDeliveryWorker)); + } catch (error) { + captureError(error, { lifecycle: "shutdown", error_step: "drain_failed" }); + log.error({ + lifecycle: "shutdown", + error_step: "drain_failed", + error_message: error instanceof Error ? error.message : String(error), + }); + } finally { + process.exit(shutdownExitCode); + } } -let uptimeWorker: ReturnType | null = null; - (async () => { if (UPTIME_ENV.isProduction) { - await syncSchedulers(); - uptimeWorker = startUptimeWorker(); + try { + await syncSchedulers(); + uptimeDeliveryWorker = startUptimeDeliveryWorker(); + uptimeWorker = startUptimeWorker(); + } catch (error) { + captureError(error, { error_step: "uptime_startup" }); + log.error({ + lifecycle: "startup", + error_step: "uptime_startup", + error_message: error instanceof Error ? error.message : String(error), + }); + process.exit(1); + } } else { log.info( "lifecycle", @@ -161,15 +229,16 @@ const healthCheck = Effect.gen(function* () { clientId: "health", brokers: [broker], connectionTimeout: 5000, - ...(process.env.REDPANDA_USER && - process.env.REDPANDA_PASSWORD && { - sasl: { - mechanism: "scram-sha-256", - username: process.env.REDPANDA_USER, - password: process.env.REDPANDA_PASSWORD, - }, - ssl: process.env.REDPANDA_SSL === "true", - }), + ...(process.env.REDPANDA_USER && process.env.REDPANDA_PASSWORD + ? { + sasl: { + mechanism: "scram-sha-256", + username: process.env.REDPANDA_USER, + password: process.env.REDPANDA_PASSWORD, + }, + } + : {}), + ...(process.env.REDPANDA_SSL === "true" ? { ssl: true } : {}), }); const admin = kafka.admin(); try { @@ -182,7 +251,11 @@ const healthCheck = Effect.gen(function* () { { concurrency: "unbounded" } ); - const services = { postgres, bullmqRedis, redpanda }; + const services = { + postgres, + bullmqRedis, + redpanda, + }; const status = Object.values(services).every((s) => s.status === "ok") ? "ok" : "degraded"; diff --git a/apps/uptime/src/lib/producer.test.ts b/apps/uptime/src/lib/producer.test.ts new file mode 100644 index 000000000..4d36e3635 --- /dev/null +++ b/apps/uptime/src/lib/producer.test.ts @@ -0,0 +1,157 @@ +import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test"; + +const kafkaConfigs: unknown[] = []; +const producers: Array> = []; +const captureError = mock(() => {}); + +class KafkaMock { + constructor(config: unknown) { + kafkaConfigs.push(config); + } + + producer() { + const producer = producers.shift(); + if (!producer) { + throw new Error("No test producer configured"); + } + return producer; + } +} + +mock.module("kafkajs", () => ({ + CompressionTypes: { GZIP: 1 }, + Kafka: KafkaMock, +})); + +mock.module("./tracing", () => ({ captureError })); + +const { disconnectProducer, sendUptimeEvent } = await import("./producer"); + +const environmentKeys = [ + "REDPANDA_BROKER", + "REDPANDA_PASSWORD", + "REDPANDA_SSL", + "REDPANDA_USER", +] as const; +const originalEnvironment = new Map( + environmentKeys.map((key) => [key, process.env[key]]) +); + +function createProducer( + overrides: Partial<{ + connect: () => Promise; + disconnect: () => Promise; + send: () => Promise; + }> = {} +) { + return { + connect: mock(overrides.connect ?? (() => Promise.resolve())), + disconnect: mock(overrides.disconnect ?? (() => Promise.resolve())), + send: mock(overrides.send ?? (() => Promise.resolve())), + }; +} + +beforeEach(async () => { + await disconnectProducer(); + kafkaConfigs.length = 0; + producers.length = 0; + captureError.mockClear(); + process.env.REDPANDA_BROKER = "redpanda.test:9092"; + delete process.env.REDPANDA_PASSWORD; + delete process.env.REDPANDA_SSL; + delete process.env.REDPANDA_USER; +}); + +afterAll(async () => { + await disconnectProducer(); + for (const key of environmentKeys) { + const value = originalEnvironment.get(key); + if (value === undefined) { + delete process.env[key]; + continue; + } + process.env[key] = value; + } +}); + +describe("sendUptimeEvent", () => { + test("shares one in-flight connection across concurrent cold-start sends", async () => { + let resolveConnection: (() => void) | undefined; + const producer = createProducer({ + connect: () => + new Promise((resolve) => { + resolveConnection = resolve; + }), + }); + producers.push(producer); + + const sends = Array.from({ length: 20 }, () => sendUptimeEvent({ ok: true })); + + expect(producer.connect).toHaveBeenCalledTimes(1); + expect(resolveConnection).toBeDefined(); + resolveConnection?.(); + + await Promise.all(sends); + expect(producer.send).toHaveBeenCalledTimes(20); + expect(producer.send).toHaveBeenCalledWith( + expect.objectContaining({ acks: -1 }) + ); + expect(kafkaConfigs).toHaveLength(1); + }); + + test("disconnects a failed producer and reconnects for the next event", async () => { + const failedProducer = createProducer({ + send: () => Promise.reject(new Error("broker unavailable")), + }); + const recoveredProducer = createProducer(); + producers.push(failedProducer, recoveredProducer); + + await expect(sendUptimeEvent({ attempt: 1 })).rejects.toThrow( + "broker unavailable" + ); + expect(failedProducer.disconnect).toHaveBeenCalledTimes(1); + + await expect(sendUptimeEvent({ attempt: 2 })).resolves.toBeUndefined(); + expect(recoveredProducer.connect).toHaveBeenCalledTimes(1); + expect(recoveredProducer.send).toHaveBeenCalledTimes(1); + expect(kafkaConfigs).toHaveLength(2); + }); + + test("reconnects after a failed cold-start connection", async () => { + const failedProducer = createProducer({ + connect: () => Promise.reject(new Error("broker unavailable")), + }); + const recoveredProducer = createProducer(); + producers.push(failedProducer, recoveredProducer); + + await expect(sendUptimeEvent({ attempt: 1 })).rejects.toThrow( + "broker unavailable" + ); + + await expect(sendUptimeEvent({ attempt: 2 })).resolves.toBeUndefined(); + expect(recoveredProducer.connect).toHaveBeenCalledTimes(1); + expect(recoveredProducer.send).toHaveBeenCalledTimes(1); + expect(kafkaConfigs).toHaveLength(2); + }); + + test("rejects when Kafka is not configured", async () => { + delete process.env.REDPANDA_BROKER; + + await expect(sendUptimeEvent({ ok: true })).rejects.toThrow( + "REDPANDA_BROKER not set" + ); + expect(kafkaConfigs).toEqual([]); + }); + + test("uses TLS when configured even without SASL credentials", async () => { + process.env.REDPANDA_SSL = "true"; + const producer = createProducer(); + producers.push(producer); + + await expect(sendUptimeEvent({ ok: true })).resolves.toBeUndefined(); + expect(kafkaConfigs[0]).toEqual( + expect.objectContaining({ ssl: true }) + ); + expect(kafkaConfigs[0]).not.toHaveProperty("sasl"); + }); +}); diff --git a/apps/uptime/src/lib/producer.ts b/apps/uptime/src/lib/producer.ts index 2ba8ca4e6..46974e362 100644 --- a/apps/uptime/src/lib/producer.ts +++ b/apps/uptime/src/lib/producer.ts @@ -1,15 +1,8 @@ import { CompressionTypes, Kafka, type Producer } from "kafkajs"; -import { Context, Data, Effect, Layer } from "effect"; import { captureError } from "./tracing"; const TOPIC = "analytics-uptime-checks"; -class KafkaSendError extends Data.TaggedError("KafkaSendError")<{ - cause: unknown; -}> {} - -const KafkaProducer = Context.Service("KafkaProducer"); - const connectProducer = (): Promise => { const broker = process.env.REDPANDA_BROKER; if (!broker) { @@ -21,11 +14,10 @@ const connectProducer = (): Promise => { const kafka = new Kafka({ brokers: [broker], clientId: "uptime-producer", - ...(username && - password && { - sasl: { mechanism: "scram-sha-256", username, password }, - ssl: process.env.REDPANDA_SSL === "true", - }), + ...(username && password + ? { sasl: { mechanism: "scram-sha-256", username, password } } + : {}), + ...(process.env.REDPANDA_SSL === "true" ? { ssl: true } : {}), }); const producer = kafka.producer({ @@ -37,74 +29,46 @@ const connectProducer = (): Promise => { return producer.connect().then(() => producer); }; -const KafkaProducerLive = Layer.effect( - KafkaProducer, - Effect.acquireRelease( - Effect.tryPromise({ - try: connectProducer, - catch: (cause) => { - captureError(cause, { error_step: "kafka_producer_connect" }); - return cause as Error; - }, - }), - (producer) => - Effect.tryPromise({ - try: () => producer.disconnect(), - catch: (cause) => cause, - }).pipe( - Effect.catch((cause) => { - captureError(cause, { - error_step: "kafka_producer_disconnect", - }); - return Effect.void; - }) - ) - ) -); - -const sendEvent = (event: unknown, key?: string) => - Effect.gen(function* () { - const producer = yield* KafkaProducer; - yield* Effect.tryPromise({ - try: () => - producer.send({ - topic: TOPIC, - messages: [ - { - value: JSON.stringify(event, (_k, v) => - v === undefined ? null : v - ), - key, - }, - ], - compression: CompressionTypes.GZIP, - }), - catch: (cause) => new KafkaSendError({ cause }), - }); - }); - -export { KafkaProducer, KafkaProducerLive, KafkaSendError, sendEvent }; - let singletonProducer: Producer | null = null; -let singletonConnected = false; +let singletonConnection: Promise | null = null; -async function ensureProducer(): Promise { - if (singletonConnected && singletonProducer) { - return singletonProducer; +function ensureProducer(): Promise { + if (singletonProducer) { + return Promise.resolve(singletonProducer); } - if (!process.env.REDPANDA_BROKER) { - return null; + if (singletonConnection) { + return singletonConnection; } + singletonConnection = connectProducer() + .then((producer) => { + singletonProducer = producer; + return producer; + }) + .catch((error) => { + captureError(error, { error_step: "kafka_producer_connect" }); + singletonProducer = null; + throw error; + }) + .finally(() => { + singletonConnection = null; + }); + + return singletonConnection; +} + +async function resetProducer(producer: Producer): Promise { + if (singletonProducer !== producer) { + return; + } + + singletonProducer = null; + try { - singletonProducer = await connectProducer(); - singletonConnected = true; - return singletonProducer; + await producer.disconnect(); } catch (error) { - captureError(error, { error_step: "kafka_producer_connect" }); - singletonConnected = false; - return null; + captureError(error, { error_step: "kafka_producer_disconnect" }); } } @@ -113,13 +77,11 @@ export async function sendUptimeEvent( key?: string ): Promise { const p = await ensureProducer(); - if (!p) { - return; - } try { await p.send({ topic: TOPIC, + acks: -1, messages: [ { value: JSON.stringify(event, (_k, v) => (v === undefined ? null : v)), @@ -130,18 +92,16 @@ export async function sendUptimeEvent( }); } catch (error) { captureError(error, { error_step: "kafka_producer_send" }); + await resetProducer(p); + throw error; } } export async function disconnectProducer(): Promise { - if (!singletonProducer) { + const producer = + singletonProducer ?? (await singletonConnection?.catch(() => null)); + if (!producer) { return; } - try { - await singletonProducer.disconnect(); - } catch (error) { - captureError(error, { error_step: "kafka_producer_disconnect" }); - } - singletonProducer = null; - singletonConnected = false; + await resetProducer(producer); } diff --git a/apps/uptime/src/sync-schedulers.test.ts b/apps/uptime/src/sync-schedulers.test.ts new file mode 100644 index 000000000..a312a4230 --- /dev/null +++ b/apps/uptime/src/sync-schedulers.test.ts @@ -0,0 +1,71 @@ +import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import * as actualDb from "@databuddy/db"; +import * as actualSchema from "@databuddy/db/schema"; +import * as actualRedis from "@databuddy/redis"; +import * as actualEvlog from "evlog"; + +const monitors = [{ granularity: "five_minutes", id: "schedule-1" }]; +const upsertJobScheduler = mock(async () => undefined); +const dbSelect = mock(() => ({ + from: () => ({ where: async () => monitors }), +})); +const logInfo = mock(() => {}); + +mock.module("@databuddy/db", () => ({ + ...actualDb, + db: { select: dbSelect }, + eq: mock(() => undefined), +})); +mock.module("@databuddy/db/schema", () => ({ + ...actualSchema, + uptimeSchedules: { + granularity: "granularity", + id: "id", + isPaused: "isPaused", + }, +})); +mock.module("@databuddy/redis", () => ({ + ...actualRedis, + getUptimeQueue: () => ({ upsertJobScheduler }), + UPTIME_CHECK_JOB_NAME: "uptime-check", + UPTIME_JOB_OPTIONS: { attempts: 1_000_000 }, + uptimeSchedulerId: (scheduleId: string) => `uptime-${scheduleId}`, +})); +mock.module("evlog", () => ({ + ...actualEvlog, + log: { error: mock(() => {}), info: logInfo }, +})); + +const { syncSchedulers } = await import("./sync-schedulers"); + +beforeEach(() => { + dbSelect.mockClear(); + logInfo.mockClear(); + upsertJobScheduler.mockClear(); +}); + +afterAll(() => { + mock.module("@databuddy/db", () => actualDb); + mock.module("@databuddy/db/schema", () => actualSchema); + mock.module("@databuddy/redis", () => actualRedis); + mock.module("evlog", () => actualEvlog); +}); + +describe("syncSchedulers", () => { + test("upserts an active scheduler with current durable job options", async () => { + await syncSchedulers(); + + expect(upsertJobScheduler).toHaveBeenCalledWith( + "uptime-schedule-1", + { pattern: "*/5 * * * *" }, + { + data: { scheduleId: "schedule-1", trigger: "scheduled" }, + name: "uptime-check", + opts: { attempts: 1_000_000 }, + } + ); + expect(logInfo).toHaveBeenCalledWith( + expect.objectContaining({ failed: 0, total: 1, upserted: 1 }) + ); + }); +}); diff --git a/apps/uptime/src/sync-schedulers.ts b/apps/uptime/src/sync-schedulers.ts index da55ad9e2..4747f5a68 100644 --- a/apps/uptime/src/sync-schedulers.ts +++ b/apps/uptime/src/sync-schedulers.ts @@ -31,15 +31,6 @@ const syncMonitor = ( ) => Effect.gen(function* () { const schedulerId = uptimeSchedulerId(monitor.id); - - const existing = yield* Effect.tryPromise({ - try: () => queue.getJobScheduler(schedulerId), - catch: (cause) => cause, - }); - if (existing) { - return "skipped" as const; - } - const pattern = CRON_GRANULARITIES[monitor.granularity]; if (!pattern) { return yield* Effect.fail( @@ -66,8 +57,6 @@ const syncMonitor = ( ), catch: (cause) => cause, }); - - return "created" as const; }); const syncAll = Effect.gen(function* () { @@ -85,17 +74,12 @@ const syncAll = Effect.gen(function* () { catch: (cause) => cause, }); - const created = yield* Ref.make(0); - const skipped = yield* Ref.make(0); + const upserted = yield* Ref.make(0); const failed = yield* Ref.make(0); for (const monitor of monitors) { yield* syncMonitor(monitor, queue).pipe( - Effect.tap((result) => - result === "created" - ? Ref.update(created, (n) => n + 1) - : Ref.update(skipped, (n) => n + 1) - ), + Effect.tap(() => Ref.update(upserted, (n) => n + 1)), Effect.catch((error) => { if (error instanceof UnknownGranularity) { log.error({ @@ -116,17 +100,12 @@ const syncAll = Effect.gen(function* () { ); } - const [c, s, f] = yield* Effect.all([ - Ref.get(created), - Ref.get(skipped), - Ref.get(failed), - ]); + const [u, f] = yield* Effect.all([Ref.get(upserted), Ref.get(failed)]); log.info({ sync: "scheduler", total: monitors.length, - created: c, - skipped: s, + upserted: u, failed: f, }); }); diff --git a/apps/uptime/src/types.ts b/apps/uptime/src/types.ts index f207bb3eb..0b225fa3d 100644 --- a/apps/uptime/src/types.ts +++ b/apps/uptime/src/types.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + export const MonitorStatus = { DOWN: 0, UP: 1, @@ -5,30 +7,49 @@ export const MonitorStatus = { MAINTENANCE: 3, } as const; -export interface UptimeData { - attempt: number; - check_type: string; - content_hash: string; - env: string; - error: string; - failure_streak: number; - http_code: number; - json_data?: string; - probe_ip: string; - probe_region: string; - redirect_count: number; - response_bytes: number; - retries: number; - site_id: string; - ssl_expiry: number; - ssl_valid: number; - status: number; - timestamp: number; - total_ms: number; - ttfb_ms: number; - url: string; - user_agent: string; -} +export const uptimeDataSchema = z.object({ + attempt: z.number(), + check_type: z.string(), + content_hash: z.string(), + env: z.string(), + error: z.string(), + event_id: z.string(), + failure_streak: z.number(), + http_code: z.number(), + json_data: z.string().optional(), + probe_ip: z.string(), + probe_region: z.string(), + redirect_count: z.number(), + response_bytes: z.number(), + retries: z.number(), + site_id: z.string(), + ssl_expiry: z.number(), + ssl_valid: z.number(), + status: z.number(), + timestamp: z.number(), + total_ms: z.number(), + ttfb_ms: z.number(), + url: z.string(), + user_agent: z.string(), +}); + +const requiredUnknownSchema = z + .unknown() + .refine((value) => value !== undefined, "Required"); + +export const uptimeCheckJobDataSchema = z + .object({ + delivery: z.object({ event: requiredUnknownSchema }).optional(), + scheduleId: z.string(), + trigger: z.enum(["manual", "scheduled"]), + }) + .passthrough(); + +export const uptimeDeliveryJobDataSchema = z.object({ + event: requiredUnknownSchema, +}); + +export type UptimeData = z.infer; export type ScheduleLookupReason = "not_found" | "malformed" | "transient"; diff --git a/apps/uptime/src/uptime-transition-alerts.test.ts b/apps/uptime/src/uptime-transition-alerts.test.ts index f0a921a48..d41a5f1cc 100644 --- a/apps/uptime/src/uptime-transition-alerts.test.ts +++ b/apps/uptime/src/uptime-transition-alerts.test.ts @@ -16,6 +16,7 @@ const baseUptimeData: UptimeData = { attempt: 1, check_type: "http", content_hash: "", + event_id: "uptime-event-1", env: "production", error: "", failure_streak: 0, diff --git a/apps/uptime/src/uptime-transition-alerts.ts b/apps/uptime/src/uptime-transition-alerts.ts index 1c1f64e58..87c755019 100644 --- a/apps/uptime/src/uptime-transition-alerts.ts +++ b/apps/uptime/src/uptime-transition-alerts.ts @@ -91,7 +91,7 @@ export function shouldReleaseTransitionClaim( if (firedAlarmCount > 0) { // A successful non-email destination owns the transition claim. Releasing // it would duplicate that delivery; retrying only the deferred email needs - // a durable per-destination outbox. + // a dedicated durable delivery record. return false; } return sendableAlarmCount > 0 || emailDeliveryDeferred; diff --git a/apps/uptime/src/worker.test.ts b/apps/uptime/src/worker.test.ts index c1c4e2096..787d3a5c8 100644 --- a/apps/uptime/src/worker.test.ts +++ b/apps/uptime/src/worker.test.ts @@ -1,10 +1,12 @@ import { beforeEach, describe, expect, it } from "bun:test"; +import type { UptimeCheckJobData } from "@databuddy/redis"; import type { ScheduleData } from "./actions"; import type { UptimeData } from "./types"; import { DEFAULT_UPTIME_WORKER_CONCURRENCY, getUptimeWorkerConcurrency, processUptimeCheck, + processUptimeDeliveryJob, processUptimeJob, type UptimeWorkerDeps, } from "./worker"; @@ -18,11 +20,13 @@ const calls = { cacheBust: boolean | undefined; extractHealth: boolean | undefined; }>, + checkpoint: [] as UptimeData[], + delivery: [] as UptimeData[], email: [] as Array<{ schedule: ScheduleData; data: UptimeData }>, loggerFields: [] as Array>, loggerEmitted: [] as Array, + order: [] as string[], reaped: [] as string[], - send: [] as Array<{ data: UptimeData; monitorId: string }>, }; let lookupResult: @@ -55,6 +59,7 @@ function uptimeData(values: Partial = {}): UptimeData { attempt: 1, check_type: "http", content_hash: "hash", + event_id: "uptime-event-1", env: "test", error: "", failure_streak: 0, @@ -104,6 +109,10 @@ function deps(): UptimeWorkerDeps { error: () => {}, } as never; }, + enqueueUptimeDelivery: async (data) => { + calls.delivery.push(data); + calls.order.push("enqueue"); + }, getPreviousMonitorStatus: async () => previousStatus, isHealthExtractionEnabled: (config) => typeof config === "object" && @@ -111,30 +120,50 @@ function deps(): UptimeWorkerDeps { "enabled" in config && config.enabled === true, lookupSchedule: async () => lookupResult, + sendUptimeEvent: async () => {}, reapOrphanScheduler: async (scheduleId: string) => { calls.reaped.push(scheduleId); if (reapBehaviour === "throw") { throw new Error("redis reap blew up"); } }, - sendUptimeEvent: async (data, monitorId) => { - calls.send.push({ data, monitorId }); - }, fireTransitionAlerts: async (payload) => { calls.email.push(payload); + calls.order.push("alert"); return { transition_kind: null, alarms_fired: 0 }; }, }; } +type UptimeEventCheckpoint = (data: UptimeData) => Promise; +const noOpCheckpoint: UptimeEventCheckpoint = async () => {}; + +function processUptimeCheckForTest( + scheduleId: string, + trigger: "manual" | "scheduled", + workerDeps: UptimeWorkerDeps = deps(), + jobMeta?: { id?: string; attempt?: number }, + checkpoint: UptimeEventCheckpoint = noOpCheckpoint +) { + return processUptimeCheck( + scheduleId, + trigger, + workerDeps, + jobMeta, + checkpoint + ); +} + beforeEach(() => { calls.captureError = []; calls.check = []; + calls.checkpoint = []; + calls.delivery = []; calls.email = []; calls.loggerFields = []; calls.loggerEmitted = []; + calls.order = []; calls.reaped = []; - calls.send = []; lookupResult = { success: true, data: schedule() }; checkResult = { success: true, data: uptimeData() }; previousStatus = 0; @@ -173,6 +202,7 @@ describe("processUptimeCheck", () => { { name: "surprise", data: { scheduleId: "schedule-1", trigger: "scheduled" }, + updateData: async () => {}, }, deps() ) @@ -186,18 +216,64 @@ describe("processUptimeCheck", () => { { name: "uptime-check", data: { scheduleId: "schedule-1", trigger: "manual" }, + updateData: async (data) => { + calls.checkpoint.push(data.delivery?.event as UptimeData); + }, }, deps() ); expect(calls.check).toHaveLength(1); + expect(calls.checkpoint).toEqual([uptimeData()]); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ uptime_trigger: "manual" }) ); }); + it("preserves unknown job metadata when checkpointing a delivery", async () => { + let checkpoint: unknown; + const data = { + scheduleId: "schedule-1", + trigger: "manual", + futureMetadata: { retryToken: "retry-1" }, + } as UptimeCheckJobData & { + futureMetadata: { retryToken: string }; + }; + + await processUptimeJob( + { + name: "uptime-check", + data, + updateData: async (updated) => { + checkpoint = updated; + }, + }, + deps() + ); + + expect(checkpoint).toMatchObject({ + delivery: { event: uptimeData() }, + futureMetadata: { retryToken: "retry-1" }, + }); + }); + + it("rejects malformed BullMQ job envelopes before accessing their fields", async () => { + await expect( + processUptimeJob( + { + name: "uptime-check", + data: null as never, + updateData: async () => {}, + }, + deps() + ) + ).rejects.toThrow("Invalid uptime job payload"); + + expect(calls.check).toEqual([]); + }); + it("runs a scheduled check and emits events, status, and transition email work", async () => { - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.check).toEqual([ { @@ -208,10 +284,9 @@ describe("processUptimeCheck", () => { extractHealth: true, }, ]); - expect(calls.send).toEqual([ - { data: uptimeData(), monitorId: "website-1" }, - ]); + expect(calls.delivery).toEqual([uptimeData()]); expect(calls.email).toHaveLength(1); + expect(calls.order).toEqual(["enqueue", "alert"]); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ schedule_id: "schedule-1", @@ -229,6 +304,7 @@ describe("processUptimeCheck", () => { ); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ + event_id: "uptime-event-1", outcome: "up", previous_uptime_status: 0, ttfb_ms: 10, @@ -236,7 +312,7 @@ describe("processUptimeCheck", () => { }) ); expect(calls.loggerFields).toContainEqual( - expect.objectContaining({ kafka_sent: true }) + expect.objectContaining({ delivery_queue_admitted: true }) ); expect(calls.loggerEmitted).toHaveLength(1); }); @@ -244,7 +320,7 @@ describe("processUptimeCheck", () => { it("records -1 when no previous monitor status exists", async () => { previousStatus = undefined; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ previous_uptime_status: -1 }) @@ -257,7 +333,7 @@ describe("processUptimeCheck", () => { data: schedule({ website: null, websiteId: null, timeout: null }), }; - await processUptimeCheck("schedule-only", "manual", deps()); + await processUptimeCheckForTest("schedule-only", "manual", deps()); expect(calls.check).toEqual([ { @@ -285,10 +361,10 @@ describe("processUptimeCheck", () => { it("skips paused schedules without running the check", async () => { lookupResult = { success: true, data: schedule({ isPaused: true }) }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.check).toEqual([]); - expect(calls.send).toEqual([]); + expect(calls.delivery).toEqual([]); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ organization_id: "org-1" }) ); @@ -301,7 +377,7 @@ describe("processUptimeCheck", () => { it("skips missing schedules without throwing", async () => { lookupResult = { success: false, error: "not found" }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.check).toEqual([]); expect(calls.loggerFields).toContainEqual( @@ -320,7 +396,7 @@ describe("processUptimeCheck", () => { reason: "not_found", }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual(["schedule-1"]); @@ -339,7 +415,7 @@ describe("processUptimeCheck", () => { reason: "malformed", }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual(["schedule-1"]); @@ -355,7 +431,7 @@ describe("processUptimeCheck", () => { reason: "transient", }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual([]); @@ -367,7 +443,7 @@ describe("processUptimeCheck", () => { it("does NOT reap when reason is missing on legacy failures (fail-open)", async () => { lookupResult = { success: false, error: "boom" }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual([]); @@ -381,7 +457,7 @@ describe("processUptimeCheck", () => { }; reapBehaviour = "throw"; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual(["schedule-1"]); @@ -406,7 +482,7 @@ describe("processUptimeCheck", () => { checkResult = { success: false, error: "timeout" }; await expect( - processUptimeCheck("schedule-1", "scheduled", deps()) + processUptimeCheckForTest("schedule-1", "scheduled", deps()) ).rejects.toThrow("timeout"); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ @@ -417,20 +493,162 @@ describe("processUptimeCheck", () => { expect(calls.loggerEmitted).toHaveLength(1); }); - it("captures producer errors on the wide event without failing the job", async () => { + it("persists the exact event before enqueueing it for delivery", async () => { + await processUptimeCheckForTest( + "schedule-1", + "manual", + deps(), + undefined, + async (data) => { + calls.checkpoint.push(data); + calls.order.push("checkpoint"); + } + ); + + expect(calls.checkpoint).toEqual([uptimeData()]); + expect(calls.delivery).toEqual([uptimeData()]); + expect(calls.order).toEqual(["checkpoint", "enqueue", "alert"]); + }); + + it("retries the source job when the durable Redis checkpoint fails", async () => { + await expect( + processUptimeCheckForTest( + "schedule-1", + "manual", + deps(), + undefined, + async () => { + throw new Error("redis unavailable"); + } + ) + ).rejects.toThrow("redis unavailable"); + + expect(calls.delivery).toEqual([]); + expect(calls.email).toEqual([]); + expect(calls.captureError).toContainEqual( + expect.objectContaining({ + context: expect.objectContaining({ + error_step: "uptime_delivery_checkpoint", + event_id: "uptime-event-1", + }), + }) + ); + }); + + it("retries the source job when delivery queue admission fails", async () => { + const failingDeps = deps(); + failingDeps.enqueueUptimeDelivery = async () => { + throw new Error("redis unavailable"); + }; + + await expect( + processUptimeCheckForTest("schedule-1", "manual", failingDeps) + ).rejects.toThrow("redis unavailable"); + + expect(calls.email).toEqual([]); + expect(calls.captureError).toContainEqual( + expect.objectContaining({ + context: expect.objectContaining({ + error_step: "uptime_delivery_enqueue", + event_id: "uptime-event-1", + }), + }) + ); + }); + + it("replays a checkpointed event without running another probe", async () => { + await processUptimeJob( + { + name: "uptime-check", + data: { + delivery: { event: uptimeData() }, + scheduleId: "schedule-1", + trigger: "scheduled", + }, + updateData: async () => {}, + }, + deps() + ); + + expect(calls.check).toEqual([]); + expect(calls.delivery).toEqual([uptimeData()]); + expect(calls.email).toHaveLength(1); + }); + + it("rejects malformed checkpointed delivery payloads before replaying", async () => { + await expect( + processUptimeJob( + { + name: "uptime-check", + data: { + delivery: { event: { ...uptimeData(), http_code: "200" } }, + scheduleId: "schedule-1", + trigger: "scheduled", + }, + updateData: async () => {}, + }, + deps() + ) + ).rejects.toThrow("Invalid persisted uptime delivery payload"); + + expect(calls.check).toEqual([]); + expect(calls.delivery).toEqual([]); + }); + + it("retries a delivery job when Redpanda rejects it", async () => { const failingDeps = deps(); failingDeps.sendUptimeEvent = async () => { - throw new Error("producer unavailable"); + throw new Error("Redpanda send failed"); }; - await processUptimeCheck("schedule-1", "manual", failingDeps); + await expect( + processUptimeDeliveryJob( + { + data: { event: uptimeData() }, + id: "uptime-delivery-uptime-event-1", + name: "uptime-event-delivery", + }, + failingDeps + ) + ).rejects.toThrow("Redpanda send failed"); - expect(calls.loggerFields).toContainEqual( + expect(calls.captureError).toContainEqual( expect.objectContaining({ - kafka_sent: false, - kafka_error: "producer unavailable", + context: expect.objectContaining({ + error_step: "uptime_delivery_send", + event_id: "uptime-event-1", + }), }) ); - expect(calls.loggerEmitted).toHaveLength(1); + }); + + it("rejects malformed delivery queue payloads before sending", async () => { + await expect( + processUptimeDeliveryJob( + { + data: { event: { ...uptimeData(), site_id: 1 } }, + id: "uptime-delivery-uptime-event-1", + name: "uptime-event-delivery", + }, + deps() + ) + ).rejects.toThrow("Invalid uptime delivery payload"); + + expect(calls.captureError).toEqual([]); + }); + + it("rejects malformed delivery job envelopes before accessing their fields", async () => { + await expect( + processUptimeDeliveryJob( + { + data: null as never, + id: "uptime-delivery-uptime-event-1", + name: "uptime-event-delivery", + }, + deps() + ) + ).rejects.toThrow("Invalid uptime delivery job payload"); + + expect(calls.delivery).toEqual([]); }); }); diff --git a/apps/uptime/src/worker.ts b/apps/uptime/src/worker.ts index d8d2a5fab..f6ba28c81 100644 --- a/apps/uptime/src/worker.ts +++ b/apps/uptime/src/worker.ts @@ -1,13 +1,18 @@ import { getBullMQWorkerConnectionOptions, + getUptimeDeliveryQueue, getUptimeQueue, type UptimeCheckJobData, + type UptimeDeliveryJobData, UPTIME_CHECK_JOB_NAME, + UPTIME_DELIVERY_JOB_NAME, + UPTIME_DELIVERY_QUEUE_NAME, UPTIME_JOB_TIMEOUT_MS, UPTIME_QUEUE_NAME, + uptimeDeliveryJobId, uptimeSchedulerId, } from "@databuddy/redis"; -import { Worker } from "bullmq"; +import { type Job, Worker } from "bullmq"; import type { RequestLogger } from "evlog"; import { createLogger, log } from "evlog"; import { Cause, Data, Effect, Exit } from "effect"; @@ -24,6 +29,9 @@ import { MonitorStatus, type ActionResult, type ScheduleLookupReason, + uptimeCheckJobDataSchema, + uptimeDataSchema, + uptimeDeliveryJobDataSchema, type UptimeData, } from "./types"; import { @@ -41,11 +49,6 @@ const REAPABLE_REASONS: ReadonlySet = new Set([ "malformed", ]); -async function defaultReapOrphanScheduler(scheduleId: string): Promise { - const queue = getUptimeQueue(); - await queue.removeJobScheduler(uptimeSchedulerId(scheduleId)); -} - class SchedulePaused extends Data.TaggedError("SchedulePaused")< Record > {} @@ -54,6 +57,10 @@ class CheckFailed extends Data.TaggedError("CheckFailed")<{ message: string; }> {} +class DeliveryHandoffFailed extends Data.TaggedError("DeliveryHandoffFailed")<{ + message: string; +}> {} + export interface UptimeWorkerDeps { captureError: ( error: unknown, @@ -68,6 +75,7 @@ export interface UptimeWorkerDeps { createLogger: ( fields: Record ) => RequestLogger; + enqueueUptimeDelivery: (data: UptimeData) => Promise; fireTransitionAlerts: (options: { schedule: ScheduleData; data: UptimeData; @@ -80,22 +88,32 @@ export interface UptimeWorkerDeps { isHealthExtractionEnabled: (config: unknown) => boolean; lookupSchedule: (scheduleId: string) => Promise>; reapOrphanScheduler: (scheduleId: string) => Promise; - sendUptimeEvent: (data: UptimeData, monitorId: string) => Promise; + sendUptimeEvent: (event: unknown, key?: string) => Promise; } const uptimeWorkerDeps: UptimeWorkerDeps = { captureError, checkUptime, createLogger: (fields) => createLogger(fields), + enqueueUptimeDelivery: async (data) => { + await getUptimeDeliveryQueue().add( + UPTIME_DELIVERY_JOB_NAME, + { event: data }, + { jobId: uptimeDeliveryJobId(data.event_id) } + ); + }, getPreviousMonitorStatus, isHealthExtractionEnabled, lookupSchedule, - reapOrphanScheduler: defaultReapOrphanScheduler, + reapOrphanScheduler: async (scheduleId) => { + await getUptimeQueue().removeJobScheduler(uptimeSchedulerId(scheduleId)); + }, sendUptimeEvent, fireTransitionAlerts, }; export const DEFAULT_UPTIME_WORKER_CONCURRENCY = 10_000; +const MAX_STALLED_COUNT = 1_000_000; export function getUptimeWorkerConcurrency( value = process.env.UPTIME_WORKER_CONCURRENCY @@ -112,12 +130,15 @@ export function getUptimeWorkerConcurrency( return parsed; } -export interface UptimeWorkerJob { - attemptsMade?: number; - data: UptimeCheckJobData; - id?: string; - name: string; -} +export type UptimeWorkerJob = Pick< + Job, + "attemptsMade" | "data" | "id" | "name" | "updateData" +>; + +export type UptimeDeliveryWorkerJob = Pick< + Job, + "attemptsMade" | "data" | "id" | "name" +>; const timed = ( label: string, @@ -171,26 +192,24 @@ const fetchPreviousStatus = (monitorId: string, deps: UptimeWorkerDeps) => Effect.orElseSucceed(() => undefined) ); -const publishEvent = ( +type UptimeEventCheckpoint = (data: UptimeData) => Promise; + +const handoffDelivery = ( data: UptimeData, - monitorId: string, - deps: UptimeWorkerDeps, - log: RequestLogger + handoff: () => Promise, + errorStep: "uptime_delivery_checkpoint" | "uptime_delivery_enqueue", + deps: UptimeWorkerDeps ) => Effect.tryPromise({ - try: () => deps.sendUptimeEvent(data, monitorId), - catch: (cause) => cause, - }).pipe( - Effect.tap(() => Effect.sync(() => log.set({ kafka_sent: true }))), - Effect.catch((error) => - Effect.sync(() => - log.set({ - kafka_sent: false, - kafka_error: error instanceof Error ? error.message : "unknown", - }) - ) - ) - ); + try: handoff, + catch: (cause) => { + deps.captureError(cause, { + error_step: errorStep, + event_id: data.event_id, + }); + return new DeliveryHandoffFailed({ message: String(cause) }); + }, + }); const runTransitionAlerts = ( schedule: ScheduleData, @@ -250,7 +269,8 @@ function reapScheduler( const processCheck = ( scheduleId: string, log: RequestLogger, - deps: UptimeWorkerDeps + deps: UptimeWorkerDeps, + checkpoint: UptimeEventCheckpoint ) => Effect.gen(function* () { const schedule = yield* timed( @@ -320,6 +340,7 @@ const processCheck = ( ); log.set({ + event_id: data.event_id, outcome: data.status === MonitorStatus.UP ? "up" : "down", previous_uptime_status: previousStatus === undefined ? -1 : previousStatus, @@ -337,7 +358,31 @@ const processCheck = ( error_message: data.error || "", }); - yield* timed("kafka", publishEvent(data, monitorId, deps, log), log); + // The source BullMQ job is the durable checkpoint. Persist the exact + // probe result before queuing it for delivery so a retry never runs a + // replacement probe with a new timestamp or event ID. + yield* timed( + "delivery_checkpoint", + handoffDelivery( + data, + () => checkpoint(data), + "uptime_delivery_checkpoint", + deps + ), + log + ); + + yield* timed( + "delivery_queue_admission", + handoffDelivery( + data, + () => deps.enqueueUptimeDelivery(data), + "uptime_delivery_enqueue", + deps + ), + log + ); + log.set({ delivery_queue_admitted: true }); yield* timed( "transition_email", @@ -349,8 +394,9 @@ const processCheck = ( export async function processUptimeCheck( scheduleId: string, trigger: UptimeCheckJobData["trigger"], - deps: UptimeWorkerDeps = uptimeWorkerDeps, - jobMeta?: { id?: string; attempt?: number } + deps: UptimeWorkerDeps, + jobMeta: { id?: string; attempt?: number } | undefined, + checkpoint: UptimeEventCheckpoint ) { const startedAt = performance.now(); const log = deps.createLogger({ @@ -360,19 +406,81 @@ export async function processUptimeCheck( ...(jobMeta?.attempt ? { job_attempt: jobMeta.attempt } : {}), }); - const exit = await Effect.runPromiseExit(processCheck(scheduleId, log, deps)); + const exit = await Effect.runPromiseExit( + processCheck(scheduleId, log, deps, checkpoint) + ); log.set({ check_duration_ms: Math.round(performance.now() - startedAt) }); log.emit(); if (Exit.isFailure(exit)) { const error = Cause.squash(exit.cause); - if (error instanceof CheckFailed) { + if ( + error instanceof CheckFailed || + error instanceof DeliveryHandoffFailed + ) { throw new Error(error.message); } } } +async function replayPersistedUptimeDelivery( + job: UptimeWorkerJob, + data: UptimeData, + deps: UptimeWorkerDeps +): Promise { + const startedAt = performance.now(); + const log = deps.createLogger({ + schedule_id: job.data.scheduleId, + uptime_trigger: job.data.trigger, + event_id: data.event_id, + delivery_replay: true, + ...(job.id ? { job_id: job.id } : {}), + ...(job.attemptsMade ? { job_attempt: job.attemptsMade } : {}), + }); + + try { + await Effect.runPromise( + timed( + "delivery_queue_admission", + handoffDelivery( + data, + () => deps.enqueueUptimeDelivery(data), + "uptime_delivery_enqueue", + deps + ), + log + ) + ); + log.set({ delivery_queue_admitted: true }); + + const scheduleExit = await Effect.runPromiseExit( + resolveSchedule(job.data.scheduleId, deps) + ); + if (Exit.isSuccess(scheduleExit)) { + await Effect.runPromise( + timed( + "transition_email", + runTransitionAlerts(scheduleExit.value, data, undefined, deps, log), + log + ) + ); + } else { + const error = Cause.squash(scheduleExit.cause); + log.set({ + transition_alert_skipped: true, + transition_alert_skip_reason: + error instanceof Error ? error.message : String(error), + }); + } + } finally { + log.set({ + delivery_replay_duration_ms: Math.round(performance.now() - startedAt), + }); + log.emit(); + } +} + export async function processUptimeJob( job: UptimeWorkerJob, deps: UptimeWorkerDeps = uptimeWorkerDeps @@ -380,10 +488,67 @@ export async function processUptimeJob( if (job.name !== UPTIME_CHECK_JOB_NAME) { throw new Error(`Unknown uptime job: ${job.name}`); } - await processUptimeCheck(job.data.scheduleId, job.data.trigger, deps, { - id: job.id, - attempt: job.attemptsMade, - }); + + const parsedJobData = uptimeCheckJobDataSchema.safeParse(job.data); + if (!parsedJobData.success) { + throw new Error("Invalid uptime job payload"); + } + + const jobData = parsedJobData.data; + const persistedEvent = jobData.delivery?.event; + if (persistedEvent !== undefined) { + const parsedEvent = uptimeDataSchema.safeParse(persistedEvent); + if (!parsedEvent.success) { + throw new Error("Invalid persisted uptime delivery payload"); + } + await replayPersistedUptimeDelivery(job, parsedEvent.data, deps); + return; + } + + await processUptimeCheck( + jobData.scheduleId, + jobData.trigger, + deps, + { + id: job.id, + attempt: job.attemptsMade, + }, + async (data) => + job.updateData({ + ...jobData, + delivery: { event: data }, + }) + ); +} + +export async function processUptimeDeliveryJob( + job: UptimeDeliveryWorkerJob, + deps: UptimeWorkerDeps = uptimeWorkerDeps +): Promise { + if (job.name !== UPTIME_DELIVERY_JOB_NAME) { + throw new Error(`Unknown uptime delivery job: ${job.name}`); + } + const parsedJobData = uptimeDeliveryJobDataSchema.safeParse(job.data); + if (!parsedJobData.success) { + throw new Error("Invalid uptime delivery job payload"); + } + + const parsedEvent = uptimeDataSchema.safeParse(parsedJobData.data.event); + if (!parsedEvent.success) { + throw new Error("Invalid uptime delivery payload"); + } + + const data = parsedEvent.data; + try { + await deps.sendUptimeEvent(data, data.site_id); + } catch (error) { + deps.captureError(error, { + error_step: "uptime_delivery_send", + event_id: data.event_id, + job_id: job.id ?? "", + }); + throw error; + } } export function startUptimeWorker() { @@ -394,20 +559,24 @@ export function startUptimeWorker() { connection: getBullMQWorkerConnectionOptions(), concurrency: getUptimeWorkerConcurrency(), lockDuration: UPTIME_JOB_TIMEOUT_MS * 3, + maxStalledCount: MAX_STALLED_COUNT, stalledInterval: UPTIME_JOB_TIMEOUT_MS * 4, } ); worker.on("failed", (job, error) => { const attemptsMade = job?.attemptsMade ?? 0; - const maxAttempts = job?.opts?.attempts ?? 3; + const maxAttempts = job?.opts?.attempts ?? 1_000_000; const isFinalAttempt = attemptsMade >= maxAttempts; + const parsedJobData = job + ? uptimeCheckJobDataSchema.safeParse(job.data) + : undefined; captureError(error, { error_step: "uptime_worker_job_failed", - schedule_id: job?.data.scheduleId ?? "", + schedule_id: parsedJobData?.success ? parsedJobData.data.scheduleId : "", job_id: job?.id ?? "", - trigger: job?.data.trigger ?? "", + trigger: parsedJobData?.success ? parsedJobData.data.trigger : "", attempts_used: attemptsMade, attempts_max: maxAttempts, is_final_attempt: isFinalAttempt, @@ -431,3 +600,50 @@ export function startUptimeWorker() { return worker; } + +export function startUptimeDeliveryWorker() { + const worker = new Worker( + UPTIME_DELIVERY_QUEUE_NAME, + (job) => processUptimeDeliveryJob(job), + { + connection: getBullMQWorkerConnectionOptions(), + concurrency: 1, + lockDuration: UPTIME_JOB_TIMEOUT_MS * 3, + maxStalledCount: MAX_STALLED_COUNT, + stalledInterval: UPTIME_JOB_TIMEOUT_MS * 4, + } + ); + + worker.on("failed", (job, error) => { + const parsedJobData = job + ? uptimeDeliveryJobDataSchema.safeParse(job.data) + : undefined; + const parsedEvent = parsedJobData?.success + ? uptimeDataSchema.safeParse(parsedJobData.data.event) + : undefined; + captureError(error, { + error_step: "uptime_delivery_worker_job_failed", + event_id: parsedEvent?.success ? parsedEvent.data.event_id : "", + job_id: job?.id ?? "", + attempts_used: job?.attemptsMade ?? 0, + attempts_max: job?.opts?.attempts ?? 1_000_000, + }); + }); + + worker.on("stalled", (jobId) => { + log.warn({ + service: "uptime", + error_step: "uptime_delivery_worker_job_stalled", + error_message: "BullMQ delivery job stalled", + job_id: jobId, + }); + }); + + worker.on("error", (error) => { + captureError(error, { + error_step: "uptime_delivery_worker_error", + }); + }); + + return worker; +} diff --git a/bun.lock b/bun.lock index 0dff8b712..d0cb64131 100644 --- a/bun.lock +++ b/bun.lock @@ -106,7 +106,6 @@ "@maxmind/geoip2-node": "^6.3.4", "@types/ua-parser-js": "^0.7.39", "async-mutex": "^0.5.0", - "effect": "^4.0.0-beta.90", "elysia": "catalog:", "evlog": "catalog:", "kafkajs": "^2.2.4", @@ -433,6 +432,7 @@ "elysia": "catalog:", "evlog": "catalog:", "kafkajs": "^2.2.4", + "zod": "catalog:", }, }, "packages/ai": { @@ -4857,8 +4857,6 @@ "@databuddy/test/drizzle-orm": ["drizzle-orm@1.0.0-rc.1", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@effect/sql-pg": ">=4.0.0-beta.58 || >=4.0.0", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@sinclair/typebox": ">=0.34.8", "@sqlitecloud/drivers": ">=1.0.653", "@tidbcloud/serverless": "*", "@tursodatabase/database": ">=0.2.1", "@tursodatabase/database-common": ">=0.2.1", "@tursodatabase/database-wasm": ">=0.2.1", "@types/better-sqlite3": "*", "@types/mssql": "^9.1.4", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "arktype": ">=2.0.0", "better-sqlite3": ">=9.3.0", "bun-types": "*", "effect": ">=4.0.0-beta.58 || >=4.0.0", "expo-sqlite": ">=14.0.0", "mssql": "^11.0.1", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5", "typebox": ">=1.0.0", "valibot": ">=1.0.0-beta.7", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@effect/sql-pg", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@sinclair/typebox", "@sqlitecloud/drivers", "@tidbcloud/serverless", "@tursodatabase/database", "@tursodatabase/database-common", "@tursodatabase/database-wasm", "@types/better-sqlite3", "@types/mssql", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "arktype", "better-sqlite3", "bun-types", "effect", "expo-sqlite", "mssql", "mysql2", "pg", "postgres", "sql.js", "sqlite3", "typebox", "valibot", "zod"] }, "sha512-jGCqAgxpz+OSHP2jQGooUHBxnFMTYl0TTRSfULBl52VNf7CtyNRnazUi+VdbSxvJrDP2lnIsmUh5O+HhKeSJCg=="], - "@databuddy/uptime/effect": ["effect@4.0.0-beta.66", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "fast-check": "^4.6.0", "find-my-way-ts": "^0.1.6", "ini": "^6.0.0", "kubernetes-types": "^1.30.0", "msgpackr": "^1.11.9", "multipasta": "^0.2.7", "toml": "^4.1.1", "uuid": "^13.0.0", "yaml": "^2.8.3" } }, "sha512-4arEr62cziFa8BBVDUwJCJJmaVepXf/kRg7KtC0h8+bufngscrHbwWFhr9c+HonwOF+31U3iD3xUJmw9KzX7Dw=="], - "@dxup/nuxt/tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="], "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], @@ -5641,12 +5639,6 @@ "@databuddy/docs/resend/svix": ["svix@1.92.2", "", { "dependencies": { "standardwebhooks": "1.0.0" } }, "sha512-ZmuA3UVvlnF9EgxlzmPtF7CKjQb64Z6OFlyfdDfU0sdcC7dJa+3aOYX5B9mA+RS6ch1AxBa4UP/l6KmqfGtWBQ=="], - "@databuddy/uptime/effect/ini": ["ini@6.0.0", "", {}, "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ=="], - - "@databuddy/uptime/effect/msgpackr": ["msgpackr@1.11.12", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.2" } }, "sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg=="], - - "@databuddy/uptime/effect/uuid": ["uuid@13.0.2", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw=="], - "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="], "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="], @@ -6461,8 +6453,6 @@ "zip-stream/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "@databuddy/uptime/effect/msgpackr/msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="], - "@inquirer/core/wrap-ansi/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], "@mapbox/node-pre-gyp/node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], @@ -6747,18 +6737,6 @@ "zip-stream/readable-stream/string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ=="], - "archiver-utils/glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], "nuxt/oxc-parser/@oxc-parser/binding-wasm32-wasi/@emnapi/core/@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], diff --git a/infra/ingest/vector.Dockerfile b/infra/ingest/vector.Dockerfile index 123c27b6c..5c29e7d7f 100644 --- a/infra/ingest/vector.Dockerfile +++ b/infra/ingest/vector.Dockerfile @@ -5,9 +5,12 @@ FROM timberio/vector:0.50.0-alpine # When building from infra/ingest/: docker build -f vector.Dockerfile -t vector . COPY infra/ingest/vector.yaml /etc/vector/vector.yaml +# Vector's source of truth is Redpanda. This service intentionally keeps no +# delivery state on Railway's ephemeral filesystem. + WORKDIR /etc/vector HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \ CMD ["vector", "validate", "/etc/vector/vector.yaml"] -CMD ["--config", "/etc/vector/vector.yaml"] \ No newline at end of file +CMD ["--config", "/etc/vector/vector.yaml"] diff --git a/infra/ingest/vector.yaml b/infra/ingest/vector.yaml index 6c59ef367..9f6540e3d 100644 --- a/infra/ingest/vector.yaml +++ b/infra/ingest/vector.yaml @@ -1,3 +1,10 @@ +# Railway service filesystems are ephemeral. Redpanda is the durable backlog: +# each acknowledged ClickHouse sink below holds its Kafka offset until the +# warehouse confirms delivery, so an interrupted process replays from Kafka. + +acknowledgements: + enabled: true + sources: redpanda: type: kafka @@ -31,9 +38,7 @@ transforms: - redpanda route: events: '.topic == "analytics-events"' - errors: '.topic == "analytics-errors"' error_spans: '.topic == "analytics-error-spans"' - web_vitals: '.topic == "analytics-web-vitals"' vitals_spans: '.topic == "analytics-vitals-spans"' custom_events: '.topic == "analytics-custom-events"' ai_traffic_spans: '.topic == "analytics-ai-traffic-spans"' @@ -63,26 +68,6 @@ sinks: encoding: timestamp_format: unix_ms - clickhouse_errors: - type: clickhouse - inputs: - - route_analytics.errors - endpoint: "${CLICKHOUSE_URL}" - database: analytics - table: errors - auth: - strategy: basic - user: "${CLICKHOUSE_USER}" - password: "${CLICKHOUSE_PASSWORD}" - batch: - max_events: 5000 - max_bytes: 5000000 - timeout_secs: 5 - date_time_best_effort: true - skip_unknown_fields: true - encoding: - timestamp_format: unix_ms - clickhouse_error_spans: type: clickhouse inputs: @@ -103,26 +88,6 @@ sinks: encoding: timestamp_format: unix_ms - clickhouse_web_vitals: - type: clickhouse - inputs: - - route_analytics.web_vitals - endpoint: "${CLICKHOUSE_URL}" - database: analytics - table: web_vitals - auth: - strategy: basic - user: "${CLICKHOUSE_USER}" - password: "${CLICKHOUSE_PASSWORD}" - batch: - max_events: 5000 - max_bytes: 5000000 - timeout_secs: 5 - date_time_best_effort: true - skip_unknown_fields: true - encoding: - timestamp_format: unix_ms - clickhouse_vitals_spans: type: clickhouse inputs: @@ -264,9 +229,17 @@ sinks: timestamp_format: unix_ms dead_letters: - type: file + type: kafka inputs: - route_analytics._unmatched - path: /var/log/vector/dead_letters.log + bootstrap_servers: "${REDPANDA_BROKER}" + topic: analytics-vector-dead-letters + sasl: + enabled: true + mechanism: SCRAM-SHA-256 + username: "${VECTOR_KAFKA_USER}" + password: "${VECTOR_KAFKA_PASSWORD}" + tls: + enabled: true encoding: codec: json diff --git a/packages/ai/src/ai/agents/cache.test.ts b/packages/ai/src/ai/agents/cache.test.ts index 51227ffc0..b53d10f00 100644 --- a/packages/ai/src/ai/agents/cache.test.ts +++ b/packages/ai/src/ai/agents/cache.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createInertRedisModule } from "../../test-utils/redis-mock"; interface RedisEntry { ttl: number; @@ -44,8 +45,8 @@ const mockEnrichAgentContext = vi.fn( }) => `fresh:${opts.organizationId ?? opts.userId}:${opts.websiteId}` ); -const passthroughCacheable = unknown>(fn: T) => - fn; +const bunRedisModule = + typeof Bun === "undefined" ? undefined : await import("@databuddy/redis"); vi.mock("@databuddy/auth", () => ({ auth: {}, @@ -54,121 +55,18 @@ vi.mock("@databuddy/auth", () => ({ }, })); -vi.mock("@databuddy/redis", () => ({ - AGENT_CONTEXT_SNAPSHOT_PREFIX: "agent:context-snapshot", - UPTIME_CHECK_JOB_NAME: "uptime-check", - UPTIME_JOB_OPTIONS: {}, - UPTIME_JOB_TIMEOUT_MS: 60_000, - UPTIME_QUEUE_NAME: "uptime-checks", - INSIGHTS_DISPATCH_JOB_NAME: "insights-dispatch", - INSIGHTS_GENERATE_WEBSITE_JOB_NAME: "insights-generate-website", - INSIGHTS_JOB_OPTIONS: {}, - INSIGHTS_JOB_TIMEOUT_MS: 120_000, - INSIGHTS_QUEUE_ENV_PREFIX: "INSIGHTS", - INSIGHTS_QUEUE_NAME: "insights-generation", - activeStreamKey: (id: string) => `active:${id}`, - appendStreamChunk: vi.fn(async () => undefined), - cacheNamespaces: { - agentTelemetryWebsiteExists: "agent-telemetry:website-exists", - apiKeyByHash: "api-key-by-hash", - apiKeyOwnerId: "api_key_owner_id", - billingOwner: "rpc:billing_owner", - flag: "flag", - flagsClient: "flags-client", - flagsDefinitions: "flags-definitions", - flagsUser: "flags-user", - mcpInsights: "mcp:insights", - memberRole: "rpc:member_role", - organizationOwner: "rpc:org_owner", - organizationRole: "rpc:org_role", - slackChannelBinding: "slack-channel-binding", - slackIntegrationByTeam: "slack-integration-by-team", - statusPage: "status-page", - userPreferences: "user-prefs", - websiteById: "website_by_id", - websiteCache: "website-cache", - websiteDomain: "website-domain", - websiteDomainsBatch: "website-domains-batch", - websiteWithOwner: "website_with_owner_v2", - }, - cacheTags: { - billingOwner: (ownerId: string) => `billing-owner:${ownerId}`, - flagClient: (clientId: string) => `flag-client:${clientId}`, - flagKey: (clientId: string, flagKey: string) => - `flag-key:${clientId}:${flagKey}`, - flagUser: (clientId: string, userId: string) => - `flag-user:${clientId}:${userId}`, - organization: (organizationId: string) => `organization:${organizationId}`, - website: (websiteId: string) => `website:${websiteId}`, - }, - cacheable: passthroughCacheable, - clearActiveStream: vi.fn(async () => undefined), - closeInsightsQueue: vi.fn(async () => undefined), - closeUptimeQueue: vi.fn(async () => undefined), - createDrizzleCache: () => ({}), - getActiveStream: vi.fn(async () => null), - getAgentContextSnapshotKey: getSnapshotKey, - getBullMQConnectionOptions: vi.fn(() => ({})), - getBullMQWorkerConnectionOptions: vi.fn(() => ({})), - getCachedLink: vi.fn(async () => null), - getCacheableKey: vi.fn( - (prefix: string, args: unknown[]) => `${prefix}:${JSON.stringify(args)}` - ), - getLinkCacheKey: vi.fn((slug: string) => `link:${slug}`), - getRateLimitHeaders: vi.fn(() => ({})), - getInsightsQueue: vi.fn(() => ({})), - getRedisCache: () => mockRedisClient, - getUptimeQueue: vi.fn(() => ({})), - invalidateAgentContextSnapshot: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForOwner: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForWebsite: vi.fn(async () => 0), - invalidateCacheableKey: vi.fn(async () => 0), - invalidateCacheablePattern: vi.fn(async () => 0), - invalidateCacheablePrefix: vi.fn(async () => 0), - invalidateCacheableTag: vi.fn(async () => 0), - invalidateCacheableTags: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateCacheableWithArgs: vi.fn(async () => 0), - invalidateBillingOwnerCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateFlagReadCaches: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateInsightsCachesForOrganization: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateLinkCache: vi.fn(async () => undefined), - invalidateLinkCaches: vi.fn(async () => undefined), - invalidateOrganizationMembershipCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateSlackChannelBindingCache: vi.fn(async () => undefined), - invalidateSlackIntegrationCache: vi.fn(async () => undefined), - invalidateStatusPageCache: vi.fn(async () => 0), - invalidateUserPreferencesCache: vi.fn(async () => undefined), - invalidateWebsiteReadCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - enqueueInsightsResume: vi.fn(async () => "queued"), - insightsResumeJobId: (replyId: string) => `insights-reply-${replyId}`, - insightsWebsiteJobId: (runId: string, websiteId: string) => - `insights-website-${runId}-${websiteId}`, - isClickRecorded: vi.fn(async () => false), - markStreamDone: vi.fn(async () => undefined), - ratelimit: vi.fn(async () => ({ success: true })), - readStreamHistory: vi.fn(async () => []), - redis: mockRedisClient, - setActiveStream: vi.fn(async () => undefined), - setCachedLink: vi.fn(async () => undefined), - setCachedLinkNotFound: vi.fn(async () => undefined), - shouldRecordClick: vi.fn(async () => true), - shutdownRedis: vi.fn(async () => undefined), - streamBufferKey: (id: string) => `stream:${id}`, - uptimeImmediateJobId: (id: string) => `uptime:immediate:${id}`, - uptimeSchedulerId: (id: string) => `uptime:scheduler:${id}`, -})); +vi.mock("@databuddy/redis", async (importOriginal) => { + const actualRedis = importOriginal + ? await importOriginal() + : bunRedisModule; + if (!actualRedis) { + throw new Error("Unable to load the Redis test module"); + } + return createInertRedisModule(actualRedis, { + getRedisCache: () => mockRedisClient, + redis: mockRedisClient, + }); +}); vi.mock("../../lib/supermemory", () => ({ formatMemoryForPrompt: vi.fn(() => ""), diff --git a/packages/ai/src/ai/mcp/conversation-store.test.ts b/packages/ai/src/ai/mcp/conversation-store.test.ts index db6f8614f..18ae1a2c4 100644 --- a/packages/ai/src/ai/mcp/conversation-store.test.ts +++ b/packages/ai/src/ai/mcp/conversation-store.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createInertRedisModule } from "../../test-utils/redis-mock"; const redisStore = new Map(); let failGet = false; @@ -21,130 +22,26 @@ const mockRedisClient = { }), }; -vi.mock("@databuddy/redis", () => ({ - AGENT_CONTEXT_SNAPSHOT_PREFIX: "agent:context-snapshot", - UPTIME_CHECK_JOB_NAME: "uptime-check", - UPTIME_JOB_OPTIONS: {}, - UPTIME_JOB_TIMEOUT_MS: 60_000, - UPTIME_QUEUE_NAME: "uptime-checks", - INSIGHTS_DISPATCH_JOB_NAME: "insights-dispatch", - INSIGHTS_GENERATE_WEBSITE_JOB_NAME: "insights-generate-website", - INSIGHTS_JOB_OPTIONS: {}, - INSIGHTS_JOB_TIMEOUT_MS: 120_000, - INSIGHTS_QUEUE_ENV_PREFIX: "INSIGHTS", - INSIGHTS_QUEUE_NAME: "insights-generation", - activeStreamKey: (id: string) => `active:${id}`, - appendStreamChunk: vi.fn(async () => undefined), - cacheNamespaces: { - agentTelemetryWebsiteExists: "agent-telemetry:website-exists", - apiKeyByHash: "api-key-by-hash", - apiKeyOwnerId: "api_key_owner_id", - billingOwner: "rpc:billing_owner", - flag: "flag", - flagsClient: "flags-client", - flagsDefinitions: "flags-definitions", - flagsUser: "flags-user", - mcpInsights: "mcp:insights", - memberRole: "rpc:member_role", - organizationOwner: "rpc:org_owner", - organizationRole: "rpc:org_role", - slackChannelBinding: "slack-channel-binding", - slackIntegrationByTeam: "slack-integration-by-team", - statusPage: "status-page", - userPreferences: "user-prefs", - websiteById: "website_by_id", - websiteCache: "website-cache", - websiteDomain: "website-domain", - websiteDomainsBatch: "website-domains-batch", - websiteWithOwner: "website_with_owner_v2", - }, - cacheTags: { - billingOwner: (ownerId: string) => `billing-owner:${ownerId}`, - flagClient: (clientId: string) => `flag-client:${clientId}`, - flagKey: (clientId: string, flagKey: string) => - `flag-key:${clientId}:${flagKey}`, - flagUser: (clientId: string, userId: string) => - `flag-user:${clientId}:${userId}`, - organization: (organizationId: string) => `organization:${organizationId}`, - website: (websiteId: string) => `website:${websiteId}`, - }, - cacheable: unknown>(fn: T) => fn, - clearActiveStream: vi.fn(async () => undefined), - closeInsightsQueue: vi.fn(async () => undefined), - closeUptimeQueue: vi.fn(async () => undefined), - createDrizzleCache: () => ({}), - getActiveStream: vi.fn(async () => null), - getAgentContextSnapshotKey: ( - userId: string, - websiteId: string, - organizationId?: string | null - ) => `agent:context-snapshot:${organizationId ?? userId}:${websiteId}`, - getBullMQConnectionOptions: vi.fn(() => ({})), - getBullMQWorkerConnectionOptions: vi.fn(() => ({})), - getCachedLink: vi.fn(async () => null), - getCacheableKey: vi.fn( - (prefix: string, args: unknown[]) => `${prefix}:${JSON.stringify(args)}` - ), - getLinkCacheKey: vi.fn((slug: string) => `link:${slug}`), - getRateLimitHeaders: vi.fn(() => ({})), - getRedisCache: () => { - if (redisUnavailable) { - throw new Error("redis unavailable"); - } - return mockRedisClient; - }, - getInsightsQueue: vi.fn(() => ({})), - getUptimeQueue: vi.fn(() => ({})), - invalidateAgentContextSnapshot: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForOwner: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForWebsite: vi.fn(async () => 0), - invalidateCacheableKey: vi.fn(async () => 0), - invalidateCacheablePattern: vi.fn(async () => 0), - invalidateCacheablePrefix: vi.fn(async () => 0), - invalidateCacheableTag: vi.fn(async () => 0), - invalidateCacheableTags: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateCacheableWithArgs: vi.fn(async () => 0), - invalidateBillingOwnerCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateFlagReadCaches: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateInsightsCachesForOrganization: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateLinkCache: vi.fn(async () => undefined), - invalidateLinkCaches: vi.fn(async () => undefined), - invalidateOrganizationMembershipCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateSlackChannelBindingCache: vi.fn(async () => undefined), - invalidateSlackIntegrationCache: vi.fn(async () => undefined), - invalidateStatusPageCache: vi.fn(async () => 0), - invalidateUserPreferencesCache: vi.fn(async () => undefined), - invalidateWebsiteReadCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - enqueueInsightsResume: vi.fn(async () => "queued"), - insightsResumeJobId: (replyId: string) => `insights-reply-${replyId}`, - insightsWebsiteJobId: (runId: string, websiteId: string) => - `insights-website-${runId}-${websiteId}`, - isClickRecorded: vi.fn(async () => false), - markStreamDone: vi.fn(async () => undefined), - ratelimit: vi.fn(async () => ({ success: true })), - readStreamHistory: vi.fn(async () => []), - redis: mockRedisClient, - setActiveStream: vi.fn(async () => undefined), - setCachedLink: vi.fn(async () => undefined), - setCachedLinkNotFound: vi.fn(async () => undefined), - shouldRecordClick: vi.fn(async () => true), - shutdownRedis: vi.fn(async () => undefined), - streamBufferKey: (id: string) => `stream:${id}`, - uptimeImmediateJobId: (id: string) => `uptime:immediate:${id}`, - uptimeSchedulerId: (id: string) => `uptime:scheduler:${id}`, -})); +const bunRedisModule = + typeof Bun === "undefined" ? undefined : await import("@databuddy/redis"); + +vi.mock("@databuddy/redis", async (importOriginal) => { + const actualRedis = importOriginal + ? await importOriginal() + : bunRedisModule; + if (!actualRedis) { + throw new Error("Unable to load the Redis test module"); + } + return createInertRedisModule(actualRedis, { + getRedisCache: () => { + if (redisUnavailable) { + throw new Error("redis unavailable"); + } + return mockRedisClient; + }, + redis: mockRedisClient, + }); +}); const { appendToConversation, getConversationHistory } = await import( "./conversation-store" diff --git a/packages/ai/src/query/builders/uptime.ts b/packages/ai/src/query/builders/uptime.ts index 6cc581515..eed0c9f3a 100644 --- a/packages/ai/src/query/builders/uptime.ts +++ b/packages/ai/src/query/builders/uptime.ts @@ -1,3 +1,4 @@ +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; import type { SimpleQueryConfig } from "../types"; /** @@ -18,6 +19,23 @@ import type { SimpleQueryConfig } from "../types"; */ const UPTIME_TABLE = "uptime.uptime_monitor"; +const UPTIME_REPLAY_IDENTITY = TABLE_COLUMNS[UPTIME_TABLE].join(", "); + +/** + * A replay keeps the original check payload and timestamp. The deployed + * ClickHouse table deliberately remains migration-free, so reduce only exact + * duplicate payloads with fields already present in that schema before + * aggregates or window functions consume the source. + */ +function uptimeEventSource(scope: string): string { + return `( + SELECT * + FROM ${UPTIME_TABLE} + WHERE + ${scope} + LIMIT 1 BY ${UPTIME_REPLAY_IDENTITY} + )`; +} export const UptimeBuilders: Record = { uptime_overview: { @@ -44,11 +62,11 @@ export const UptimeBuilders: Record = { avg(ttfb_ms) as avg_ttfb, any(ssl_expiry) as ssl_expiry, min(ssl_valid) as ssl_valid - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} `, params: { websiteId, startDate, endDate }, }; @@ -125,11 +143,11 @@ export const UptimeBuilders: Record = { ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ) as next_ts - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= parseDateTimeBestEffort({startDate:String}, {timezone:String}) AND timestamp <= parseDateTimeBestEffort(concat({endDate:String}, ' 23:59:59'), {timezone:String}) + `)} ) GROUP BY date ) @@ -157,11 +175,11 @@ export const UptimeBuilders: Record = { http_code, COUNT(*) as count, round((COUNT(*) / sum(COUNT(*)) OVER ()) * 100, 2) as percentage - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} GROUP BY status, http_code ORDER BY count DESC `, @@ -195,11 +213,11 @@ export const UptimeBuilders: Record = { probe_ip, ssl_valid, error - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} ORDER BY timestamp DESC LIMIT {limit:UInt32} OFFSET {offset:UInt32} @@ -242,12 +260,12 @@ export const UptimeBuilders: Record = { min(total_ms) as min_response_time, max(total_ms) as max_response_time, avg(ttfb_ms) as avg_ttfb - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) - AND status = 1 + `)} + WHERE status = 1 GROUP BY date ORDER BY date ASC `, @@ -277,11 +295,11 @@ export const UptimeBuilders: Record = { argMax(ssl_expiry, timestamp) as latest_ssl_expiry, argMax(ssl_valid, timestamp) as latest_ssl_valid, countIf(ssl_valid = 0) as invalid_ssl_checks - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} GROUP BY site_id ) `, @@ -308,11 +326,11 @@ export const UptimeBuilders: Record = { if((countIf(status = 1) + countIf(status = 0)) = 0, 0, round((countIf(status = 1) / (countIf(status = 1) + countIf(status = 0))) * 100, 2)) as uptime_percentage, avg(total_ms) as avg_response_time, quantileTDigest(0.95)(total_ms) as p95_response_time - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} GROUP BY probe_region ORDER BY uptime_percentage DESC `, diff --git a/packages/ai/src/query/simple-builder.test.ts b/packages/ai/src/query/simple-builder.test.ts index 3c94e15f0..b07f6f402 100644 --- a/packages/ai/src/query/simple-builder.test.ts +++ b/packages/ai/src/query/simple-builder.test.ts @@ -1,3 +1,4 @@ +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; import { describe, expect, it } from "vitest"; import { QueryBuilders } from "./builders"; import { @@ -1029,6 +1030,46 @@ describe("SimpleQueryBuilder.compile", () => { expect(sql).not.toContain("event_name = 'pageview'"); }); + it.each([ + "uptime_overview", + "uptime_time_series", + "uptime_status_breakdown", + "uptime_recent_checks", + "uptime_response_time_trends", + "uptime_ssl_status", + "uptime_by_region", + ])( + "deduplicates replayed uptime deliveries in %s using existing check fields", + (type) => { + const config = QueryBuilders[type]; + if (!config) { + throw new Error(`${type} builder is missing`); + } + + const { sql } = compileBuilder(type, config); + + expect(sql).toContain( + `LIMIT 1 BY ${TABLE_COLUMNS["uptime.uptime_monitor"].join(", ")}` + ); + expect(sql).not.toContain("ORDER BY timestamp DESC LIMIT 1 BY"); + expect(sql).not.toContain("event_id"); + expect(sql.match(/FROM uptime\.uptime_monitor/g)).toHaveLength(1); + } + ); + + it("deduplicates uptime events before the downtime window function", () => { + const config = QueryBuilders.uptime_time_series; + if (!config) { + throw new Error("uptime_time_series builder is missing"); + } + + const { sql } = compileBuilder("uptime_time_series", config); + + expect(sql).toMatch( + /leadInFrame\(timestamp, 1, now\(\)\)[\s\S]*?FROM \([\s\S]*?LIMIT 1 BY site_id, url, timestamp, status, http_code, ttfb_ms, total_ms/ + ); + }); + it("counts only page-view sessions in session metrics", () => { const config = QueryBuilders.session_metrics; if (!config) { diff --git a/packages/ai/src/test-utils/redis-mock.ts b/packages/ai/src/test-utils/redis-mock.ts new file mode 100644 index 000000000..48fbfdf8d --- /dev/null +++ b/packages/ai/src/test-utils/redis-mock.ts @@ -0,0 +1,56 @@ +type RedisModule = typeof import("@databuddy/redis"); + +const noop = () => undefined; +const noopAsync = async () => undefined; + +const pureFunctionExports = [ + "activeStreamKey", + "getAgentContextSnapshotKey", + "getCacheableKey", + "getCacheableTagIndexKey", + "getLinkCacheKey", + "getRateLimitHeaders", + "insightsResumeJobId", + "insightsWebsiteJobId", + "streamBufferKey", + "uptimeDeliveryJobId", + "uptimeImmediateJobId", + "uptimeSchedulerId", +] as const satisfies readonly (keyof RedisModule)[]; + +/** + * Preserves the full Redis barrel shape while making every effectful function + * inert by default. New function exports therefore cannot open Redis or + * BullMQ connections from a test that only needs a small mocked surface. + */ +export function createInertRedisModule( + actualRedis: RedisModule, + options: { getRedisCache: () => unknown; redis: unknown } +): RedisModule { + const module: Record = Object.fromEntries( + Object.entries(actualRedis).map(([name, value]) => [ + name, + typeof value === "function" ? noop : value, + ]) + ); + + for (const name of pureFunctionExports) { + module[name] = actualRedis[name]; + } + + return { + ...module, + cacheable: unknown>(fn: T) => fn, + createDrizzleCache: () => ({ + cleanupEmptySets: noopAsync, + invalidateByKey: noopAsync, + invalidateByTables: noopAsync, + invalidateByTags: noopAsync, + withCache: async ({ queryFn }: { queryFn: () => Promise }) => + queryFn(), + }), + default: options.redis, + getRedisCache: options.getRedisCache, + redis: options.redis, + } as unknown as RedisModule; +} diff --git a/packages/redis/uptime-queue.test.ts b/packages/redis/uptime-queue.test.ts index ef95845a3..c309cdaa8 100644 --- a/packages/redis/uptime-queue.test.ts +++ b/packages/redis/uptime-queue.test.ts @@ -27,10 +27,15 @@ process.env.BULLMQ_REDIS_URL = "redis://queue-user:queue-pass@queue.test:6381/4" const { closeUptimeQueue, + getUptimeDeliveryQueue, getUptimeQueue, UPTIME_CHECK_JOB_NAME, + UPTIME_DELIVERY_JOB_NAME, + UPTIME_DELIVERY_JOB_OPTIONS, + UPTIME_DELIVERY_QUEUE_NAME, UPTIME_JOB_OPTIONS, UPTIME_QUEUE_NAME, + uptimeDeliveryJobId, uptimeImmediateJobId, uptimeSchedulerId, } = await import("./uptime-queue"); @@ -71,15 +76,39 @@ describe("uptime queue", () => { expect(constructorCalls).toHaveLength(1); }); + it("constructs a separate durable delivery queue", () => { + const queue = getUptimeDeliveryQueue(); + + expect(queue).toBeInstanceOf(MockQueue); + expect(constructorCalls).toHaveLength(1); + expect(constructorCalls[0]).toEqual({ + name: UPTIME_DELIVERY_QUEUE_NAME, + options: { + connection: { + host: "queue.test", + port: 6381, + username: "queue-user", + password: "queue-pass", + db: 4, + maxRetriesPerRequest: 1, + }, + defaultJobOptions: UPTIME_DELIVERY_JOB_OPTIONS, + }, + }); + }); + it("closes and resets the singleton", async () => { const first = getUptimeQueue(); + const firstDelivery = getUptimeDeliveryQueue(); await closeUptimeQueue(); const second = getUptimeQueue(); + const secondDelivery = getUptimeDeliveryQueue(); - expect(closeCalls).toBe(1); + expect(closeCalls).toBe(2); expect(second).not.toBe(first); - expect(constructorCalls).toHaveLength(2); + expect(secondDelivery).not.toBe(firstDelivery); + expect(constructorCalls).toHaveLength(4); }); it("uses stable queue constants and namespaced ids", () => { @@ -88,9 +117,25 @@ describe("uptime queue", () => { expect(UPTIME_CHECK_JOB_NAME).toBe("uptime-check"); expect(UPTIME_QUEUE_NAME).toBe("uptime-checks"); + expect(UPTIME_DELIVERY_JOB_NAME).toBe("uptime-event-delivery"); + expect(UPTIME_DELIVERY_QUEUE_NAME).toBe("uptime-event-delivery"); expect(uptimeSchedulerId("schedule-1")).toBe("uptime-schedule-1"); + expect(uptimeDeliveryJobId("event-1")).toBe("uptime-delivery-event-1"); expect(first.startsWith("uptime-manual-schedule-1-")).toBe(true); expect(second.startsWith("uptime-manual-schedule-1-")).toBe(true); expect(first).not.toBe(second); }); + + it("keeps source and delivery payloads for retry instead of expiring failures", () => { + expect(UPTIME_JOB_OPTIONS).toMatchObject({ + attempts: 1_000_000, + backoff: { delay: 30_000, type: "fixed" }, + removeOnFail: false, + }); + expect(UPTIME_DELIVERY_JOB_OPTIONS).toMatchObject({ + attempts: 1_000_000, + backoff: { delay: 30_000, type: "fixed" }, + removeOnFail: false, + }); + }); }); diff --git a/packages/redis/uptime-queue.ts b/packages/redis/uptime-queue.ts index e398cd94a..ec461e58f 100644 --- a/packages/redis/uptime-queue.ts +++ b/packages/redis/uptime-queue.ts @@ -3,31 +3,60 @@ import { getBullMQConnectionOptions } from "./bullmq"; export const UPTIME_QUEUE_NAME = "uptime-checks"; export const UPTIME_CHECK_JOB_NAME = "uptime-check"; +export const UPTIME_DELIVERY_QUEUE_NAME = "uptime-event-delivery"; +export const UPTIME_DELIVERY_JOB_NAME = UPTIME_DELIVERY_QUEUE_NAME; export const UPTIME_JOB_TIMEOUT_MS = 30_000; - -export const UPTIME_JOB_OPTIONS = { - attempts: 3, +const UPTIME_RETRY_ATTEMPTS = 1_000_000; +const UPTIME_RETRY_DELAY_MS = 30_000; +const UPTIME_RETRY_OPTIONS = { + attempts: UPTIME_RETRY_ATTEMPTS, backoff: { - type: "exponential", - delay: 2000, + type: "fixed", + delay: UPTIME_RETRY_DELAY_MS, }, + removeOnFail: false, +}; + +/** + * Check jobs remain recoverable after an infrastructure outage. A target that + * is down produces an UptimeData result and completes normally, so these + * retries only cover transient processing or durable-handoff failures. + */ +export const UPTIME_JOB_OPTIONS = { + ...UPTIME_RETRY_OPTIONS, removeOnComplete: { age: 24 * 3600, count: 1000, }, - removeOnFail: { + // A failed job can contain the only durable copy of a completed probe. + // Retain it for inspection/re-drive instead of silently expiring it. +}; + +export const UPTIME_DELIVERY_JOB_OPTIONS = { + ...UPTIME_RETRY_OPTIONS, + // Keep completed IDs long enough to make a retry after an ambiguous queue + // add idempotent. Replaying after that is still safe because event_id stays + // stable through the analytics pipeline. + removeOnComplete: { age: 7 * 24 * 3600, - count: 5000, + count: 10_000, }, }; export interface UptimeCheckJobData { + // The uptime worker validates this JSON payload before replaying it. + delivery?: { event: unknown }; scheduleId: string; trigger: "manual" | "scheduled"; } +export interface UptimeDeliveryJobData { + event: unknown; +} + let uptimeQueue: Queue | null = null; +let uptimeDeliveryQueue: Queue | null = null; export function getUptimeQueue(): Queue { uptimeQueue ??= new Queue(UPTIME_QUEUE_NAME, { @@ -38,13 +67,23 @@ export function getUptimeQueue(): Queue { return uptimeQueue; } +export function getUptimeDeliveryQueue(): Queue { + uptimeDeliveryQueue ??= new Queue( + UPTIME_DELIVERY_QUEUE_NAME, + { + connection: getBullMQConnectionOptions(), + defaultJobOptions: UPTIME_DELIVERY_JOB_OPTIONS, + } + ); + + return uptimeDeliveryQueue; +} + export async function closeUptimeQueue(): Promise { - if (!uptimeQueue) { - return; - } - const queue = uptimeQueue; + const queues = [uptimeQueue, uptimeDeliveryQueue]; uptimeQueue = null; - await queue.close(); + uptimeDeliveryQueue = null; + await Promise.all(queues.map((queue) => queue?.close())); } export function uptimeSchedulerId(scheduleId: string): string { @@ -54,3 +93,7 @@ export function uptimeSchedulerId(scheduleId: string): string { export function uptimeImmediateJobId(scheduleId: string): string { return `uptime-manual-${scheduleId}-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; } + +export function uptimeDeliveryJobId(eventId: string): string { + return `uptime-delivery-${eventId}`; +} diff --git a/packages/rpc/src/routers/status-page.ts b/packages/rpc/src/routers/status-page.ts index 5dbc16272..79963be64 100644 --- a/packages/rpc/src/routers/status-page.ts +++ b/packages/rpc/src/routers/status-page.ts @@ -1,5 +1,6 @@ import { and, db, desc, eq, inArray, withTransaction } from "@databuddy/db"; import { chQuery } from "@databuddy/db/clickhouse"; +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; import { incidentAffectedMonitors, incidentUpdates, @@ -33,6 +34,22 @@ import { } from "./status-page-health"; const UPTIME_TABLE = "uptime.uptime_monitor"; +const UPTIME_REPLAY_IDENTITY = TABLE_COLUMNS[UPTIME_TABLE].join(", "); + +/** + * Replaying a persisted delivery preserves its original check payload and timestamp. + * Deduplicate exact physical payloads using only existing ClickHouse columns + * before status-page aggregates and windows consume the source. + */ +function uptimeEventSource(scope: string): string { + return `( + SELECT * + FROM ${UPTIME_TABLE} + WHERE + ${scope} + LIMIT 1 BY ${UPTIME_REPLAY_IDENTITY} + )`; +} const DAILY_UPTIME_SQL = `SELECT site_id, @@ -66,11 +83,11 @@ const DAILY_UPTIME_SQL = `SELECT ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ) as next_ts - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id IN ({siteIds:Array(String)}) AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} ) GROUP BY site_id, date ) @@ -81,9 +98,10 @@ const LATEST_CHECK_SQL = `SELECT max(timestamp) as last_timestamp, argMax(status, timestamp) as last_status, argMax(http_code, timestamp) as last_http_code - FROM ${UPTIME_TABLE} - WHERE site_id IN ({siteIds:Array(String)}) + FROM ${uptimeEventSource(` + site_id IN ({siteIds:Array(String)}) AND timestamp >= now() - INTERVAL 7 DAY + `)} GROUP BY site_id`; const dailyUptimeSchema = z.object({ @@ -230,12 +248,6 @@ function groupDailyRows(rows: DailyRow[]): Map { return grouped; } -function indexLatestChecks( - rows: LatestCheckRow[] -): Map { - return new Map(rows.map((row) => [row.site_id, row])); -} - function applyIncidentImpacts( monitors: z.infer[], activeIncidents: z.infer[], @@ -404,7 +416,9 @@ async function _fetchStatusPageData( const websiteMap = new Map(websiteRows.map((w) => [w.id, w] as const)); const dailyBySite = groupDailyRows(allDailyData); - const latestBySite = indexLatestChecks(allRecentChecks); + const latestBySite = new Map( + allRecentChecks.map((row) => [row.site_id, row]) + ); const monitors = schedules.map((schedule) => { const siteId = schedule.websiteId ?? schedule.id; diff --git a/packages/rpc/src/services/uptime-scheduler.test.ts b/packages/rpc/src/services/uptime-scheduler.test.ts index f22416ea3..2f4e7a2a4 100644 --- a/packages/rpc/src/services/uptime-scheduler.test.ts +++ b/packages/rpc/src/services/uptime-scheduler.test.ts @@ -2,19 +2,16 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; const UPTIME_CHECK_JOB_NAME = "uptime-check"; const UPTIME_JOB_OPTIONS = { - attempts: 3, + attempts: 1_000_000, backoff: { - type: "exponential", - delay: 2000, + type: "fixed", + delay: 30_000, }, removeOnComplete: { age: 24 * 3600, count: 1000, }, - removeOnFail: { - age: 7 * 24 * 3600, - count: 5000, - }, + removeOnFail: false, }; const calls: { diff --git a/packages/tracker/src/core/client.ts b/packages/tracker/src/core/client.ts index 9e47db6ea..e72a0f60f 100644 --- a/packages/tracker/src/core/client.ts +++ b/packages/tracker/src/core/client.ts @@ -116,30 +116,6 @@ export class HttpClient { return this.abortedResult(retryCount); } - if ( - retryCount === 0 && - typeof navigator !== "undefined" && - navigator.sendBeacon && - options.keepalive - ) { - try { - const blob = new Blob([JSON.stringify(data ?? {})], { - type: "application/json", - }); - if (navigator.sendBeacon(url, blob)) { - return { - ok: true, - data: null, - status: null, - attempts: 1, - transport: "beacon", - }; - } - } catch { - // Fall through to fetch so callers receive a verifiable outcome. - } - } - try { const response = await fetch(url, { method: "POST", diff --git a/packages/tracker/src/core/tracker.ts b/packages/tracker/src/core/tracker.ts index d4518fe07..445376cd9 100644 --- a/packages/tracker/src/core/tracker.ts +++ b/packages/tracker/src/core/tracker.ts @@ -649,8 +649,7 @@ export class BaseTracker { deliveryGeneration === this.deliveryGeneration && !result.ok && result.retryable && - !this.shouldBlockQueuedDelivery() && - deliveryGeneration === this.deliveryGeneration + !this.shouldBlockQueuedDelivery() ) { queue.unshift(...items); meta.retryAttempts += 1; diff --git a/packages/tracker/src/plugins/pixel.ts b/packages/tracker/src/plugins/pixel.ts index b7764b8f7..d66df097a 100644 --- a/packages/tracker/src/plugins/pixel.ts +++ b/packages/tracker/src/plugins/pixel.ts @@ -11,6 +11,13 @@ const PIXEL_TYPE_BY_ENDPOINT: Record = { "/vitals": "web_vitals", "/errors": "error", }; +const MAX_PIXEL_RETRY_DELAY_MS = 30_000; +const PIXEL_LOAD_TIMEOUT_MS = 10_000; + +interface PixelDeliveryResult { + attempts: number; + success: boolean; +} function safeStringify(value: unknown): string { const seen = new WeakSet(); @@ -49,11 +56,45 @@ function flattenIntoParams( export function initPixelTracking(tracker: BaseTracker) { tracker.options.enableBatching = false; + let deliveryGeneration = 0; + const activeImageCancels = new Set<() => void>(); + const cancelPendingRequests = tracker.api.cancelPendingRequests.bind( + tracker.api + ); + tracker.api.cancelPendingRequests = () => { + deliveryGeneration += 1; + for (const cancel of activeImageCancels) { + cancel(); + } + cancelPendingRequests(); + }; + + const configuredMaxRetries = tracker.options.maxRetries; + const maxRetries = + tracker.options.enableRetries === false + ? 0 + : typeof configuredMaxRetries === "number" && + Number.isFinite(configuredMaxRetries) && + configuredMaxRetries >= 0 + ? Math.floor(configuredMaxRetries) + : 3; + const initialRetryDelay = Math.max( + 0, + tracker.options.initialRetryDelay ?? 500 + ); + + const waitForRetry = (retry: number): Promise => + new Promise((resolve) => { + setTimeout( + resolve, + Math.min(initialRetryDelay * 2 ** retry, MAX_PIXEL_RETRY_DELAY_MS) + ); + }); const sendOnePixel = ( eventType: string, data: Record - ): Promise<{ success: boolean }> => { + ): Promise => { const params = new URLSearchParams(); flattenIntoParams(params, data); @@ -76,21 +117,66 @@ export function initPixelTracking(tracker: BaseTracker) { url.searchParams.append(key, value); }); - return new Promise((resolve) => { - const img = new Image(); - img.onload = () => resolve({ success: true }); - img.onerror = () => resolve({ success: false }); - img.src = url.toString(); - }); + const generation = deliveryGeneration; + const load = (): Promise => + new Promise((resolve) => { + const img = new Image(); + let settled = false; + let timeout: ReturnType | undefined; + const finish = (success: boolean) => { + if (settled) { + return; + } + settled = true; + if (timeout !== undefined) { + clearTimeout(timeout); + } + activeImageCancels.delete(cancel); + img.onload = null; + img.onerror = null; + resolve(success); + }; + const cancel = () => { + if (settled) { + return; + } + img.onload = null; + img.onerror = null; + img.src = ""; + finish(false); + }; + + activeImageCancels.add(cancel); + timeout = setTimeout(cancel, PIXEL_LOAD_TIMEOUT_MS); + img.onload = () => finish(true); + img.onerror = () => finish(false); + img.src = url.toString(); + }); + + return (async () => { + for (let retry = 0; retry <= maxRetries; retry += 1) { + if (generation !== deliveryGeneration) { + return { attempts: retry, success: false }; + } + if (await load()) { + return { attempts: retry + 1, success: true }; + } + if (retry < maxRetries && generation === deliveryGeneration) { + await waitForRetry(retry); + } + } + + return { attempts: maxRetries + 1, success: false }; + })(); }; const sendToPixel = async ( endpoint: string, data: unknown - ): Promise<{ success: boolean }> => { + ): Promise => { const eventType = PIXEL_TYPE_BY_ENDPOINT[endpoint]; if (!eventType) { - return { success: false }; + return { attempts: 1, success: false }; } if (Array.isArray(data)) { @@ -98,14 +184,17 @@ export function initPixelTracking(tracker: BaseTracker) { data.map((event) => event && typeof event === "object" ? sendOnePixel(eventType, event as Record) - : Promise.resolve({ success: false }) + : Promise.resolve({ attempts: 1, success: false }) ) ); - return { success: results.every((r) => r.success) }; + return { + attempts: Math.max(1, ...results.map((result) => result.attempts)), + success: results.every((result) => result.success), + }; } if (typeof data !== "object" || data === null) { - return { success: false }; + return { attempts: 1, success: false }; } return sendOnePixel(eventType, data as Record); }; @@ -120,7 +209,7 @@ export function initPixelTracking(tracker: BaseTracker) { ok: true, data: null, status: null, - attempts: 1, + attempts: result.attempts, transport: "beacon", }; } @@ -130,15 +219,13 @@ export function initPixelTracking(tracker: BaseTracker) { message: "Tracking pixel failed to load", status: null, retryable: true, - attempts: 1, + attempts: result.attempts, transport: "fetch", }; }; - tracker.sendBeacon = (data: unknown, endpoint = "/") => { - sendToPixel(endpoint, data); - return true; - }; - - tracker.sendBatchBeacon = () => false; + // Image loads cannot synchronously prove remote acceptance. Returning false + // keeps BaseTracker's queue for its fetch-style pixel fallback instead of + // dropping it immediately on an unverified load. + tracker.sendBeacon = () => false; } diff --git a/packages/tracker/tests/unit/http-client.test.ts b/packages/tracker/tests/unit/http-client.test.ts index c32045c0a..603fd3f3a 100644 --- a/packages/tracker/tests/unit/http-client.test.ts +++ b/packages/tracker/tests/unit/http-client.test.ts @@ -3,6 +3,7 @@ import { HttpClient, type HttpResult } from "../../src/core/client"; import { BaseTracker } from "../../src/core/tracker"; const originalFetch = globalThis.fetch; +const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); class DeliveryTestTracker extends BaseTracker { private deliveryBlocked = false; @@ -27,6 +28,11 @@ afterEach(() => { jest.useRealTimers(); } globalThis.fetch = originalFetch; + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + return; + } + Reflect.deleteProperty(globalThis, "navigator"); }); async function flushMicrotasks(): Promise { @@ -201,6 +207,36 @@ describe("HttpClient", () => { }); describe("BaseTracker delivery outcomes", () => { + test("requeues a retryable batch despite an accepted Beacon", async () => { + jest.useFakeTimers(); + const sendBeacon = mock(() => true); + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { sendBeacon }, + }); + const fetchMock = mock(async () => + Response.json({ error: "Temporarily unavailable" }, { status: 503 }) + ); + globalThis.fetch = fetchMock as typeof fetch; + const tracker = new DeliveryTestTracker({ + clientId: "site_example", + enableRetries: false, + }); + const event = { eventId: "event_1", timestamp: 1 }; + await tracker.addToBatch(event); + + const result = await tracker.flushBatch(); + + expect(sendBeacon).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + ok: false, + status: "failed", + retryable: true, + }); + expect(tracker.batchQueue).toEqual([event]); + }); + test("keeps a retryable failed batch queued", async () => { jest.useFakeTimers(); const tracker = new DeliveryTestTracker({ clientId: "site_example" }); diff --git a/packages/tracker/tests/unit/pixel.test.ts b/packages/tracker/tests/unit/pixel.test.ts new file mode 100644 index 000000000..f784f61d8 --- /dev/null +++ b/packages/tracker/tests/unit/pixel.test.ts @@ -0,0 +1,143 @@ +import { afterEach, describe, expect, jest, mock, test } from "bun:test"; +import type { BaseTracker } from "../../src/core/tracker"; +import type { TrackerOptions } from "../../src/core/types"; +import { initPixelTracking } from "../../src/plugins/pixel"; + +const originalImage = Object.getOwnPropertyDescriptor(globalThis, "Image"); + +afterEach(() => { + if (jest.isFakeTimers()) { + jest.clearAllTimers(); + jest.useRealTimers(); + } + if (originalImage) { + Object.defineProperty(globalThis, "Image", originalImage); + return; + } + Reflect.deleteProperty(globalThis, "Image"); +}); + +function createTracker( + overrides: Partial = {} +): BaseTracker { + return { + api: { + cancelPendingRequests: mock(() => {}), + }, + options: { + clientId: "site_example", + enableRetries: true, + initialRetryDelay: 0, + maxRetries: 1, + sdk: "web", + sdkVersion: "2.0.0", + ...overrides, + }, + } as unknown as BaseTracker; +} + +function installImageOutcomes( + outcomes: Array<"error" | "load" | "pending"> +): string[] { + const requests: string[] = []; + + class MockImage { + onerror: (() => void) | null = null; + onload: (() => void) | null = null; + + set src(value: string) { + requests.push(value); + if (!value) { + return; + } + const outcome = outcomes.shift() ?? "error"; + if (outcome === "pending") { + return; + } + queueMicrotask(() => { + if (outcome === "load") { + this.onload?.(); + return; + } + this.onerror?.(); + }); + } + } + + Object.defineProperty(globalThis, "Image", { + configurable: true, + value: MockImage, + }); + return requests; +} + +describe("pixel transport", () => { + test("retries an unacknowledged pixel load with the same event identity", async () => { + const requests = installImageOutcomes(["error", "load"]); + const tracker = createTracker(); + initPixelTracking(tracker); + + const result = await tracker.api.fetch("/", { + eventId: "event_1", + name: "pageview", + }); + + expect(result).toMatchObject({ ok: true, attempts: 2 }); + expect(requests).toHaveLength(2); + const first = new URL(requests[0] ?? ""); + const second = new URL(requests[1] ?? ""); + expect(first.searchParams.get("eventId")).toBe("event_1"); + expect(second.searchParams.get("eventId")).toBe("event_1"); + }); + + test("does not report an unverified image load as a beacon success", () => { + const tracker = createTracker(); + initPixelTracking(tracker); + + expect(tracker.sendBeacon({ eventId: "event_1" }, "/")).toBe(false); + }); + + test("cancels an active image request when tracking is cleared", async () => { + const requests = installImageOutcomes(["pending"]); + const tracker = createTracker(); + initPixelTracking(tracker); + + const delivery = tracker.api.fetch("/", { eventId: "event_1" }); + tracker.api.cancelPendingRequests(); + + expect(await delivery).toMatchObject({ + ok: false, + attempts: 1, + }); + expect(requests).toHaveLength(2); + expect(requests[1]).toBe(""); + }); + + test("bounds a pixel image load that never completes", async () => { + jest.useFakeTimers(); + const requests = installImageOutcomes(["pending"]); + const tracker = createTracker({ maxRetries: 0 }); + initPixelTracking(tracker); + + const delivery = tracker.api.fetch("/", { eventId: "event_1" }); + jest.advanceTimersByTime(10_000); + + expect(await delivery).toMatchObject({ + ok: false, + attempts: 1, + }); + expect(requests).toHaveLength(2); + expect(requests[1]).toBe(""); + }); + + test("uses the retry fallback for an invalid maxRetries value", async () => { + const requests = installImageOutcomes(["error", "error", "error", "error"]); + const tracker = createTracker({ maxRetries: Number.NaN }); + initPixelTracking(tracker); + + const result = await tracker.api.fetch("/", { eventId: "event_1" }); + + expect(result).toMatchObject({ ok: false, attempts: 4 }); + expect(requests).toHaveLength(4); + }); +});