From 90f9b534e862737530a056f4de10d799f98ad6fd Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:38:23 +0300 Subject: [PATCH 01/25] fix(ingest): make Vector delivery durable --- infra/ingest/vector.Dockerfile | 5 +- infra/ingest/vector.yaml | 88 +++++++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/infra/ingest/vector.Dockerfile b/infra/ingest/vector.Dockerfile index 123c27b6c6..5c29e7d7fc 100644 --- a/infra/ingest/vector.Dockerfile +++ b/infra/ingest/vector.Dockerfile @@ -5,9 +5,12 @@ FROM timberio/vector:0.50.0-alpine # When building from infra/ingest/: docker build -f vector.Dockerfile -t vector . COPY infra/ingest/vector.yaml /etc/vector/vector.yaml +# Vector's source of truth is Redpanda. This service intentionally keeps no +# delivery state on Railway's ephemeral filesystem. + WORKDIR /etc/vector HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \ CMD ["vector", "validate", "/etc/vector/vector.yaml"] -CMD ["--config", "/etc/vector/vector.yaml"] \ No newline at end of file +CMD ["--config", "/etc/vector/vector.yaml"] diff --git a/infra/ingest/vector.yaml b/infra/ingest/vector.yaml index 6c59ef3670..a848c5a6fa 100644 --- a/infra/ingest/vector.yaml +++ b/infra/ingest/vector.yaml @@ -1,3 +1,7 @@ +# Railway service filesystems are ephemeral. Redpanda is the durable backlog: +# each acknowledged ClickHouse sink below holds its Kafka offset until the +# warehouse confirms delivery, so an interrupted process replays from Kafka. + sources: redpanda: type: kafka @@ -58,6 +62,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -78,6 +88,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -98,6 +114,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -118,6 +140,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -138,6 +166,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -158,6 +192,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -178,6 +218,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -198,6 +244,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -218,6 +270,12 @@ sinks: max_events: 50000 max_bytes: 5000000 timeout_secs: 60 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -238,6 +296,12 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -258,15 +322,35 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: timestamp_format: unix_ms dead_letters: - type: file + type: kafka inputs: - route_analytics._unmatched - path: /var/log/vector/dead_letters.log + bootstrap_servers: "${REDPANDA_BROKER}" + topic: analytics-vector-dead-letters + sasl: + enabled: true + mechanism: SCRAM-SHA-256 + username: "${VECTOR_KAFKA_USER}" + password: "${VECTOR_KAFKA_PASSWORD}" + tls: + enabled: true + acknowledgements: + enabled: true + buffer: + type: memory + max_events: 500 + when_full: block encoding: codec: json From 00c664cea8fdd6095f2a7176863739f3a21c178d Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:40:01 +0300 Subject: [PATCH 02/25] fix(uptime): durably relay checks through BullMQ --- apps/uptime/src/actions.ts | 2 + apps/uptime/src/index.ts | 161 +++++++--- apps/uptime/src/lib/producer.test.ts | 144 +++++++++ apps/uptime/src/lib/producer.ts | 138 ++++----- apps/uptime/src/types.ts | 1 + .../src/uptime-transition-alerts.test.ts | 1 + apps/uptime/src/uptime-transition-alerts.ts | 2 +- apps/uptime/src/worker.test.ts | 184 +++++++++-- apps/uptime/src/worker.ts | 285 ++++++++++++++++-- packages/redis/uptime-queue.test.ts | 49 ++- packages/redis/uptime-queue.ts | 67 +++- .../rpc/src/services/uptime-scheduler.test.ts | 11 +- 12 files changed, 840 insertions(+), 205 deletions(-) create mode 100644 apps/uptime/src/lib/producer.test.ts diff --git a/apps/uptime/src/actions.ts b/apps/uptime/src/actions.ts index 01c168fb33..d7ac9538c1 100644 --- a/apps/uptime/src/actions.ts +++ b/apps/uptime/src/actions.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { connect } from "node:tls"; import { db } from "@databuddy/db"; import { @@ -429,6 +430,7 @@ const runUptimeCheck = ( site_id: siteId, url: normalizedUrl, timestamp, + event_id: randomUUID(), status: pingResult.ok ? MonitorStatus.UP : MonitorStatus.DOWN, http_code: pingResult.statusCode, ttfb_ms: pingResult.ttfb, diff --git a/apps/uptime/src/index.ts b/apps/uptime/src/index.ts index a5a87a9427..fcac07a5e3 100644 --- a/apps/uptime/src/index.ts +++ b/apps/uptime/src/index.ts @@ -15,7 +15,7 @@ import { import { disconnectProducer } from "./lib/producer"; import { captureError } from "./lib/tracing"; import { syncSchedulers } from "./sync-schedulers"; -import { startUptimeWorker } from "./worker"; +import { startUptimeDeliveryWorker, startUptimeWorker } from "./worker"; initLogger({ env: { @@ -29,6 +29,12 @@ initLogger({ sampling: {}, }); +let shuttingDown = false; +let shutdownExitCode = 0; +let uptimeWorker: ReturnType | null = null; +let uptimeDeliveryWorker: ReturnType | null = + null; + process.on("unhandledRejection", (reason, _promise) => { captureError(reason, { process: "unhandledRejection" }); log.error({ @@ -45,54 +51,116 @@ process.on("uncaughtException", (error) => { error_stack: error instanceof Error ? error.stack : undefined, error_source: "process", }); + // Continuing after an uncaught exception can corrupt worker state. Drain the + // durable relay within its normal bound, then let the platform restart us. + shutdown("uncaughtException", 1).catch((shutdownError) => { + captureError(shutdownError, { + process: "uncaughtException", + error_step: "fatal_shutdown", + }); + process.exit(1); + }); }); const DRAIN_TIMEOUT_MS = 10_000; -const drainAll = (worker: ReturnType | null) => - Effect.all( - [ - Effect.tryPromise({ - try: () => worker?.close() ?? Promise.resolve(), - catch: (c) => c, - }), - Effect.tryPromise({ try: () => closeUptimeQueue(), catch: (c) => c }), - Effect.tryPromise({ - try: () => flushBatchedUptimeDrain(), - catch: (c) => c, - }), - Effect.tryPromise({ - try: () => shutdownPostgres(), - catch: (c) => c, - }), - Effect.tryPromise({ try: () => disconnectProducer(), catch: (c) => c }), - ], - { concurrency: "unbounded" } - ).pipe( +const drainStep = (step: string, action: () => Promise) => + Effect.tryPromise({ + try: action, + catch: (cause) => cause, + }).pipe( + Effect.catch((cause) => + Effect.sync(() => + log.error({ + lifecycle: "shutdown", + error_step: step, + error_message: cause instanceof Error ? cause.message : String(cause), + }) + ) + ) + ); + +const drainAll = ( + worker: ReturnType | null, + deliveryWorker: ReturnType | null +) => + Effect.gen(function* () { + // Stop source admission before delivery so an acknowledged checkpoint is + // never handed to a producer that is already closing. Each cleanup is + // isolated: a failed close must never skip the queue, producer, or log drain. + yield* drainStep( + "uptime_worker_close", + () => worker?.close() ?? Promise.resolve() + ); + yield* drainStep( + "uptime_delivery_worker_close", + () => deliveryWorker?.close() ?? Promise.resolve() + ); + yield* Effect.all( + [ + drainStep("uptime_queue_close", () => closeUptimeQueue()), + drainStep("uptime_log_flush", () => flushBatchedUptimeDrain()), + drainStep("uptime_postgres_close", () => shutdownPostgres()), + drainStep("uptime_producer_disconnect", () => disconnectProducer()), + ], + { concurrency: "unbounded" } + ); + }).pipe( Effect.timeout(`${DRAIN_TIMEOUT_MS} millis`), - Effect.catch(() => + Effect.catch((cause) => Effect.sync(() => log.error({ lifecycle: "shutdown", - error_step: "drain_timeout", + error_step: + cause && + typeof cause === "object" && + "_tag" in cause && + cause._tag === "TimeoutError" + ? "drain_timeout" + : "drain_failed", drain_timeout_ms: DRAIN_TIMEOUT_MS, + error_message: cause instanceof Error ? cause.message : String(cause), }) ) ) ); -async function shutdown(signal: string) { +async function shutdown(signal: string, exitCode = 0) { + shutdownExitCode = Math.max(shutdownExitCode, exitCode); + if (shuttingDown) { + return; + } + shuttingDown = true; log.info("lifecycle", `${signal} received, shutting down gracefully`); - await Effect.runPromise(drainAll(uptimeWorker)); - process.exit(0); + try { + await Effect.runPromise(drainAll(uptimeWorker, uptimeDeliveryWorker)); + } catch (error) { + captureError(error, { lifecycle: "shutdown", error_step: "drain_failed" }); + log.error({ + lifecycle: "shutdown", + error_step: "drain_failed", + error_message: error instanceof Error ? error.message : String(error), + }); + } finally { + process.exit(shutdownExitCode); + } } -let uptimeWorker: ReturnType | null = null; - (async () => { if (UPTIME_ENV.isProduction) { - await syncSchedulers(); - uptimeWorker = startUptimeWorker(); + try { + await syncSchedulers(); + uptimeDeliveryWorker = startUptimeDeliveryWorker(); + uptimeWorker = startUptimeWorker(); + } catch (error) { + captureError(error, { error_step: "uptime_startup" }); + log.error({ + lifecycle: "startup", + error_step: "uptime_startup", + error_message: error instanceof Error ? error.message : String(error), + }); + process.exit(1); + } } else { log.info( "lifecycle", @@ -141,17 +209,20 @@ const probe = (_name: string, fn: () => Promise) => const healthCheck = Effect.gen(function* () { const { db, sql } = yield* Effect.promise(() => import("@databuddy/db")); - const { getUptimeQueue } = yield* Effect.promise( + const { getUptimeDeliveryQueue, getUptimeQueue } = yield* Effect.promise( () => import("@databuddy/redis") ); const { Kafka } = yield* Effect.promise(() => import("kafkajs")); - const [postgres, bullmqRedis, redpanda] = yield* Effect.all( + const [postgres, bullmqRedis, deliveryRedis, redpanda] = yield* Effect.all( [ probe("postgres", () => db.execute(sql`SELECT 1`).then(() => {})), probe("bullmqRedis", async () => { await getUptimeQueue().count(); }), + probe("uptimeDeliveryRedis", async () => { + await getUptimeDeliveryQueue().count(); + }), probe("redpanda", async () => { const broker = process.env.REDPANDA_BROKER; if (!broker) { @@ -161,15 +232,16 @@ const healthCheck = Effect.gen(function* () { clientId: "health", brokers: [broker], connectionTimeout: 5000, - ...(process.env.REDPANDA_USER && - process.env.REDPANDA_PASSWORD && { - sasl: { - mechanism: "scram-sha-256", - username: process.env.REDPANDA_USER, - password: process.env.REDPANDA_PASSWORD, - }, - ssl: process.env.REDPANDA_SSL === "true", - }), + ...(process.env.REDPANDA_USER && process.env.REDPANDA_PASSWORD + ? { + sasl: { + mechanism: "scram-sha-256", + username: process.env.REDPANDA_USER, + password: process.env.REDPANDA_PASSWORD, + }, + } + : {}), + ...(process.env.REDPANDA_SSL === "true" ? { ssl: true } : {}), }); const admin = kafka.admin(); try { @@ -182,7 +254,12 @@ const healthCheck = Effect.gen(function* () { { concurrency: "unbounded" } ); - const services = { postgres, bullmqRedis, redpanda }; + const services = { + postgres, + bullmqRedis, + uptimeDeliveryRedis: deliveryRedis, + redpanda, + }; const status = Object.values(services).every((s) => s.status === "ok") ? "ok" : "degraded"; diff --git a/apps/uptime/src/lib/producer.test.ts b/apps/uptime/src/lib/producer.test.ts new file mode 100644 index 0000000000..3ad177a6fe --- /dev/null +++ b/apps/uptime/src/lib/producer.test.ts @@ -0,0 +1,144 @@ +import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test"; + +const kafkaConfigs: unknown[] = []; +const producers: Array> = []; +const captureError = mock(() => {}); + +class KafkaMock { + constructor(config: unknown) { + kafkaConfigs.push(config); + } + + producer() { + const producer = producers.shift(); + if (!producer) { + throw new Error("No test producer configured"); + } + return producer; + } +} + +mock.module("kafkajs", () => ({ + CompressionTypes: { GZIP: 1 }, + Kafka: KafkaMock, +})); + +mock.module("./tracing", () => ({ captureError })); + +const { disconnectProducer, sendUptimeEvent } = await import("./producer"); + +const environmentKeys = [ + "REDPANDA_BROKER", + "REDPANDA_PASSWORD", + "REDPANDA_SSL", + "REDPANDA_USER", +] as const; +const originalEnvironment = new Map( + environmentKeys.map((key) => [key, process.env[key]]) +); + +function createProducer( + overrides: Partial<{ + connect: () => Promise; + disconnect: () => Promise; + send: () => Promise; + }> = {} +) { + return { + connect: mock(overrides.connect ?? (() => Promise.resolve())), + disconnect: mock(overrides.disconnect ?? (() => Promise.resolve())), + send: mock(overrides.send ?? (() => Promise.resolve())), + }; +} + +beforeEach(async () => { + await disconnectProducer(); + kafkaConfigs.length = 0; + producers.length = 0; + captureError.mockClear(); + process.env.REDPANDA_BROKER = "redpanda.test:9092"; + delete process.env.REDPANDA_PASSWORD; + delete process.env.REDPANDA_SSL; + delete process.env.REDPANDA_USER; +}); + +afterAll(async () => { + await disconnectProducer(); + for (const key of environmentKeys) { + const value = originalEnvironment.get(key); + if (value === undefined) { + delete process.env[key]; + continue; + } + process.env[key] = value; + } +}); + +describe("sendUptimeEvent", () => { + test("shares one in-flight connection across concurrent cold-start sends", async () => { + let resolveConnection: (() => void) | undefined; + const producer = createProducer({ + connect: () => + new Promise((resolve) => { + resolveConnection = resolve; + }), + }); + producers.push(producer); + + const sends = Array.from({ length: 20 }, () => sendUptimeEvent({ ok: true })); + + expect(producer.connect).toHaveBeenCalledTimes(1); + expect(resolveConnection).toBeDefined(); + resolveConnection?.(); + + expect(await Promise.all(sends)).toEqual( + Array.from({ length: 20 }, () => ({ sent: true })) + ); + expect(producer.send).toHaveBeenCalledTimes(20); + expect(producer.send).toHaveBeenCalledWith( + expect.objectContaining({ acks: -1 }) + ); + expect(kafkaConfigs).toHaveLength(1); + }); + + test("disconnects a failed producer and reconnects for the next event", async () => { + const failedProducer = createProducer({ + send: () => Promise.reject(new Error("broker unavailable")), + }); + const recoveredProducer = createProducer(); + producers.push(failedProducer, recoveredProducer); + + expect(await sendUptimeEvent({ attempt: 1 })).toEqual({ + sent: false, + reason: "send_failed", + }); + expect(failedProducer.disconnect).toHaveBeenCalledTimes(1); + + expect(await sendUptimeEvent({ attempt: 2 })).toEqual({ sent: true }); + expect(recoveredProducer.connect).toHaveBeenCalledTimes(1); + expect(recoveredProducer.send).toHaveBeenCalledTimes(1); + expect(kafkaConfigs).toHaveLength(2); + }); + + test("reports an unavailable producer when Kafka is not configured", async () => { + delete process.env.REDPANDA_BROKER; + + expect(await sendUptimeEvent({ ok: true })).toEqual({ + sent: false, + reason: "producer_unavailable", + }); + expect(kafkaConfigs).toEqual([]); + }); + + test("uses TLS when configured even without SASL credentials", async () => { + process.env.REDPANDA_SSL = "true"; + const producer = createProducer(); + producers.push(producer); + + expect(await sendUptimeEvent({ ok: true })).toEqual({ sent: true }); + expect(kafkaConfigs[0]).toEqual( + expect.objectContaining({ ssl: true }) + ); + expect(kafkaConfigs[0]).not.toHaveProperty("sasl"); + }); +}); diff --git a/apps/uptime/src/lib/producer.ts b/apps/uptime/src/lib/producer.ts index 2ba8ca4e65..939a8097ba 100644 --- a/apps/uptime/src/lib/producer.ts +++ b/apps/uptime/src/lib/producer.ts @@ -1,15 +1,8 @@ import { CompressionTypes, Kafka, type Producer } from "kafkajs"; -import { Context, Data, Effect, Layer } from "effect"; import { captureError } from "./tracing"; const TOPIC = "analytics-uptime-checks"; -class KafkaSendError extends Data.TaggedError("KafkaSendError")<{ - cause: unknown; -}> {} - -const KafkaProducer = Context.Service("KafkaProducer"); - const connectProducer = (): Promise => { const broker = process.env.REDPANDA_BROKER; if (!broker) { @@ -21,11 +14,10 @@ const connectProducer = (): Promise => { const kafka = new Kafka({ brokers: [broker], clientId: "uptime-producer", - ...(username && - password && { - sasl: { mechanism: "scram-sha-256", username, password }, - ssl: process.env.REDPANDA_SSL === "true", - }), + ...(username && password + ? { sasl: { mechanism: "scram-sha-256", username, password } } + : {}), + ...(process.env.REDPANDA_SSL === "true" ? { ssl: true } : {}), }); const producer = kafka.producer({ @@ -37,89 +29,73 @@ const connectProducer = (): Promise => { return producer.connect().then(() => producer); }; -const KafkaProducerLive = Layer.effect( - KafkaProducer, - Effect.acquireRelease( - Effect.tryPromise({ - try: connectProducer, - catch: (cause) => { - captureError(cause, { error_step: "kafka_producer_connect" }); - return cause as Error; - }, - }), - (producer) => - Effect.tryPromise({ - try: () => producer.disconnect(), - catch: (cause) => cause, - }).pipe( - Effect.catch((cause) => { - captureError(cause, { - error_step: "kafka_producer_disconnect", - }); - return Effect.void; - }) - ) - ) -); - -const sendEvent = (event: unknown, key?: string) => - Effect.gen(function* () { - const producer = yield* KafkaProducer; - yield* Effect.tryPromise({ - try: () => - producer.send({ - topic: TOPIC, - messages: [ - { - value: JSON.stringify(event, (_k, v) => - v === undefined ? null : v - ), - key, - }, - ], - compression: CompressionTypes.GZIP, - }), - catch: (cause) => new KafkaSendError({ cause }), - }); - }); - -export { KafkaProducer, KafkaProducerLive, KafkaSendError, sendEvent }; - let singletonProducer: Producer | null = null; -let singletonConnected = false; - -async function ensureProducer(): Promise { - if (singletonConnected && singletonProducer) { - return singletonProducer; +let singletonConnection: Promise | null = null; + +export type UptimeEventSendResult = + | { sent: true } + | { + sent: false; + reason: "producer_unavailable" | "send_failed"; + }; + +function ensureProducer(): Promise { + if (singletonProducer) { + return Promise.resolve(singletonProducer); } if (!process.env.REDPANDA_BROKER) { - return null; + return Promise.resolve(null); } + if (singletonConnection) { + return singletonConnection; + } + + singletonConnection = connectProducer() + .then((producer) => { + singletonProducer = producer; + return producer; + }) + .catch((error) => { + captureError(error, { error_step: "kafka_producer_connect" }); + singletonProducer = null; + return null; + }) + .finally(() => { + singletonConnection = null; + }); + + return singletonConnection; +} + +async function resetProducer(producer: Producer): Promise { + if (singletonProducer !== producer) { + return; + } + + singletonProducer = null; + try { - singletonProducer = await connectProducer(); - singletonConnected = true; - return singletonProducer; + await producer.disconnect(); } catch (error) { - captureError(error, { error_step: "kafka_producer_connect" }); - singletonConnected = false; - return null; + captureError(error, { error_step: "kafka_producer_disconnect" }); } } export async function sendUptimeEvent( event: unknown, key?: string -): Promise { +): Promise { const p = await ensureProducer(); if (!p) { - return; + return { sent: false, reason: "producer_unavailable" }; } try { await p.send({ topic: TOPIC, + acks: -1, messages: [ { value: JSON.stringify(event, (_k, v) => (v === undefined ? null : v)), @@ -128,20 +104,18 @@ export async function sendUptimeEvent( ], compression: CompressionTypes.GZIP, }); + return { sent: true }; } catch (error) { captureError(error, { error_step: "kafka_producer_send" }); + await resetProducer(p); + return { sent: false, reason: "send_failed" }; } } export async function disconnectProducer(): Promise { - if (!singletonProducer) { + const producer = singletonProducer ?? (await singletonConnection); + if (!producer) { return; } - try { - await singletonProducer.disconnect(); - } catch (error) { - captureError(error, { error_step: "kafka_producer_disconnect" }); - } - singletonProducer = null; - singletonConnected = false; + await resetProducer(producer); } diff --git a/apps/uptime/src/types.ts b/apps/uptime/src/types.ts index f207bb3eb7..54aff85f8c 100644 --- a/apps/uptime/src/types.ts +++ b/apps/uptime/src/types.ts @@ -11,6 +11,7 @@ export interface UptimeData { content_hash: string; env: string; error: string; + event_id: string; failure_streak: number; http_code: number; json_data?: string; diff --git a/apps/uptime/src/uptime-transition-alerts.test.ts b/apps/uptime/src/uptime-transition-alerts.test.ts index f0a921a482..d41a5f1cc3 100644 --- a/apps/uptime/src/uptime-transition-alerts.test.ts +++ b/apps/uptime/src/uptime-transition-alerts.test.ts @@ -16,6 +16,7 @@ const baseUptimeData: UptimeData = { attempt: 1, check_type: "http", content_hash: "", + event_id: "uptime-event-1", env: "production", error: "", failure_streak: 0, diff --git a/apps/uptime/src/uptime-transition-alerts.ts b/apps/uptime/src/uptime-transition-alerts.ts index 1c1f64e581..87c755019d 100644 --- a/apps/uptime/src/uptime-transition-alerts.ts +++ b/apps/uptime/src/uptime-transition-alerts.ts @@ -91,7 +91,7 @@ export function shouldReleaseTransitionClaim( if (firedAlarmCount > 0) { // A successful non-email destination owns the transition claim. Releasing // it would duplicate that delivery; retrying only the deferred email needs - // a durable per-destination outbox. + // a dedicated durable delivery record. return false; } return sendableAlarmCount > 0 || emailDeliveryDeferred; diff --git a/apps/uptime/src/worker.test.ts b/apps/uptime/src/worker.test.ts index c1c4e2096f..439c0d92e8 100644 --- a/apps/uptime/src/worker.test.ts +++ b/apps/uptime/src/worker.test.ts @@ -5,6 +5,7 @@ import { DEFAULT_UPTIME_WORKER_CONCURRENCY, getUptimeWorkerConcurrency, processUptimeCheck, + processUptimeDeliveryJob, processUptimeJob, type UptimeWorkerDeps, } from "./worker"; @@ -18,11 +19,13 @@ const calls = { cacheBust: boolean | undefined; extractHealth: boolean | undefined; }>, + checkpoint: [] as UptimeData[], + delivery: [] as UptimeData[], email: [] as Array<{ schedule: ScheduleData; data: UptimeData }>, loggerFields: [] as Array>, loggerEmitted: [] as Array, + order: [] as string[], reaped: [] as string[], - send: [] as Array<{ data: UptimeData; monitorId: string }>, }; let lookupResult: @@ -55,6 +58,7 @@ function uptimeData(values: Partial = {}): UptimeData { attempt: 1, check_type: "http", content_hash: "hash", + event_id: "uptime-event-1", env: "test", error: "", failure_streak: 0, @@ -104,6 +108,10 @@ function deps(): UptimeWorkerDeps { error: () => {}, } as never; }, + enqueueUptimeDelivery: async (data) => { + calls.delivery.push(data); + calls.order.push("enqueue"); + }, getPreviousMonitorStatus: async () => previousStatus, isHealthExtractionEnabled: (config) => typeof config === "object" && @@ -111,30 +119,50 @@ function deps(): UptimeWorkerDeps { "enabled" in config && config.enabled === true, lookupSchedule: async () => lookupResult, + sendUptimeEvent: async () => ({ sent: true }), reapOrphanScheduler: async (scheduleId: string) => { calls.reaped.push(scheduleId); if (reapBehaviour === "throw") { throw new Error("redis reap blew up"); } }, - sendUptimeEvent: async (data, monitorId) => { - calls.send.push({ data, monitorId }); - }, fireTransitionAlerts: async (payload) => { calls.email.push(payload); + calls.order.push("alert"); return { transition_kind: null, alarms_fired: 0 }; }, }; } +type UptimeEventCheckpoint = (data: UptimeData) => Promise; +const noOpCheckpoint: UptimeEventCheckpoint = async () => {}; + +function processUptimeCheckForTest( + scheduleId: string, + trigger: "manual" | "scheduled", + workerDeps: UptimeWorkerDeps = deps(), + jobMeta?: { id?: string; attempt?: number }, + checkpoint: UptimeEventCheckpoint = noOpCheckpoint +) { + return processUptimeCheck( + scheduleId, + trigger, + workerDeps, + jobMeta, + checkpoint + ); +} + beforeEach(() => { calls.captureError = []; calls.check = []; + calls.checkpoint = []; + calls.delivery = []; calls.email = []; calls.loggerFields = []; calls.loggerEmitted = []; + calls.order = []; calls.reaped = []; - calls.send = []; lookupResult = { success: true, data: schedule() }; checkResult = { success: true, data: uptimeData() }; previousStatus = 0; @@ -173,6 +201,7 @@ describe("processUptimeCheck", () => { { name: "surprise", data: { scheduleId: "schedule-1", trigger: "scheduled" }, + updateData: async () => {}, }, deps() ) @@ -186,18 +215,22 @@ describe("processUptimeCheck", () => { { name: "uptime-check", data: { scheduleId: "schedule-1", trigger: "manual" }, + updateData: async (data) => { + calls.checkpoint.push(data.delivery?.event as UptimeData); + }, }, deps() ); expect(calls.check).toHaveLength(1); + expect(calls.checkpoint).toEqual([uptimeData()]); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ uptime_trigger: "manual" }) ); }); it("runs a scheduled check and emits events, status, and transition email work", async () => { - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.check).toEqual([ { @@ -208,10 +241,9 @@ describe("processUptimeCheck", () => { extractHealth: true, }, ]); - expect(calls.send).toEqual([ - { data: uptimeData(), monitorId: "website-1" }, - ]); + expect(calls.delivery).toEqual([uptimeData()]); expect(calls.email).toHaveLength(1); + expect(calls.order).toEqual(["enqueue", "alert"]); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ schedule_id: "schedule-1", @@ -229,6 +261,7 @@ describe("processUptimeCheck", () => { ); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ + event_id: "uptime-event-1", outcome: "up", previous_uptime_status: 0, ttfb_ms: 10, @@ -236,7 +269,7 @@ describe("processUptimeCheck", () => { }) ); expect(calls.loggerFields).toContainEqual( - expect.objectContaining({ kafka_sent: true }) + expect.objectContaining({ delivery_queue_admitted: true }) ); expect(calls.loggerEmitted).toHaveLength(1); }); @@ -244,7 +277,7 @@ describe("processUptimeCheck", () => { it("records -1 when no previous monitor status exists", async () => { previousStatus = undefined; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ previous_uptime_status: -1 }) @@ -257,7 +290,7 @@ describe("processUptimeCheck", () => { data: schedule({ website: null, websiteId: null, timeout: null }), }; - await processUptimeCheck("schedule-only", "manual", deps()); + await processUptimeCheckForTest("schedule-only", "manual", deps()); expect(calls.check).toEqual([ { @@ -285,10 +318,10 @@ describe("processUptimeCheck", () => { it("skips paused schedules without running the check", async () => { lookupResult = { success: true, data: schedule({ isPaused: true }) }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.check).toEqual([]); - expect(calls.send).toEqual([]); + expect(calls.delivery).toEqual([]); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ organization_id: "org-1" }) ); @@ -301,7 +334,7 @@ describe("processUptimeCheck", () => { it("skips missing schedules without throwing", async () => { lookupResult = { success: false, error: "not found" }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); expect(calls.check).toEqual([]); expect(calls.loggerFields).toContainEqual( @@ -320,7 +353,7 @@ describe("processUptimeCheck", () => { reason: "not_found", }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual(["schedule-1"]); @@ -339,7 +372,7 @@ describe("processUptimeCheck", () => { reason: "malformed", }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual(["schedule-1"]); @@ -355,7 +388,7 @@ describe("processUptimeCheck", () => { reason: "transient", }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual([]); @@ -367,7 +400,7 @@ describe("processUptimeCheck", () => { it("does NOT reap when reason is missing on legacy failures (fail-open)", async () => { lookupResult = { success: false, error: "boom" }; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual([]); @@ -381,7 +414,7 @@ describe("processUptimeCheck", () => { }; reapBehaviour = "throw"; - await processUptimeCheck("schedule-1", "scheduled", deps()); + await processUptimeCheckForTest("schedule-1", "scheduled", deps()); await flushMicrotasks(); expect(calls.reaped).toEqual(["schedule-1"]); @@ -406,7 +439,7 @@ describe("processUptimeCheck", () => { checkResult = { success: false, error: "timeout" }; await expect( - processUptimeCheck("schedule-1", "scheduled", deps()) + processUptimeCheckForTest("schedule-1", "scheduled", deps()) ).rejects.toThrow("timeout"); expect(calls.loggerFields).toContainEqual( expect.objectContaining({ @@ -417,20 +450,113 @@ describe("processUptimeCheck", () => { expect(calls.loggerEmitted).toHaveLength(1); }); - it("captures producer errors on the wide event without failing the job", async () => { + it("persists the exact event before enqueueing it for delivery", async () => { + await processUptimeCheckForTest( + "schedule-1", + "manual", + deps(), + undefined, + async (data) => { + calls.checkpoint.push(data); + calls.order.push("checkpoint"); + } + ); + + expect(calls.checkpoint).toEqual([uptimeData()]); + expect(calls.delivery).toEqual([uptimeData()]); + expect(calls.order).toEqual(["checkpoint", "enqueue", "alert"]); + }); + + it("retries the source job when the durable Redis checkpoint fails", async () => { + await expect( + processUptimeCheckForTest( + "schedule-1", + "manual", + deps(), + undefined, + async () => { + throw new Error("redis unavailable"); + } + ) + ).rejects.toThrow("redis unavailable"); + + expect(calls.delivery).toEqual([]); + expect(calls.email).toEqual([]); + expect(calls.captureError).toContainEqual( + expect.objectContaining({ + context: expect.objectContaining({ + error_step: "uptime_delivery_checkpoint", + event_id: "uptime-event-1", + }), + }) + ); + }); + + it("retries the source job when delivery queue admission fails", async () => { const failingDeps = deps(); - failingDeps.sendUptimeEvent = async () => { - throw new Error("producer unavailable"); + failingDeps.enqueueUptimeDelivery = async () => { + throw new Error("redis unavailable"); }; - await processUptimeCheck("schedule-1", "manual", failingDeps); + await expect( + processUptimeCheckForTest("schedule-1", "manual", failingDeps) + ).rejects.toThrow("redis unavailable"); - expect(calls.loggerFields).toContainEqual( + expect(calls.email).toEqual([]); + expect(calls.captureError).toContainEqual( expect.objectContaining({ - kafka_sent: false, - kafka_error: "producer unavailable", + context: expect.objectContaining({ + error_step: "uptime_delivery_enqueue", + event_id: "uptime-event-1", + }), + }) + ); + }); + + it("replays a checkpointed event without running another probe", async () => { + await processUptimeJob( + { + name: "uptime-check", + data: { + delivery: { event: uptimeData() }, + scheduleId: "schedule-1", + trigger: "scheduled", + }, + updateData: async () => {}, + }, + deps() + ); + + expect(calls.check).toEqual([]); + expect(calls.delivery).toEqual([uptimeData()]); + expect(calls.email).toHaveLength(1); + }); + + it("retries a delivery job when Redpanda does not acknowledge it", async () => { + const failingDeps = deps(); + failingDeps.sendUptimeEvent = async () => ({ + sent: false, + reason: "send_failed", + }); + + await expect( + processUptimeDeliveryJob( + { + data: { event: uptimeData() }, + id: "uptime-delivery-uptime-event-1", + name: "uptime-event-delivery", + }, + failingDeps + ) + ).rejects.toThrow("Redpanda delivery failed: send_failed"); + + expect(calls.captureError).toContainEqual( + expect.objectContaining({ + context: expect.objectContaining({ + error_step: "uptime_delivery_send", + event_id: "uptime-event-1", + }), }) ); - expect(calls.loggerEmitted).toHaveLength(1); }); }); diff --git a/apps/uptime/src/worker.ts b/apps/uptime/src/worker.ts index d8d2a5fab7..56c971679f 100644 --- a/apps/uptime/src/worker.ts +++ b/apps/uptime/src/worker.ts @@ -1,10 +1,15 @@ import { getBullMQWorkerConnectionOptions, + getUptimeDeliveryQueue, getUptimeQueue, type UptimeCheckJobData, + type UptimeDeliveryJobData, UPTIME_CHECK_JOB_NAME, + UPTIME_DELIVERY_JOB_NAME, + UPTIME_DELIVERY_QUEUE_NAME, UPTIME_JOB_TIMEOUT_MS, UPTIME_QUEUE_NAME, + uptimeDeliveryJobId, uptimeSchedulerId, } from "@databuddy/redis"; import { Worker } from "bullmq"; @@ -18,7 +23,7 @@ import { lookupSchedule, } from "./actions"; import { isHealthExtractionEnabled } from "./json-parser"; -import { sendUptimeEvent } from "./lib/producer"; +import { sendUptimeEvent, type UptimeEventSendResult } from "./lib/producer"; import { captureError } from "./lib/tracing"; import { MonitorStatus, @@ -54,6 +59,10 @@ class CheckFailed extends Data.TaggedError("CheckFailed")<{ message: string; }> {} +class DeliveryHandoffFailed extends Data.TaggedError("DeliveryHandoffFailed")<{ + message: string; +}> {} + export interface UptimeWorkerDeps { captureError: ( error: unknown, @@ -68,6 +77,7 @@ export interface UptimeWorkerDeps { createLogger: ( fields: Record ) => RequestLogger; + enqueueUptimeDelivery: (data: UptimeData) => Promise; fireTransitionAlerts: (options: { schedule: ScheduleData; data: UptimeData; @@ -80,13 +90,25 @@ export interface UptimeWorkerDeps { isHealthExtractionEnabled: (config: unknown) => boolean; lookupSchedule: (scheduleId: string) => Promise>; reapOrphanScheduler: (scheduleId: string) => Promise; - sendUptimeEvent: (data: UptimeData, monitorId: string) => Promise; + sendUptimeEvent: ( + event: unknown, + key?: string + ) => Promise; +} + +async function defaultEnqueueUptimeDelivery(data: UptimeData): Promise { + await getUptimeDeliveryQueue().add( + UPTIME_DELIVERY_JOB_NAME, + { event: data }, + { jobId: uptimeDeliveryJobId(data.event_id) } + ); } const uptimeWorkerDeps: UptimeWorkerDeps = { captureError, checkUptime, createLogger: (fields) => createLogger(fields), + enqueueUptimeDelivery: defaultEnqueueUptimeDelivery, getPreviousMonitorStatus, isHealthExtractionEnabled, lookupSchedule, @@ -96,6 +118,7 @@ const uptimeWorkerDeps: UptimeWorkerDeps = { }; export const DEFAULT_UPTIME_WORKER_CONCURRENCY = 10_000; +const MAX_STALLED_COUNT = 1_000_000; export function getUptimeWorkerConcurrency( value = process.env.UPTIME_WORKER_CONCURRENCY @@ -117,6 +140,14 @@ export interface UptimeWorkerJob { data: UptimeCheckJobData; id?: string; name: string; + updateData: (data: UptimeCheckJobData) => Promise; +} + +export interface UptimeDeliveryWorkerJob { + attemptsMade?: number; + data: UptimeDeliveryJobData; + id?: string; + name: string; } const timed = ( @@ -171,26 +202,24 @@ const fetchPreviousStatus = (monitorId: string, deps: UptimeWorkerDeps) => Effect.orElseSucceed(() => undefined) ); -const publishEvent = ( +type UptimeEventCheckpoint = (data: UptimeData) => Promise; + +const handoffDelivery = ( data: UptimeData, - monitorId: string, - deps: UptimeWorkerDeps, - log: RequestLogger + handoff: () => Promise, + errorStep: "uptime_delivery_checkpoint" | "uptime_delivery_enqueue", + deps: UptimeWorkerDeps ) => Effect.tryPromise({ - try: () => deps.sendUptimeEvent(data, monitorId), - catch: (cause) => cause, - }).pipe( - Effect.tap(() => Effect.sync(() => log.set({ kafka_sent: true }))), - Effect.catch((error) => - Effect.sync(() => - log.set({ - kafka_sent: false, - kafka_error: error instanceof Error ? error.message : "unknown", - }) - ) - ) - ); + try: handoff, + catch: (cause) => { + deps.captureError(cause, { + error_step: errorStep, + event_id: data.event_id, + }); + return new DeliveryHandoffFailed({ message: String(cause) }); + }, + }); const runTransitionAlerts = ( schedule: ScheduleData, @@ -250,7 +279,8 @@ function reapScheduler( const processCheck = ( scheduleId: string, log: RequestLogger, - deps: UptimeWorkerDeps + deps: UptimeWorkerDeps, + checkpoint: UptimeEventCheckpoint ) => Effect.gen(function* () { const schedule = yield* timed( @@ -320,6 +350,7 @@ const processCheck = ( ); log.set({ + event_id: data.event_id, outcome: data.status === MonitorStatus.UP ? "up" : "down", previous_uptime_status: previousStatus === undefined ? -1 : previousStatus, @@ -337,7 +368,31 @@ const processCheck = ( error_message: data.error || "", }); - yield* timed("kafka", publishEvent(data, monitorId, deps, log), log); + // The source BullMQ job is the durable checkpoint. Persist the exact + // probe result before queuing it for delivery so a retry never runs a + // replacement probe with a new timestamp or event ID. + yield* timed( + "delivery_checkpoint", + handoffDelivery( + data, + () => checkpoint(data), + "uptime_delivery_checkpoint", + deps + ), + log + ); + + yield* timed( + "delivery_queue_admission", + handoffDelivery( + data, + () => deps.enqueueUptimeDelivery(data), + "uptime_delivery_enqueue", + deps + ), + log + ); + log.set({ delivery_queue_admitted: true }); yield* timed( "transition_email", @@ -349,8 +404,9 @@ const processCheck = ( export async function processUptimeCheck( scheduleId: string, trigger: UptimeCheckJobData["trigger"], - deps: UptimeWorkerDeps = uptimeWorkerDeps, - jobMeta?: { id?: string; attempt?: number } + deps: UptimeWorkerDeps, + jobMeta: { id?: string; attempt?: number } | undefined, + checkpoint: UptimeEventCheckpoint ) { const startedAt = performance.now(); const log = deps.createLogger({ @@ -360,19 +416,98 @@ export async function processUptimeCheck( ...(jobMeta?.attempt ? { job_attempt: jobMeta.attempt } : {}), }); - const exit = await Effect.runPromiseExit(processCheck(scheduleId, log, deps)); + const exit = await Effect.runPromiseExit( + processCheck(scheduleId, log, deps, checkpoint) + ); log.set({ check_duration_ms: Math.round(performance.now() - startedAt) }); log.emit(); if (Exit.isFailure(exit)) { const error = Cause.squash(exit.cause); - if (error instanceof CheckFailed) { + if ( + error instanceof CheckFailed || + error instanceof DeliveryHandoffFailed + ) { throw new Error(error.message); } } } +function isUptimeData(value: unknown): value is UptimeData { + if (typeof value !== "object" || value === null) { + return false; + } + + const event = value as { + event_id?: unknown; + site_id?: unknown; + timestamp?: unknown; + }; + return ( + typeof event.event_id === "string" && + typeof event.site_id === "string" && + typeof event.timestamp === "number" + ); +} + +async function replayPersistedUptimeDelivery( + job: UptimeWorkerJob, + data: UptimeData, + deps: UptimeWorkerDeps +): Promise { + const startedAt = performance.now(); + const log = deps.createLogger({ + schedule_id: job.data.scheduleId, + uptime_trigger: job.data.trigger, + event_id: data.event_id, + delivery_replay: true, + ...(job.id ? { job_id: job.id } : {}), + ...(job.attemptsMade ? { job_attempt: job.attemptsMade } : {}), + }); + + try { + await Effect.runPromise( + timed( + "delivery_queue_admission", + handoffDelivery( + data, + () => deps.enqueueUptimeDelivery(data), + "uptime_delivery_enqueue", + deps + ), + log + ) + ); + log.set({ delivery_queue_admitted: true }); + + const scheduleExit = await Effect.runPromiseExit( + resolveSchedule(job.data.scheduleId, deps) + ); + if (Exit.isSuccess(scheduleExit)) { + await Effect.runPromise( + timed( + "transition_email", + runTransitionAlerts(scheduleExit.value, data, undefined, deps, log), + log + ) + ); + } else { + const error = Cause.squash(scheduleExit.cause); + log.set({ + transition_alert_skipped: true, + transition_alert_skip_reason: + error instanceof Error ? error.message : String(error), + }); + } + } finally { + log.set({ + delivery_replay_duration_ms: Math.round(performance.now() - startedAt), + }); + log.emit(); + } +} + export async function processUptimeJob( job: UptimeWorkerJob, deps: UptimeWorkerDeps = uptimeWorkerDeps @@ -380,10 +515,57 @@ export async function processUptimeJob( if (job.name !== UPTIME_CHECK_JOB_NAME) { throw new Error(`Unknown uptime job: ${job.name}`); } - await processUptimeCheck(job.data.scheduleId, job.data.trigger, deps, { - id: job.id, - attempt: job.attemptsMade, - }); + + const persistedEvent = job.data.delivery?.event; + if (persistedEvent !== undefined) { + if (!isUptimeData(persistedEvent)) { + throw new Error("Invalid persisted uptime delivery payload"); + } + await replayPersistedUptimeDelivery(job, persistedEvent, deps); + return; + } + + await processUptimeCheck( + job.data.scheduleId, + job.data.trigger, + deps, + { + id: job.id, + attempt: job.attemptsMade, + }, + async (data) => + job.updateData({ + ...job.data, + delivery: { event: data }, + }) + ); +} + +export async function processUptimeDeliveryJob( + job: UptimeDeliveryWorkerJob, + deps: UptimeWorkerDeps = uptimeWorkerDeps +): Promise { + if (job.name !== UPTIME_DELIVERY_JOB_NAME) { + throw new Error(`Unknown uptime delivery job: ${job.name}`); + } + if (!isUptimeData(job.data.event)) { + throw new Error("Invalid uptime delivery payload"); + } + + const data = job.data.event; + try { + const result = await deps.sendUptimeEvent(data, data.site_id); + if (!result.sent) { + throw new Error(`Redpanda delivery failed: ${result.reason}`); + } + } catch (error) { + deps.captureError(error, { + error_step: "uptime_delivery_send", + event_id: data.event_id, + job_id: job.id ?? "", + }); + throw error; + } } export function startUptimeWorker() { @@ -394,13 +576,14 @@ export function startUptimeWorker() { connection: getBullMQWorkerConnectionOptions(), concurrency: getUptimeWorkerConcurrency(), lockDuration: UPTIME_JOB_TIMEOUT_MS * 3, + maxStalledCount: MAX_STALLED_COUNT, stalledInterval: UPTIME_JOB_TIMEOUT_MS * 4, } ); worker.on("failed", (job, error) => { const attemptsMade = job?.attemptsMade ?? 0; - const maxAttempts = job?.opts?.attempts ?? 3; + const maxAttempts = job?.opts?.attempts ?? 1_000_000; const isFinalAttempt = attemptsMade >= maxAttempts; captureError(error, { @@ -431,3 +614,45 @@ export function startUptimeWorker() { return worker; } + +export function startUptimeDeliveryWorker() { + const worker = new Worker( + UPTIME_DELIVERY_QUEUE_NAME, + (job) => processUptimeDeliveryJob(job), + { + connection: getBullMQWorkerConnectionOptions(), + concurrency: 1, + lockDuration: UPTIME_JOB_TIMEOUT_MS * 3, + maxStalledCount: MAX_STALLED_COUNT, + stalledInterval: UPTIME_JOB_TIMEOUT_MS * 4, + } + ); + + worker.on("failed", (job, error) => { + captureError(error, { + error_step: "uptime_delivery_worker_job_failed", + event_id: + job && isUptimeData(job.data.event) ? job.data.event.event_id : "", + job_id: job?.id ?? "", + attempts_used: job?.attemptsMade ?? 0, + attempts_max: job?.opts?.attempts ?? 1_000_000, + }); + }); + + worker.on("stalled", (jobId) => { + log.warn({ + service: "uptime", + error_step: "uptime_delivery_worker_job_stalled", + error_message: "BullMQ delivery job stalled", + job_id: jobId, + }); + }); + + worker.on("error", (error) => { + captureError(error, { + error_step: "uptime_delivery_worker_error", + }); + }); + + return worker; +} diff --git a/packages/redis/uptime-queue.test.ts b/packages/redis/uptime-queue.test.ts index ef95845a37..c309cdaa89 100644 --- a/packages/redis/uptime-queue.test.ts +++ b/packages/redis/uptime-queue.test.ts @@ -27,10 +27,15 @@ process.env.BULLMQ_REDIS_URL = "redis://queue-user:queue-pass@queue.test:6381/4" const { closeUptimeQueue, + getUptimeDeliveryQueue, getUptimeQueue, UPTIME_CHECK_JOB_NAME, + UPTIME_DELIVERY_JOB_NAME, + UPTIME_DELIVERY_JOB_OPTIONS, + UPTIME_DELIVERY_QUEUE_NAME, UPTIME_JOB_OPTIONS, UPTIME_QUEUE_NAME, + uptimeDeliveryJobId, uptimeImmediateJobId, uptimeSchedulerId, } = await import("./uptime-queue"); @@ -71,15 +76,39 @@ describe("uptime queue", () => { expect(constructorCalls).toHaveLength(1); }); + it("constructs a separate durable delivery queue", () => { + const queue = getUptimeDeliveryQueue(); + + expect(queue).toBeInstanceOf(MockQueue); + expect(constructorCalls).toHaveLength(1); + expect(constructorCalls[0]).toEqual({ + name: UPTIME_DELIVERY_QUEUE_NAME, + options: { + connection: { + host: "queue.test", + port: 6381, + username: "queue-user", + password: "queue-pass", + db: 4, + maxRetriesPerRequest: 1, + }, + defaultJobOptions: UPTIME_DELIVERY_JOB_OPTIONS, + }, + }); + }); + it("closes and resets the singleton", async () => { const first = getUptimeQueue(); + const firstDelivery = getUptimeDeliveryQueue(); await closeUptimeQueue(); const second = getUptimeQueue(); + const secondDelivery = getUptimeDeliveryQueue(); - expect(closeCalls).toBe(1); + expect(closeCalls).toBe(2); expect(second).not.toBe(first); - expect(constructorCalls).toHaveLength(2); + expect(secondDelivery).not.toBe(firstDelivery); + expect(constructorCalls).toHaveLength(4); }); it("uses stable queue constants and namespaced ids", () => { @@ -88,9 +117,25 @@ describe("uptime queue", () => { expect(UPTIME_CHECK_JOB_NAME).toBe("uptime-check"); expect(UPTIME_QUEUE_NAME).toBe("uptime-checks"); + expect(UPTIME_DELIVERY_JOB_NAME).toBe("uptime-event-delivery"); + expect(UPTIME_DELIVERY_QUEUE_NAME).toBe("uptime-event-delivery"); expect(uptimeSchedulerId("schedule-1")).toBe("uptime-schedule-1"); + expect(uptimeDeliveryJobId("event-1")).toBe("uptime-delivery-event-1"); expect(first.startsWith("uptime-manual-schedule-1-")).toBe(true); expect(second.startsWith("uptime-manual-schedule-1-")).toBe(true); expect(first).not.toBe(second); }); + + it("keeps source and delivery payloads for retry instead of expiring failures", () => { + expect(UPTIME_JOB_OPTIONS).toMatchObject({ + attempts: 1_000_000, + backoff: { delay: 30_000, type: "fixed" }, + removeOnFail: false, + }); + expect(UPTIME_DELIVERY_JOB_OPTIONS).toMatchObject({ + attempts: 1_000_000, + backoff: { delay: 30_000, type: "fixed" }, + removeOnFail: false, + }); + }); }); diff --git a/packages/redis/uptime-queue.ts b/packages/redis/uptime-queue.ts index e398cd94a2..ec461e58ff 100644 --- a/packages/redis/uptime-queue.ts +++ b/packages/redis/uptime-queue.ts @@ -3,31 +3,60 @@ import { getBullMQConnectionOptions } from "./bullmq"; export const UPTIME_QUEUE_NAME = "uptime-checks"; export const UPTIME_CHECK_JOB_NAME = "uptime-check"; +export const UPTIME_DELIVERY_QUEUE_NAME = "uptime-event-delivery"; +export const UPTIME_DELIVERY_JOB_NAME = UPTIME_DELIVERY_QUEUE_NAME; export const UPTIME_JOB_TIMEOUT_MS = 30_000; - -export const UPTIME_JOB_OPTIONS = { - attempts: 3, +const UPTIME_RETRY_ATTEMPTS = 1_000_000; +const UPTIME_RETRY_DELAY_MS = 30_000; +const UPTIME_RETRY_OPTIONS = { + attempts: UPTIME_RETRY_ATTEMPTS, backoff: { - type: "exponential", - delay: 2000, + type: "fixed", + delay: UPTIME_RETRY_DELAY_MS, }, + removeOnFail: false, +}; + +/** + * Check jobs remain recoverable after an infrastructure outage. A target that + * is down produces an UptimeData result and completes normally, so these + * retries only cover transient processing or durable-handoff failures. + */ +export const UPTIME_JOB_OPTIONS = { + ...UPTIME_RETRY_OPTIONS, removeOnComplete: { age: 24 * 3600, count: 1000, }, - removeOnFail: { + // A failed job can contain the only durable copy of a completed probe. + // Retain it for inspection/re-drive instead of silently expiring it. +}; + +export const UPTIME_DELIVERY_JOB_OPTIONS = { + ...UPTIME_RETRY_OPTIONS, + // Keep completed IDs long enough to make a retry after an ambiguous queue + // add idempotent. Replaying after that is still safe because event_id stays + // stable through the analytics pipeline. + removeOnComplete: { age: 7 * 24 * 3600, - count: 5000, + count: 10_000, }, }; export interface UptimeCheckJobData { + // The uptime worker validates this JSON payload before replaying it. + delivery?: { event: unknown }; scheduleId: string; trigger: "manual" | "scheduled"; } +export interface UptimeDeliveryJobData { + event: unknown; +} + let uptimeQueue: Queue | null = null; +let uptimeDeliveryQueue: Queue | null = null; export function getUptimeQueue(): Queue { uptimeQueue ??= new Queue(UPTIME_QUEUE_NAME, { @@ -38,13 +67,23 @@ export function getUptimeQueue(): Queue { return uptimeQueue; } +export function getUptimeDeliveryQueue(): Queue { + uptimeDeliveryQueue ??= new Queue( + UPTIME_DELIVERY_QUEUE_NAME, + { + connection: getBullMQConnectionOptions(), + defaultJobOptions: UPTIME_DELIVERY_JOB_OPTIONS, + } + ); + + return uptimeDeliveryQueue; +} + export async function closeUptimeQueue(): Promise { - if (!uptimeQueue) { - return; - } - const queue = uptimeQueue; + const queues = [uptimeQueue, uptimeDeliveryQueue]; uptimeQueue = null; - await queue.close(); + uptimeDeliveryQueue = null; + await Promise.all(queues.map((queue) => queue?.close())); } export function uptimeSchedulerId(scheduleId: string): string { @@ -54,3 +93,7 @@ export function uptimeSchedulerId(scheduleId: string): string { export function uptimeImmediateJobId(scheduleId: string): string { return `uptime-manual-${scheduleId}-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; } + +export function uptimeDeliveryJobId(eventId: string): string { + return `uptime-delivery-${eventId}`; +} diff --git a/packages/rpc/src/services/uptime-scheduler.test.ts b/packages/rpc/src/services/uptime-scheduler.test.ts index f22416ea31..2f4e7a2a49 100644 --- a/packages/rpc/src/services/uptime-scheduler.test.ts +++ b/packages/rpc/src/services/uptime-scheduler.test.ts @@ -2,19 +2,16 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; const UPTIME_CHECK_JOB_NAME = "uptime-check"; const UPTIME_JOB_OPTIONS = { - attempts: 3, + attempts: 1_000_000, backoff: { - type: "exponential", - delay: 2000, + type: "fixed", + delay: 30_000, }, removeOnComplete: { age: 24 * 3600, count: 1000, }, - removeOnFail: { - age: 7 * 24 * 3600, - count: 5000, - }, + removeOnFail: false, }; const calls: { From 42ff5df5efa319c72c4caf5dc17b24d876d1ef8f Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:40:30 +0300 Subject: [PATCH 03/25] fix(analytics): deduplicate replayed uptime checks --- packages/ai/src/query/builders/uptime.ts | 71 +++++++++++++++----- packages/ai/src/query/simple-builder.test.ts | 40 +++++++++++ packages/rpc/src/routers/status-page.ts | 49 ++++++++++++-- 3 files changed, 141 insertions(+), 19 deletions(-) diff --git a/packages/ai/src/query/builders/uptime.ts b/packages/ai/src/query/builders/uptime.ts index 6cc581515d..ab0e0c5057 100644 --- a/packages/ai/src/query/builders/uptime.ts +++ b/packages/ai/src/query/builders/uptime.ts @@ -18,6 +18,47 @@ import type { SimpleQueryConfig } from "../types"; */ const UPTIME_TABLE = "uptime.uptime_monitor"; +const UPTIME_REPLAY_IDENTITY = [ + "site_id", + "url", + "timestamp", + "status", + "http_code", + "ttfb_ms", + "total_ms", + "attempt", + "retries", + "failure_streak", + "response_bytes", + "content_hash", + "redirect_count", + "probe_region", + "probe_ip", + "ssl_expiry", + "ssl_valid", + "env", + "check_type", + "user_agent", + "error", + "json_data", +].join(", "); + +/** + * A replay keeps the original check payload and timestamp. The deployed + * ClickHouse table deliberately remains migration-free, so reduce only exact + * duplicate payloads with fields already present in that schema before + * aggregates or window functions consume the source. + */ +function uptimeEventSource(scope: string): string { + return `( + SELECT * + FROM ${UPTIME_TABLE} + WHERE + ${scope} + ORDER BY timestamp DESC + LIMIT 1 BY ${UPTIME_REPLAY_IDENTITY} + )`; +} export const UptimeBuilders: Record = { uptime_overview: { @@ -44,11 +85,11 @@ export const UptimeBuilders: Record = { avg(ttfb_ms) as avg_ttfb, any(ssl_expiry) as ssl_expiry, min(ssl_valid) as ssl_valid - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} `, params: { websiteId, startDate, endDate }, }; @@ -125,11 +166,11 @@ export const UptimeBuilders: Record = { ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ) as next_ts - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= parseDateTimeBestEffort({startDate:String}, {timezone:String}) AND timestamp <= parseDateTimeBestEffort(concat({endDate:String}, ' 23:59:59'), {timezone:String}) + `)} ) GROUP BY date ) @@ -157,11 +198,11 @@ export const UptimeBuilders: Record = { http_code, COUNT(*) as count, round((COUNT(*) / sum(COUNT(*)) OVER ()) * 100, 2) as percentage - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} GROUP BY status, http_code ORDER BY count DESC `, @@ -195,11 +236,11 @@ export const UptimeBuilders: Record = { probe_ip, ssl_valid, error - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} ORDER BY timestamp DESC LIMIT {limit:UInt32} OFFSET {offset:UInt32} @@ -242,12 +283,12 @@ export const UptimeBuilders: Record = { min(total_ms) as min_response_time, max(total_ms) as max_response_time, avg(ttfb_ms) as avg_ttfb - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) - AND status = 1 + `)} + WHERE status = 1 GROUP BY date ORDER BY date ASC `, @@ -277,11 +318,11 @@ export const UptimeBuilders: Record = { argMax(ssl_expiry, timestamp) as latest_ssl_expiry, argMax(ssl_valid, timestamp) as latest_ssl_valid, countIf(ssl_valid = 0) as invalid_ssl_checks - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} GROUP BY site_id ) `, @@ -308,11 +349,11 @@ export const UptimeBuilders: Record = { if((countIf(status = 1) + countIf(status = 0)) = 0, 0, round((countIf(status = 1) / (countIf(status = 1) + countIf(status = 0))) * 100, 2)) as uptime_percentage, avg(total_ms) as avg_response_time, quantileTDigest(0.95)(total_ms) as p95_response_time - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id = {websiteId:String} AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} GROUP BY probe_region ORDER BY uptime_percentage DESC `, diff --git a/packages/ai/src/query/simple-builder.test.ts b/packages/ai/src/query/simple-builder.test.ts index 3c94e15f0c..6b8f7b1bd7 100644 --- a/packages/ai/src/query/simple-builder.test.ts +++ b/packages/ai/src/query/simple-builder.test.ts @@ -1029,6 +1029,46 @@ describe("SimpleQueryBuilder.compile", () => { expect(sql).not.toContain("event_name = 'pageview'"); }); + it.each([ + "uptime_overview", + "uptime_time_series", + "uptime_status_breakdown", + "uptime_recent_checks", + "uptime_response_time_trends", + "uptime_ssl_status", + "uptime_by_region", + ])( + "deduplicates replayed uptime deliveries in %s using existing check fields", + (type) => { + const config = QueryBuilders[type]; + if (!config) { + throw new Error(`${type} builder is missing`); + } + + const { sql } = compileBuilder(type, config); + + expect(sql).toContain( + "LIMIT 1 BY site_id, url, timestamp, status, http_code, ttfb_ms, total_ms" + ); + expect(sql).toContain("user_agent, error, json_data"); + expect(sql).not.toContain("event_id"); + expect(sql.match(/FROM uptime\.uptime_monitor/g)).toHaveLength(1); + } + ); + + it("deduplicates uptime events before the downtime window function", () => { + const config = QueryBuilders.uptime_time_series; + if (!config) { + throw new Error("uptime_time_series builder is missing"); + } + + const { sql } = compileBuilder("uptime_time_series", config); + + expect(sql).toMatch( + /leadInFrame\(timestamp, 1, now\(\)\)[\s\S]*?FROM \([\s\S]*?LIMIT 1 BY site_id, url, timestamp, status, http_code, ttfb_ms, total_ms/ + ); + }); + it("counts only page-view sessions in session metrics", () => { const config = QueryBuilders.session_metrics; if (!config) { diff --git a/packages/rpc/src/routers/status-page.ts b/packages/rpc/src/routers/status-page.ts index 5dbc162727..59c471beb5 100644 --- a/packages/rpc/src/routers/status-page.ts +++ b/packages/rpc/src/routers/status-page.ts @@ -33,6 +33,46 @@ import { } from "./status-page-health"; const UPTIME_TABLE = "uptime.uptime_monitor"; +const UPTIME_REPLAY_IDENTITY = [ + "site_id", + "url", + "timestamp", + "status", + "http_code", + "ttfb_ms", + "total_ms", + "attempt", + "retries", + "failure_streak", + "response_bytes", + "content_hash", + "redirect_count", + "probe_region", + "probe_ip", + "ssl_expiry", + "ssl_valid", + "env", + "check_type", + "user_agent", + "error", + "json_data", +].join(", "); + +/** + * Replaying a persisted delivery preserves its original check payload and timestamp. + * Deduplicate exact physical payloads using only existing ClickHouse columns + * before status-page aggregates and windows consume the source. + */ +function uptimeEventSource(scope: string): string { + return `( + SELECT * + FROM ${UPTIME_TABLE} + WHERE + ${scope} + ORDER BY timestamp DESC + LIMIT 1 BY ${UPTIME_REPLAY_IDENTITY} + )`; +} const DAILY_UPTIME_SQL = `SELECT site_id, @@ -66,11 +106,11 @@ const DAILY_UPTIME_SQL = `SELECT ORDER BY timestamp ASC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ) as next_ts - FROM ${UPTIME_TABLE} - WHERE + FROM ${uptimeEventSource(` site_id IN ({siteIds:Array(String)}) AND timestamp >= toDateTime({startDate:String}) AND timestamp <= toDateTime(concat({endDate:String}, ' 23:59:59')) + `)} ) GROUP BY site_id, date ) @@ -81,9 +121,10 @@ const LATEST_CHECK_SQL = `SELECT max(timestamp) as last_timestamp, argMax(status, timestamp) as last_status, argMax(http_code, timestamp) as last_http_code - FROM ${UPTIME_TABLE} - WHERE site_id IN ({siteIds:Array(String)}) + FROM ${uptimeEventSource(` + site_id IN ({siteIds:Array(String)}) AND timestamp >= now() - INTERVAL 7 DAY + `)} GROUP BY site_id`; const dailyUptimeSchema = z.object({ From cc1febcb7e9a99196546b04a1ef427b009912134 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:54:32 +0300 Subject: [PATCH 04/25] fix(basket): durably admit core analytics events --- apps/basket/src/index.ts | 37 +- .../src/lib/event-service.delivery.test.ts | 224 ++++++ apps/basket/src/lib/event-service.test.ts | 23 +- apps/basket/src/lib/event-service.ts | 224 +++--- apps/basket/src/lib/producer.delivery.test.ts | 224 ++++++ apps/basket/src/lib/producer.kafka.test.ts | 6 +- apps/basket/src/lib/producer.test.ts | 30 +- apps/basket/src/lib/producer.ts | 662 ++++++++++++------ apps/basket/src/lib/security.test.ts | 210 ++++-- apps/basket/src/lib/security.ts | 150 +++- apps/basket/src/routes/basket.ts | 22 +- apps/basket/src/routes/integration.test.ts | 6 +- apps/basket/src/utils/pixel.test.ts | 7 + apps/basket/src/utils/pixel.ts | 5 +- packages/tracker/src/plugins/pixel.ts | 88 ++- packages/tracker/tests/unit/pixel.test.ts | 87 +++ 16 files changed, 1596 insertions(+), 409 deletions(-) create mode 100644 apps/basket/src/lib/event-service.delivery.test.ts create mode 100644 apps/basket/src/lib/producer.delivery.test.ts create mode 100644 packages/tracker/tests/unit/pixel.test.ts diff --git a/apps/basket/src/index.ts b/apps/basket/src/index.ts index 2de440da5c..c89df0b823 100644 --- a/apps/basket/src/index.ts +++ b/apps/basket/src/index.ts @@ -8,7 +8,12 @@ import { import { shutdownPostgres } from "@databuddy/db"; import { clickHouse } from "@databuddy/db/clickhouse"; import { getRedisCache } from "@databuddy/redis/redis"; -import { disconnect, disposeRuntime, runPromise } from "@lib/producer"; +import { + disconnect, + disposeRuntime, + runPromise, + ShutdownDrainError, +} from "@lib/producer"; import { Kafka } from "kafkajs"; import { databuddyEvlogRedaction } from "@databuddy/shared/evlog-redaction"; import { @@ -79,13 +84,37 @@ async function gracefulShutdown(signal: string, exitCode = 0) { error_message: error instanceof Error ? error.message : String(error), }); const { shutdownRedis } = await import("@databuddy/redis"); + // A buffered fallback can still need ClickHouse after Kafka is unavailable. + // Drain it before flushing its failure telemetry or tearing down services. + try { + await runPromise(disconnect); + } catch (error) { + finalExitCode = 1; + if (error instanceof ShutdownDrainError) { + log.error({ + lifecycle: "producerDrain", + error_message: + "Basket producer drain timed out with undelivered events", + in_flight: error.inFlight, + residual_count: error.residualCount, + drain_timeout_ms: error.deadlineMs, + }); + } else { + logErr("producerDrain")(error); + } + } finally { + try { + await disposeRuntime(); + } catch (error) { + finalExitCode = 1; + logErr("runtimeDispose")(error); + } + } await Promise.all([ shutdownRedis().catch(logErr("redisShutdown")), shutdownPostgres().catch(logErr("postgresShutdown")), - flushBatchedAxiomDrain().catch(logErr("drainFlush")), - runPromise(disconnect).catch(logErr("shutdown")), - disposeRuntime().catch(logErr("runtimeDispose")), ]); + await flushBatchedAxiomDrain().catch(logErr("drainFlush")); closeGeoIPReader(); } catch (error) { finalExitCode = 1; diff --git a/apps/basket/src/lib/event-service.delivery.test.ts b/apps/basket/src/lib/event-service.delivery.test.ts new file mode 100644 index 0000000000..afb037246b --- /dev/null +++ b/apps/basket/src/lib/event-service.delivery.test.ts @@ -0,0 +1,224 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const { + mockApplyVisitorIdPrivacy, + mockGetDailySalt, + mockGetGeo, + mockLoggerSet, + mockMarkDuplicateReservationDelivered, + mockParseUserAgent, + mockReleaseDuplicateReservation, + mockReserveDuplicate, + mockRunPromise, + mockSend, + mockShouldAnonymizeVisitorIds, +} = vi.hoisted(() => ({ + mockApplyVisitorIdPrivacy: vi.fn((value: unknown) => String(value ?? "")), + mockGetDailySalt: vi.fn(() => Promise.resolve("daily-salt")), + mockGetGeo: vi.fn(() => + Promise.resolve({ + anonymizedIP: "anonymous-ip", + city: "San Francisco", + country: "US", + region: "CA", + }) + ), + mockLoggerSet: vi.fn(), + mockMarkDuplicateReservationDelivered: vi.fn(() => Promise.resolve()), + mockParseUserAgent: vi.fn(() => Promise.resolve({ browserName: "Chrome" })), + mockReleaseDuplicateReservation: vi.fn(() => Promise.resolve()), + mockReserveDuplicate: vi.fn(() => + Promise.resolve({ + duplicate: false, + key: "dedup:track:evt_1", + token: "reservation-token", + }) + ), + mockRunPromise: vi.fn(() => Promise.resolve()), + mockSend: vi.fn(() => ({ _tag: "MockEffect" })), + mockShouldAnonymizeVisitorIds: vi.fn(() => false), +})); + +vi.mock("@lib/producer", () => ({ + runPromise: mockRunPromise, + send: mockSend, + sendBatch: vi.fn(() => ({ _tag: "MockEffect" })), +})); + +vi.mock("@lib/security", () => ({ + applyVisitorIdPrivacy: mockApplyVisitorIdPrivacy, + getDailySalt: mockGetDailySalt, + markDuplicateReservationDelivered: mockMarkDuplicateReservationDelivered, + releaseDuplicateReservation: mockReleaseDuplicateReservation, + reserveDuplicate: mockReserveDuplicate, + shouldAnonymizeVisitorIds: mockShouldAnonymizeVisitorIds, +})); + +vi.mock("@lib/tracing", () => ({ + record: (_name: string, operation: () => Promise) => operation(), +})); + +vi.mock("@utils/ip-geo", () => ({ + extractTrustedClientIp: vi.fn(() => undefined), + getGeo: mockGetGeo, +})); + +vi.mock("@utils/user-agent", () => ({ + parseUserAgent: mockParseUserAgent, +})); + +vi.mock("evlog/elysia", () => ({ + useLogger: () => ({ set: mockLoggerSet }), +})); + +const { insertOutgoingLink, insertTrackEvent, stableAnalyticsEventId } = + await import("./event-service"); + +const request = new Request("http://localhost/ingest"); +const trackReservation = { + duplicate: false, + key: "dedup:track:evt_1", + ttl: 86_400, +}; + +describe("core event delivery", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetGeo.mockResolvedValue({ + anonymizedIP: "anonymous-ip", + city: "San Francisco", + country: "US", + region: "CA", + }); + mockParseUserAgent.mockResolvedValue({ browserName: "Chrome" }); + mockReserveDuplicate.mockResolvedValue(trackReservation); + mockRunPromise.mockResolvedValue(undefined); + mockSend.mockReturnValue({ _tag: "MockEffect" }); + mockShouldAnonymizeVisitorIds.mockReturnValue(false); + }); + + test("awaits durable delivery for core track events", async () => { + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(mockSend).toHaveBeenCalledWith( + "analytics-events", + expect.objectContaining({ client_id: "ws_test", event_name: "pageview" }) + ); + expect(mockRunPromise).toHaveBeenCalledWith(mockSend.mock.results[0]?.value); + expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( + trackReservation + ); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("releases a track reservation and returns retryable 503 when delivery fails", async () => { + const deliveryFailure = new Error("ClickHouse unavailable"); + mockRunPromise.mockRejectedValueOnce(deliveryFailure); + + await expect( + insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ) + ).rejects.toMatchObject({ + message: "Analytics delivery temporarily unavailable", + status: 503, + }); + + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("reuses the same ClickHouse UUID when a client retries a failed track event", async () => { + const deliveredEvents: Array<{ id: string }> = []; + mockSend.mockImplementation((_topic, event) => { + deliveredEvents.push(event as { id: string }); + return { _tag: "MockEffect" }; + }); + mockRunPromise + .mockRejectedValueOnce(new Error("ambiguous delivery")) + .mockResolvedValueOnce(undefined); + + await expect( + insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_retry", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_retry", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(deliveredEvents.map((event) => event.id)).toEqual([ + stableAnalyticsEventId("ws_test", "track", "evt_retry"), + stableAnalyticsEventId("ws_test", "track", "evt_retry"), + ]); + expect(mockReserveDuplicate).toHaveBeenCalledWith( + stableAnalyticsEventId("ws_test", "track", "evt_retry"), + "track", + "evt_retry" + ); + }); + + test("awaits outgoing-link delivery and releases its reservation on failure", async () => { + const reservation = { + duplicate: false, + key: "dedup:outgoing_link:evt_link_1", + ttl: 86_400, + }; + mockReserveDuplicate.mockResolvedValueOnce(reservation); + mockRunPromise.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + + await expect( + insertOutgoingLink( + { + anonymousId: "anon_1", + eventId: "evt_link_1", + href: "https://example.com/pricing", + }, + "ws_test", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSend).toHaveBeenCalledWith( + "analytics-outgoing-links", + expect.objectContaining({ client_id: "ws_test" }) + ); + expect(mockRunPromise).toHaveBeenCalledWith(mockSend.mock.results[0]?.value); + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith(reservation); + }); + + test("does not send a duplicate core event", async () => { + mockReserveDuplicate.mockResolvedValueOnce({ duplicate: true }); + + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(mockSend).not.toHaveBeenCalled(); + expect(mockRunPromise).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/basket/src/lib/event-service.test.ts b/apps/basket/src/lib/event-service.test.ts index 353fe536cd..7cdec9c70d 100644 --- a/apps/basket/src/lib/event-service.test.ts +++ b/apps/basket/src/lib/event-service.test.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from "vitest"; import { CONTROL_CHARS, longString, XSS_PAYLOADS } from "../test-helpers"; -import { buildTrackEvent, type TrackEventContext } from "./event-service"; +import { + buildTrackEvent, + stableAnalyticsEventId, + type TrackEventContext, +} from "./event-service"; // ── Fixtures ── @@ -72,7 +76,7 @@ describe("buildTrackEvent — field mapping", () => { const result = buildTrackEvent(fullTrackData, fullCtx); // Identity - expect(result.id).toBeTruthy(); // randomUUIDv7 + expect(result.id).toBe(stableAnalyticsEventId("ws_test", "track", "evt_123")); expect(result.client_id).toBe("ws_test"); // Names & content @@ -135,6 +139,21 @@ describe("buildTrackEvent — field mapping", () => { expect(result.properties).toBe('{"plan":"pro","color":"blue"}'); }); + test("uses a stable UUID for the same client retry", () => { + const first = buildTrackEvent(fullTrackData, fullCtx); + const retry = buildTrackEvent(fullTrackData, fullCtx); + const otherClient = buildTrackEvent(fullTrackData, { + ...fullCtx, + clientId: "ws_other", + }); + + expect(first.id).toBe(retry.id); + expect(first.id).not.toBe(otherClient.id); + expect(first.id).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ); + }); + test("minimal input → defaults applied", () => { const result = buildTrackEvent({ name: "click" }, fullCtx); diff --git a/apps/basket/src/lib/event-service.ts b/apps/basket/src/lib/event-service.ts index d4da14945f..d28d79e30d 100644 --- a/apps/basket/src/lib/event-service.ts +++ b/apps/basket/src/lib/event-service.ts @@ -5,11 +5,13 @@ import type { WebVitalsSpansInsert, } from "@databuddy/db/clickhouse/tables"; import type { ErrorSpan, IndividualVital } from "@databuddy/validation"; -import { runFork, runPromise, send, sendBatch } from "@lib/producer"; +import { runPromise, send, sendBatch } from "@lib/producer"; import { - checkDuplicate, getDailySalt, applyVisitorIdPrivacy, + markDuplicateReservationDelivered, + releaseDuplicateReservation, + reserveDuplicate, shouldAnonymizeVisitorIds, } from "@lib/security"; import { record } from "@lib/tracing"; @@ -23,7 +25,9 @@ import { validateSessionId, } from "@utils/validation"; import { randomUUIDv7 } from "bun"; +import { createError } from "evlog"; import { useLogger } from "evlog/elysia"; +import { createHash } from "node:crypto"; export interface TrackEventContext { anonymousId: string; @@ -47,6 +51,36 @@ export interface TrackEventContext { }; } +function deliveryUnavailable(cause: unknown) { + return createError({ + code: "basket.DELIVERY_UNAVAILABLE", + message: "Analytics delivery temporarily unavailable", + status: 503, + why: "Databuddy could not durably accept the event.", + fix: "Retry the same event after a short delay.", + cause: cause instanceof Error ? cause : new Error(String(cause)), + }); +} + +/** + * ClickHouse stores analytics ids as UUIDs, while public client event ids can + * be arbitrary strings. Derive a valid, stable UUID so the same client retry + * preserves its physical identity without accepting arbitrary input as UUID. + */ +export function stableAnalyticsEventId( + clientId: string, + eventType: "outgoing_link" | "track", + eventId: string +): string { + const digest = createHash("sha256") + .update(JSON.stringify([clientId, eventType, eventId])) + .digest("hex"); + const variant = "89ab".charAt(Number.parseInt(digest.charAt(16), 16) % 4); + const uuid = `${digest.slice(0, 12)}5${digest.slice(13, 16)}${variant}${digest.slice(17, 32)}`; + + return `${uuid.slice(0, 8)}-${uuid.slice(8, 12)}-${uuid.slice(12, 16)}-${uuid.slice(16, 20)}-${uuid.slice(20)}`; +} + export function buildTrackEvent( trackData: any, ctx: TrackEventContext @@ -55,7 +89,7 @@ export function buildTrackEvent( typeof trackData.timestamp === "number" ? trackData.timestamp : ctx.now; return { - id: randomUUIDv7(), + id: stableAnalyticsEventId(ctx.clientId, "track", ctx.eventId), client_id: ctx.clientId, event_name: sanitizeString( trackData.name, @@ -130,54 +164,58 @@ export function insertTrackEvent( eventId = randomUUIDv7(); } - const [isDuplicate, geoData] = await Promise.all([ - checkDuplicate(eventId, "track"), - getGeo(ip, request), - ]); - - if (isDuplicate) { + const deliveryId = stableAnalyticsEventId(clientId, "track", eventId); + const reservation = await reserveDuplicate(deliveryId, "track", eventId); + if (reservation.duplicate) { return; } - const trustedCountry = extractTrustedClientIp(request) - ? geoData.country - : undefined; - const anonymizeVisitorIds = shouldAnonymizeVisitorIds( - trackData.anonymizeVisitorIds, - trustedCountry - ); - const [salt, ua] = await Promise.all([ - anonymizeVisitorIds ? getDailySalt() : Promise.resolve(undefined), - parseUserAgent(userAgent), - ]); - - log.set({ - event: { id: eventId, name: trackData.name, path: trackData.path }, - geo: { - country: geoData.country, - region: geoData.region, - city: geoData.city, - }, - }); - - const anonymousId = applyVisitorIdPrivacy( - trackData.anonymousId, - anonymizeVisitorIds, - salt - ); - - const now = Date.now(); - - const trackEvent = buildTrackEvent(trackData, { - clientId, - eventId, - anonymousId, - geo: geoData, - ua, - now, - }); - - runFork(send("analytics-events", trackEvent)); + try { + const geoData = await getGeo(ip, request); + const trustedCountry = extractTrustedClientIp(request) + ? geoData.country + : undefined; + const anonymizeVisitorIds = shouldAnonymizeVisitorIds( + trackData.anonymizeVisitorIds, + trustedCountry + ); + const [salt, ua] = await Promise.all([ + anonymizeVisitorIds ? getDailySalt() : Promise.resolve(undefined), + parseUserAgent(userAgent), + ]); + + log.set({ + event: { id: eventId, name: trackData.name, path: trackData.path }, + geo: { + country: geoData.country, + region: geoData.region, + city: geoData.city, + }, + }); + + const anonymousId = applyVisitorIdPrivacy( + trackData.anonymousId, + anonymizeVisitorIds, + salt + ); + + const now = Date.now(); + + const trackEvent = buildTrackEvent(trackData, { + clientId, + eventId, + anonymousId, + geo: geoData, + ua, + now, + }); + + await runPromise(send("analytics-events", trackEvent)); + await markDuplicateReservationDelivered(reservation); + } catch (error) { + await releaseDuplicateReservation(reservation); + throw deliveryUnavailable(error); + } }); } @@ -197,46 +235,62 @@ export function insertOutgoingLink( eventId = randomUUIDv7(); } - if (await checkDuplicate(eventId, "outgoing_link")) { + const deliveryId = stableAnalyticsEventId( + clientId, + "outgoing_link", + eventId + ); + const reservation = await reserveDuplicate( + deliveryId, + "outgoing_link", + eventId + ); + if (reservation.duplicate) { return; } - log.set({ - event: { id: eventId, type: "outgoing_link", href: linkData.href }, - }); - - const now = Date.now(); - - const trustedIp = extractTrustedClientIp(request); - const visitorCountry = - linkData.anonymizeVisitorIds === "auto" && trustedIp - ? (await getGeo(trustedIp, request)).country - : undefined; - const anonymizeVisitorIds = shouldAnonymizeVisitorIds( - linkData.anonymizeVisitorIds, - visitorCountry - ); - const salt = anonymizeVisitorIds ? await getDailySalt() : undefined; - - const outgoingLinkEvent: OutgoingLinksInsert = { - id: randomUUIDv7(), - client_id: clientId, - anonymous_id: applyVisitorIdPrivacy( - linkData.anonymousId, - anonymizeVisitorIds, - salt - ), - session_id: validateSessionId(linkData.sessionId), - href: sanitizeUrl(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), - text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), - properties: linkData.properties - ? JSON.stringify(linkData.properties) - : "{}", - timestamp: - typeof linkData.timestamp === "number" ? linkData.timestamp : now, - }; - - runFork(send("analytics-outgoing-links", outgoingLinkEvent)); + try { + log.set({ + event: { id: eventId, type: "outgoing_link", href: linkData.href }, + }); + + const now = Date.now(); + + const trustedIp = extractTrustedClientIp(request); + const visitorCountry = + linkData.anonymizeVisitorIds === "auto" && trustedIp + ? (await getGeo(trustedIp, request)).country + : undefined; + const anonymizeVisitorIds = shouldAnonymizeVisitorIds( + linkData.anonymizeVisitorIds, + visitorCountry + ); + const salt = anonymizeVisitorIds ? await getDailySalt() : undefined; + + const outgoingLinkEvent: OutgoingLinksInsert = { + id: deliveryId, + client_id: clientId, + anonymous_id: applyVisitorIdPrivacy( + linkData.anonymousId, + anonymizeVisitorIds, + salt + ), + session_id: validateSessionId(linkData.sessionId), + href: sanitizeUrl(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), + text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), + properties: linkData.properties + ? JSON.stringify(linkData.properties) + : "{}", + timestamp: + typeof linkData.timestamp === "number" ? linkData.timestamp : now, + }; + + await runPromise(send("analytics-outgoing-links", outgoingLinkEvent)); + await markDuplicateReservationDelivered(reservation); + } catch (error) { + await releaseDuplicateReservation(reservation); + throw deliveryUnavailable(error); + } }); } diff --git a/apps/basket/src/lib/producer.delivery.test.ts b/apps/basket/src/lib/producer.delivery.test.ts new file mode 100644 index 0000000000..554ecc1c39 --- /dev/null +++ b/apps/basket/src/lib/producer.delivery.test.ts @@ -0,0 +1,224 @@ +import type { ClickHouseClient } from "@clickhouse/client"; +import { Effect } from "effect"; +import { describe, expect, test, vi } from "vitest"; +import type { ProducerConfig } from "./producer"; + +const { mockCaptureError } = vi.hoisted(() => ({ + mockCaptureError: vi.fn(), +})); + +vi.mock("@databuddy/db/clickhouse", () => ({ + clickHouse: {}, + TABLE_NAMES: { + ai_traffic_spans: "analytics.ai_traffic_spans", + blocked_traffic: "analytics.blocked_traffic", + custom_events: "analytics.custom_events", + error_spans: "analytics.error_spans", + events: "analytics.events", + link_visits: "analytics.link_visits", + outgoing_links: "analytics.outgoing_links", + web_vitals_spans: "analytics.web_vitals_spans", + }, +})); + +vi.mock("@lib/tracing", () => ({ + captureError: mockCaptureError, + record: (_name: string, fn: () => Promise) => fn(), +})); + +const { createProducerEffects } = await import("./producer"); + +const topicMap = { "analytics-events": "analytics.events" }; + +const baseConfig: ProducerConfig = { + bufferHardMax: 2, + bufferInterval: 60_000, + bufferMax: 1, + broker: undefined, + chunkSize: 100, + kafkaTimeout: 1_000, + maxProducerRetries: 0, + password: undefined, + producerRetryDelay: 1, + reconnectCooldown: 1, + selfHost: true, + shutdownDrainRetryDelay: 1, + shutdownDrainTimeout: 50, + username: undefined, +}; + +const event = (id: string) => ({ + client_id: "ws_1", + event_id: id, + timestamp: 1, +}); + +async function makeEffects( + insert: (input: unknown) => Promise, + config: Partial = {} +) { + return Effect.runPromise( + createProducerEffects( + { ...baseConfig, ...config }, + null, + { insert } as unknown as ClickHouseClient, + topicMap + ) + ); +} + +describe("producer delivery guarantees", () => { + test("resolves a core send only after direct ClickHouse fallback succeeds", async () => { + const insert = vi.fn(() => Promise.resolve()); + const effects = await makeEffects(insert); + + await Effect.runPromise(effects.sendOne("analytics-events", event("event_1"))); + + expect(insert).toHaveBeenCalledWith( + expect.objectContaining({ + format: "JSONEachRow", + table: "analytics.events", + values: [event("event_1")], + }) + ); + const stats = await Effect.runPromise(effects.stats); + expect(stats).toMatchObject({ bufferSize: 0, flushed: 1 }); + }); + + test("returns a retryable error instead of acknowledging a failed direct fallback", async () => { + const effects = await makeEffects(() => Promise.reject(new Error("offline"))); + + await expect( + Effect.runPromise(effects.sendOne("analytics-events", event("event_1"))) + ).rejects.toMatchObject({ + _tag: "ClickHouseFallbackError", + retryable: true, + table: "analytics.events", + topic: "analytics-events", + }); + + const stats = await Effect.runPromise(effects.stats); + expect(stats).toMatchObject({ bufferSize: 0, errors: 1 }); + }); + + test("rejects buffered delivery atomically when the in-memory buffer is full", async () => { + const effects = await makeEffects(() => Promise.resolve()); + + await Effect.runPromise( + effects.sendManyBuffered("analytics-events", [event("event_1"), event("event_2")]) + ); + await expect( + Effect.runPromise( + effects.sendOneBuffered("analytics-events", event("event_3")) + ) + ).rejects.toMatchObject({ + _tag: "BufferOverflowError", + bufferHardMax: 2, + bufferLength: 2, + eventCount: 1, + retryable: true, + }); + + const stats = await Effect.runPromise(effects.stats); + expect(stats).toMatchObject({ bufferSize: 2, dropped: 0 }); + }); + + test("drains every buffered event before shutdown completes", async () => { + const insert = vi.fn(() => Promise.resolve()); + const effects = await makeEffects(insert, { bufferHardMax: 3 }); + + await Effect.runPromise( + effects.sendManyBuffered("analytics-events", [ + event("event_1"), + event("event_2"), + event("event_3"), + ]) + ); + await Effect.runPromise(effects.shutDown); + + expect(insert).toHaveBeenCalledTimes(3); + const stats = await Effect.runPromise(effects.stats); + expect(stats).toMatchObject({ bufferSize: 0, flushed: 3 }); + }); + + test("rejects new events with a retryable error once shutdown begins", async () => { + let releaseInsert: (() => void) | undefined; + const effects = await makeEffects( + () => + new Promise((resolve) => { + releaseInsert = resolve; + }), + { shutdownDrainTimeout: 500 } + ); + + await Effect.runPromise( + effects.sendOneBuffered("analytics-events", event("event_1")) + ); + const shutdown = Effect.runPromise(effects.shutDown); + await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); + + await expect( + Effect.runPromise( + effects.sendOneBuffered("analytics-events", event("event_2")) + ) + ).rejects.toMatchObject({ + _tag: "ProducerShuttingDownError", + eventCount: 1, + retryable: true, + }); + + releaseInsert?.(); + await shutdown; + }); + + test("keeps an active direct delivery counted when shutdown rejects another send", async () => { + let releaseInsert: (() => void) | undefined; + const effects = await makeEffects( + () => + new Promise((resolve) => { + releaseInsert = resolve; + }), + { shutdownDrainTimeout: 500 } + ); + + const activeDelivery = Effect.runPromise( + effects.sendOne("analytics-events", event("event_1")) + ); + await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); + + const shutdown = Effect.runPromise(effects.shutDown); + await expect( + Effect.runPromise(effects.sendOne("analytics-events", event("event_2"))) + ).rejects.toMatchObject({ + _tag: "ProducerShuttingDownError", + retryable: true, + }); + + const stats = await Effect.runPromise(effects.stats); + expect(stats.inFlight).toBe(1); + + releaseInsert?.(); + await activeDelivery; + await shutdown; + }); + + test("reports retained residual events when shutdown cannot drain before its deadline", async () => { + const effects = await makeEffects( + () => Promise.reject(new Error("ClickHouse unavailable")), + { shutdownDrainTimeout: 20 } + ); + + await Effect.runPromise( + effects.sendOneBuffered("analytics-events", event("event_1")) + ); + await expect(Effect.runPromise(effects.shutDown)).rejects.toMatchObject({ + _tag: "ShutdownDrainError", + inFlight: 0, + residualCount: 1, + retryable: true, + }); + + const stats = await Effect.runPromise(effects.stats); + expect(stats.bufferSize).toBe(1); + }); +}); diff --git a/apps/basket/src/lib/producer.kafka.test.ts b/apps/basket/src/lib/producer.kafka.test.ts index 2509b93ff4..13c32bb972 100644 --- a/apps/basket/src/lib/producer.kafka.test.ts +++ b/apps/basket/src/lib/producer.kafka.test.ts @@ -93,7 +93,7 @@ afterAll(async () => { }); describe("producer Kafka send failure handling", () => { - test("backs off after a send failure and still disconnects on shutdown", async () => { + test("falls back to direct ClickHouse writes and still disconnects on shutdown", async () => { await runPromise( send("analytics-events", { client_id: "ws_1", @@ -116,7 +116,9 @@ describe("producer Kafka send failure handling", () => { expect(mockProducer).toHaveBeenCalledTimes(1); expect(mockConnect).toHaveBeenCalledTimes(1); expect(mockSend).toHaveBeenCalledTimes(1); - expect(stats?.bufferSize).toBe(2); + expect(mockClickHouseInsert).toHaveBeenCalledTimes(2); + expect(stats?.bufferSize).toBe(0); + expect(stats?.flushed).toBe(2); expect(stats?.connected).toBe(false); expect(stats?.failed).toBe(true); expect(stats?.failedCount).toBe(1); diff --git a/apps/basket/src/lib/producer.test.ts b/apps/basket/src/lib/producer.test.ts index 12358ea996..8ad1747984 100644 --- a/apps/basket/src/lib/producer.test.ts +++ b/apps/basket/src/lib/producer.test.ts @@ -29,7 +29,13 @@ vi.mock("@lib/tracing", () => ({ record: (_name: string, fn: Function) => Promise.resolve().then(() => fn()), })); -const { disposeRuntime, getStats, runPromise, send } = await import("./producer"); +const { + disposeRuntime, + getStats, + runPromise, + send, + sendBuffered, +} = await import("./producer"); beforeEach(async () => { mockCaptureError.mockClear(); @@ -48,7 +54,7 @@ afterAll(async () => { describe("producer fallback topics", () => { test("blocked traffic is buffered for ClickHouse fallback", async () => { await runPromise( - send("analytics-blocked-traffic", { + sendBuffered("analytics-blocked-traffic", { id: "blocked_1", client_id: "ws_1", timestamp: Date.now(), @@ -61,13 +67,19 @@ describe("producer fallback topics", () => { }); test("unknown topics include the missing topic in error context", async () => { - await runPromise( - send("analytics-unmapped-topic", { - id: "evt_1", - client_id: "ws_1", - timestamp: Date.now(), - }) - ); + await expect( + runPromise( + send("analytics-unmapped-topic", { + id: "evt_1", + client_id: "ws_1", + timestamp: Date.now(), + }) + ) + ).rejects.toMatchObject({ + _tag: "UnknownKafkaTopicError", + retryable: false, + topic: "analytics-unmapped-topic", + }); const stats = await runPromise(getStats); expect(stats?.errors).toBe(1); diff --git a/apps/basket/src/lib/producer.ts b/apps/basket/src/lib/producer.ts index 5605dc0653..b681624b58 100644 --- a/apps/basket/src/lib/producer.ts +++ b/apps/basket/src/lib/producer.ts @@ -21,17 +21,56 @@ export class KafkaSendError extends Data.TaggedError("KafkaSendError")<{ }> {} export class BufferOverflowError extends Data.TaggedError( "BufferOverflowError" -)<{ readonly bufferLength: number }> {} +)<{ + readonly bufferHardMax: number; + readonly bufferLength: number; + readonly eventCount: number; + readonly retryable: true; + readonly topic: string; +}> {} +export class ProducerShuttingDownError extends Data.TaggedError( + "ProducerShuttingDownError" +)<{ + readonly eventCount: number; + readonly retryable: true; +}> {} +export class ProducerUnavailableError extends Data.TaggedError( + "ProducerUnavailableError" +)<{ readonly retryable: true }> {} +export class UnknownKafkaTopicError extends Data.TaggedError( + "UnknownKafkaTopicError" +)<{ + readonly retryable: false; + readonly topic: string; +}> {} +export class ClickHouseFallbackError extends Data.TaggedError( + "ClickHouseFallbackError" +)<{ + readonly cause?: Error; + readonly retryable: true; + readonly table: string; + readonly topic: string; +}> {} export class FlushError extends Data.TaggedError("FlushError")<{ readonly table: string; readonly cause?: Error; }> {} +export class ShutdownDrainError extends Data.TaggedError("ShutdownDrainError")<{ + readonly deadlineMs: number; + readonly inFlight: number; + readonly residualCount: number; + readonly retryable: true; +}> {} export type ProducerError = | KafkaConnectionError | KafkaSendError | BufferOverflowError - | FlushError; + | FlushError + | ProducerShuttingDownError + | ProducerUnavailableError + | UnknownKafkaTopicError + | ClickHouseFallbackError; interface BufferedEvent { event: unknown; @@ -48,6 +87,7 @@ interface ProducerState { failedCount: number; flushed: number; flushing: boolean; + inFlight: number; lastErrorTime: number | null; lastRetry: number; producerInitialized: boolean; @@ -55,7 +95,7 @@ interface ProducerState { shuttingDown: boolean; } -interface ProducerConfig { +export interface ProducerConfig { broker?: string; bufferHardMax: number; bufferInterval: number; @@ -67,9 +107,35 @@ interface ProducerConfig { producerRetryDelay: number; reconnectCooldown: number; selfHost: boolean; + shutdownDrainRetryDelay: number; + shutdownDrainTimeout: number; username?: string; } +export interface ProducerEffects { + flush: Effect.Effect; + sendMany: ( + topic: string, + events: unknown[] + ) => Effect.Effect; + sendManyBuffered: ( + topic: string, + events: unknown[] + ) => Effect.Effect; + sendOne: ( + topic: string, + event: unknown, + key?: string + ) => Effect.Effect; + sendOneBuffered: ( + topic: string, + event: unknown, + key?: string + ) => Effect.Effect; + shutDown: Effect.Effect; + stats: Effect.Effect; +} + const INITIAL_STATE: ProducerState = { buffer: [], sent: 0, @@ -85,6 +151,7 @@ const INITIAL_STATE: ProducerState = { producerInitialized: false, shuttingDown: false, flushing: false, + inFlight: 0, }; function toError(err: unknown): Error { @@ -121,55 +188,13 @@ async function insertClickHouseChunks( } } -function rebufferOrDropEvents({ - bufferHardMax, - events, - inc, - ref, - table, - error, -}: { - bufferHardMax: number; - error: FlushError; - events: unknown[]; - inc: (field: keyof ProducerState, n?: number) => Effect.Effect; - ref: Ref.Ref; - table: string; -}) { - return Ref.get(ref).pipe( - Effect.flatMap((state) => { - if (state.buffer.length + events.length <= bufferHardMax) { - return Ref.update(ref, (current) => ({ - ...current, - buffer: [ - ...current.buffer, - ...events.map((event) => ({ table, event })), - ], - errors: current.errors + 1, - })); - } - - return inc("dropped", events.length).pipe( - Effect.tap(() => inc("errors", 1)), - Effect.tap(() => - Effect.sync(() => - captureError(error.cause, { - message: `Dropped ${String(events.length)} events - buffer full`, - }) - ) - ) - ); - }) - ); -} - function makeProducerEffects( config: ProducerConfig, kafka: Producer | null, ch: ClickHouseClient, topicMap: Record, ref: Ref.Ref -) { +): ProducerEffects { const enabled = !config.selfHost && Boolean(config.broker); const inc = (field: keyof ProducerState, n = 1) => @@ -226,24 +251,26 @@ function makeProducerEffects( ); }); - const flush: Effect.Effect = Effect.gen(function* () { - const pre = yield* Ref.get(ref); - if (pre.buffer.length === 0 || pre.flushing) { - return; - } + const flush: Effect.Effect = Effect.gen(function* () { + const items = yield* Ref.modify(ref, (s) => { + if (s.buffer.length === 0 || s.flushing) { + return [null, s] as const; + } - const batchSize = Math.min(pre.buffer.length, config.bufferMax); - const items = yield* Ref.modify(ref, (s) => [ - s.buffer.slice(0, batchSize), - { ...s, buffer: s.buffer.slice(batchSize), flushing: true }, - ]); + const batchSize = Math.min(s.buffer.length, config.bufferMax); + return [s.buffer.slice(0, batchSize), { ...s, flushing: true }] as const; + }); - const grouped = groupBufferedEvents(items); + if (!items) { + return true; + } - yield* Effect.forEach( - grouped.entries(), - ([table, entries]: [string, BufferedEvent[]]) => { + const grouped = groupBufferedEvents(items); + return yield* Effect.forEach( + Array.from(grouped.entries()), + ([table, entries]) => { const events = entries.map((entry) => entry.event); + const flushedEntries = new Set(entries); return Effect.tryPromise({ try: () => record("clickhouseFallbackInsert", () => @@ -251,135 +278,252 @@ function makeProducerEffects( ), catch: (e) => new FlushError({ table, cause: toError(e) }), }).pipe( - Effect.tap(() => inc("flushed", events.length)), + Effect.tap(() => + Ref.update(ref, (s) => ({ + ...s, + buffer: s.buffer.filter((entry) => !flushedEntries.has(entry)), + flushed: s.flushed + entries.length, + })) + ), + Effect.as(true), Effect.catchTag("FlushError", (error) => - rebufferOrDropEvents({ - bufferHardMax: config.bufferHardMax, - error, - events, - inc, - ref, - table, - }) + Ref.update(ref, (s) => ({ + ...s, + errors: s.errors + 1, + lastErrorTime: Date.now(), + })).pipe( + Effect.tap(() => + Effect.sync(() => + captureError(error.cause, { + message: + "ClickHouse fallback insert failed; retaining events for retry", + table, + }) + ) + ), + Effect.as(false) + ) ) ); }, { concurrency: "unbounded" } + ).pipe( + Effect.map((results) => results.every(Boolean)), + Effect.ensuring(Ref.update(ref, (s) => ({ ...s, flushing: false }))) ); - - yield* Ref.update(ref, (s) => ({ ...s, flushing: false })); }); - const toBuffer = ( - topic: string, - event: unknown - ): Effect.Effect => + const resolveTopic = ( + topic: string + ): Effect.Effect => Effect.gen(function* () { const table = topicMap[topic]; - if (!table) { - yield* inc("errors", 1); - yield* Effect.sync(() => - captureError( - createError({ - code: "basket.UNKNOWN_KAFKA_TOPIC", - message: "Unknown Kafka topic", - status: 500, - why: `Topic "${topic}" is not mapped to a ClickHouse table.`, - fix: "Check topicMap configuration.", - }), - { topic } - ) - ); - return; + if (table) { + return table; } - const result = yield* Ref.modify(ref, (s) => { - if (s.shuttingDown) { - return ["shutdown" as const, { ...s, dropped: s.dropped + 1 }]; - } - if (s.buffer.length >= config.bufferHardMax) { - return ["overflow" as const, { ...s, dropped: s.dropped + 1 }]; - } - return [ - "ok" as const, - { + yield* inc("errors", 1); + yield* Effect.sync(() => + captureError( + createError({ + code: "basket.UNKNOWN_KAFKA_TOPIC", + message: "Unknown Kafka topic", + status: 500, + why: `Topic "${topic}" is not mapped to a ClickHouse table.`, + fix: "Check topicMap configuration.", + }), + { topic } + ) + ); + return yield* Effect.fail( + new UnknownKafkaTopicError({ retryable: false, topic }) + ); + }); + + const persistDirectly = ( + topic: string, + events: unknown[] + ): Effect.Effect => + Effect.gen(function* () { + const table = yield* resolveTopic(topic); + yield* Effect.tryPromise({ + try: () => + record("clickhouseDirectFallbackInsert", () => + insertClickHouseChunks(ch, table, events, config.chunkSize) + ), + catch: (e) => + new ClickHouseFallbackError({ + cause: toError(e), + retryable: true, + table, + topic, + }), + }).pipe( + Effect.tap(() => inc("flushed", events.length)), + Effect.catchTag("ClickHouseFallbackError", (error) => + Ref.update(ref, (s) => ({ ...s, - buffer: [...s.buffer, { table, event }], - buffered: s.buffered + 1, - }, - ]; - }); - - if (result === "overflow") { - return yield* Effect.fail( - new BufferOverflowError({ - bufferLength: (yield* Ref.get(ref)).buffer.length, + errors: s.errors + 1, + lastErrorTime: Date.now(), + })).pipe( + Effect.tap(() => + Effect.sync(() => + captureError(error.cause, { + message: + "Direct ClickHouse fallback insert failed; rejecting delivery", + table, + topic, + }) + ) + ), + Effect.flatMap(() => Effect.fail(error)) + ) + ) + ); + }); + + const bufferAll = ( + topic: string, + events: unknown[] + ): Effect.Effect => + Effect.gen(function* () { + const table = yield* resolveTopic(topic); + + type BufferAppendResult = + | { bufferLength: number; overflow: true } + | { overflow: false }; + const result = yield* Ref.modify( + ref, + (s): [BufferAppendResult, ProducerState] => { + if (s.buffer.length + events.length > config.bufferHardMax) { + return [ + { bufferLength: s.buffer.length, overflow: true }, + { + ...s, + errors: s.errors + 1, + lastErrorTime: Date.now(), + }, + ]; + } + + return [ + { overflow: false }, + { + ...s, + buffer: [ + ...s.buffer, + ...events.map((event) => ({ table, event })), + ], + buffered: s.buffered + events.length, + }, + ]; + } + ); + + if (result.overflow) { + const error = new BufferOverflowError({ + bufferHardMax: config.bufferHardMax, + bufferLength: result.bufferLength, + eventCount: events.length, + retryable: true, + topic, + }); + yield* Effect.sync(() => + captureError(error, { + message: "ClickHouse fallback buffer is full", + topic, }) ); + return yield* Effect.fail(error); } }); - const bufferAll = (topic: string, events: unknown[]) => - Effect.forEach(events, (e) => toBuffer(topic, e), { - discard: true, - }); + const acquireSendSlot = (eventCount: number) => + Ref.modify(ref, (s) => { + if (s.shuttingDown) { + return [false, s] as const; + } + return [true, { ...s, inFlight: s.inFlight + eventCount }] as const; + }).pipe( + Effect.flatMap((accepted) => + accepted + ? Effect.void + : Effect.fail( + new ProducerShuttingDownError({ + eventCount, + retryable: true, + }) + ) + ) + ); + + const releaseSendSlot = (eventCount: number) => + Ref.update(ref, (s) => ({ + ...s, + inFlight: Math.max(0, s.inFlight - eventCount), + })); const sendViaKafka = ( topic: string, messages: Array<{ value: string; key?: string }>, - fallbackEvents: unknown[] + fallbackEvents: unknown[], + fallback: "buffer" | "direct" ): Effect.Effect => - Effect.gen(function* () { - const s = yield* Ref.get(ref); - if (s.shuttingDown) { - yield* bufferAll(topic, fallbackEvents); - return; - } - - if (enabled && kafka) { - const isConnected = yield* connect; - if (isConnected) { - const sent = yield* Effect.tryPromise({ - try: () => - kafka.send({ - topic, - messages, - timeout: config.kafkaTimeout, - compression: CompressionTypes.GZIP, - }), - catch: (e) => new KafkaSendError({ topic, cause: toError(e) }), - }).pipe( - Effect.tap(() => inc("sent", messages.length)), - Effect.as(true), - Effect.catchTag("KafkaSendError", (err) => - Ref.update(ref, (st) => ({ - ...st, - connectionFailed: true, - connected: false, - lastRetry: Date.now(), - failedCount: st.failedCount + messages.length, - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(err.cause, { - message: "Redpanda send failed, buffering to ClickHouse", - message_count: messages.length, - topic, - }) + acquireSendSlot(fallbackEvents.length).pipe( + Effect.flatMap(() => + Effect.gen(function* () { + if (enabled && kafka) { + const isConnected = yield* connect; + if (isConnected) { + const sent = yield* Effect.tryPromise({ + try: () => + kafka.send({ + topic, + messages, + timeout: config.kafkaTimeout, + compression: CompressionTypes.GZIP, + }), + catch: (e) => new KafkaSendError({ topic, cause: toError(e) }), + }).pipe( + Effect.tap(() => inc("sent", messages.length)), + Effect.as(true), + Effect.catchTag("KafkaSendError", (err) => + Ref.update(ref, (st) => ({ + ...st, + connectionFailed: true, + connected: false, + lastRetry: Date.now(), + failedCount: st.failedCount + messages.length, + })).pipe( + Effect.tap(() => + Effect.sync(() => + captureError(err.cause, { + message: + "Redpanda send failed, falling back to ClickHouse", + message_count: messages.length, + topic, + }) + ) + ), + Effect.as(false) ) - ), - Effect.as(false) - ) - ) - ); - if (sent) { + ) + ); + if (sent) { + return; + } + } + } + + if (fallback === "direct") { + yield* persistDirectly(topic, fallbackEvents); return; } - } - } - yield* bufferAll(topic, fallbackEvents); - }); + yield* bufferAll(topic, fallbackEvents); + }).pipe(Effect.ensuring(releaseSendSlot(fallbackEvents.length))) + ) + ); const sendOne = ( topic: string, @@ -394,7 +538,8 @@ function makeProducerEffects( key: key || (event as { client_id?: string }).client_id, }, ], - [event] + [event], + "direct" ); const sendMany = ( @@ -412,41 +557,117 @@ function makeProducerEffects( (e as { client_id?: string }).client_id || (e as { event_id?: string }).event_id, })), - events + events, + "direct" ); }; - const shutDown: Effect.Effect = Effect.gen(function* () { - yield* Ref.update(ref, (s) => ({ ...s, shuttingDown: true })); - yield* Effect.sleep("1 second"); - yield* flush.pipe(Effect.catch(() => Effect.void)); - const post = yield* Ref.get(ref); - if (post.buffer.length > 0 && !post.flushing) { - yield* flush.pipe(Effect.catch(() => Effect.void)); + const sendOneBuffered = ( + topic: string, + event: unknown, + key?: string + ): Effect.Effect => + sendViaKafka( + topic, + [ + { + value: stringifyEvent(event), + key: key || (event as { client_id?: string }).client_id, + }, + ], + [event], + "buffer" + ); + + const sendManyBuffered = ( + topic: string, + events: unknown[] + ): Effect.Effect => { + if (events.length === 0) { + return Effect.void; } - if (post.producerInitialized && kafka) { - yield* Effect.tryPromise({ - try: () => kafka.disconnect(), - catch: (e) => new KafkaConnectionError({ cause: toError(e) }), - }).pipe( - Effect.ensuring( - Ref.update(ref, (s) => ({ - ...s, - connected: false, - producerInitialized: false, - })) - ), - Effect.catch((err) => - Effect.sync(() => - captureError(err.cause, { - message: "Error disconnecting Redpanda producer", - }) - ) + return sendViaKafka( + topic, + events.map((e) => ({ + value: stringifyEvent(e), + key: + (e as { client_id?: string }).client_id || + (e as { event_id?: string }).event_id, + })), + events, + "buffer" + ); + }; + + const disconnectKafka = Effect.gen(function* () { + const state = yield* Ref.get(ref); + if (!(state.producerInitialized && kafka)) { + return; + } + + yield* Effect.tryPromise({ + try: () => kafka.disconnect(), + catch: (e) => new KafkaConnectionError({ cause: toError(e) }), + }).pipe( + Effect.ensuring( + Ref.update(ref, (s) => ({ + ...s, + connected: false, + producerInitialized: false, + })) + ), + Effect.catch((err) => + Effect.sync(() => + captureError(err.cause, { + message: "Error disconnecting Redpanda producer", + }) ) - ); + ) + ); + }); + + const drainUntilEmpty = Effect.gen(function* () { + while (true) { + const state = yield* Ref.get(ref); + if (state.inFlight > 0 || state.flushing) { + yield* Effect.sleep("10 millis"); + continue; + } + if (state.buffer.length === 0) { + return; + } + + const didFlush = yield* flush; + if (!didFlush) { + yield* Effect.sleep(`${config.shutdownDrainRetryDelay} millis`); + } } }); + const shutDown: Effect.Effect = Effect.gen( + function* () { + yield* Ref.update(ref, (s) => ({ ...s, shuttingDown: true })); + yield* drainUntilEmpty.pipe( + Effect.timeout(`${config.shutdownDrainTimeout} millis`), + Effect.catch(() => + Ref.get(ref).pipe( + Effect.flatMap((state) => + Effect.fail( + new ShutdownDrainError({ + deadlineMs: config.shutdownDrainTimeout, + inFlight: state.inFlight, + residualCount: state.buffer.length + state.inFlight, + retryable: true, + }) + ) + ) + ) + ), + Effect.ensuring(disconnectKafka) + ); + } + ); + const stats: Effect.Effect = Ref.get(ref).pipe( Effect.map( ({ @@ -465,9 +686,27 @@ function makeProducerEffects( ) ); - return { flush, sendOne, sendMany, shutDown, stats }; + return { + flush, + sendOne, + sendMany, + sendOneBuffered, + sendManyBuffered, + shutDown, + stats, + }; } +export const createProducerEffects = ( + config: ProducerConfig, + kafka: Producer | null, + ch: ClickHouseClient, + topicMap: Record +): Effect.Effect => + Ref.make({ ...INITIAL_STATE, buffer: [] }).pipe( + Effect.map((ref) => makeProducerEffects(config, kafka, ch, topicMap, ref)) + ); + function initializeKafka(config: ProducerConfig): Producer | null { if (config.selfHost || !config.broker) { return null; @@ -517,6 +756,7 @@ export interface ProducerStatsSnapshot { failed: boolean; failedCount: number; flushed: number; + inFlight: number; kafkaEnabled: boolean; lastErrorTime: number | null; lastRetry: number; @@ -536,6 +776,8 @@ const CONFIG: ProducerConfig = { bufferMax: 1000, bufferHardMax: 10_000, chunkSize: 5000, + shutdownDrainRetryDelay: 250, + shutdownDrainTimeout: 8000, }; const TOPIC_MAP: Record = { @@ -553,13 +795,11 @@ let fx: ReturnType | null = null; const ProducerLive = Layer.effectDiscard( Effect.gen(function* () { - const ref = yield* Ref.make({ ...INITIAL_STATE }); - const effects = makeProducerEffects( + const effects = yield* createProducerEffects( CONFIG, initializeKafka(CONFIG), clickHouse, - TOPIC_MAP, - ref + TOPIC_MAP ); fx = effects; yield* effects.flush.pipe( @@ -577,18 +817,44 @@ const withFx = ( ): Effect.Effect => Effect.suspend(() => (fx ? fn(fx) : Effect.succeed(undefined))); +const withActiveFx = ( + fn: (f: NonNullable) => Effect.Effect +): Effect.Effect => + Effect.suspend( + (): Effect.Effect => + fx + ? fn(fx) + : Effect.fail(new ProducerUnavailableError({ retryable: true })) + ); + export const send = (topic: string, event: unknown, key?: string) => - withFx((f) => f.sendOne(topic, event, key)); + withActiveFx((f) => f.sendOne(topic, event, key)); export const sendBatch = (topic: string, events: unknown[]) => - withFx((f) => f.sendMany(topic, events)); + withActiveFx((f) => f.sendMany(topic, events)); + +export const sendBuffered = (topic: string, event: unknown, key?: string) => + withActiveFx((f) => f.sendOneBuffered(topic, event, key)); + +export const sendBufferedBatch = (topic: string, events: unknown[]) => + withActiveFx((f) => f.sendManyBuffered(topic, events)); export const disconnect = withFx((f) => f.shutDown); export const getStats = withFx((f) => f.stats); export const runFork = (effect: Effect.Effect) => - runtime.runFork(effect); + runtime.runFork( + effect.pipe( + Effect.tapError((error) => + Effect.sync(() => + captureError(error, { + message: "Asynchronous producer delivery rejected", + }) + ) + ) + ) + ); export const runPromise = (effect: Effect.Effect) => runtime.runPromise(effect); diff --git a/apps/basket/src/lib/security.test.ts b/apps/basket/src/lib/security.test.ts index a34f98cc48..ed04ade96b 100644 --- a/apps/basket/src/lib/security.test.ts +++ b/apps/basket/src/lib/security.test.ts @@ -1,7 +1,9 @@ import { vi, beforeEach, describe, expect, test } from "vitest"; import { - checkDuplicate, applyVisitorIdPrivacy, + markDuplicateReservationDelivered, + releaseDuplicateReservation, + reserveDuplicate, saltAnonymousId, shouldAnonymizeVisitorIds, } from "./security"; @@ -114,16 +116,19 @@ describe("visitor ID anonymization helpers", () => { }); }); -// ── checkDuplicate (needs Redis mock) ── +// ── duplicate reservations (needs Redis mock) ── -const { mockRedisSet, mockLoggerSet, mockCaptureError } = vi.hoisted(() => ({ - mockRedisSet: vi.fn(() => Promise.resolve("OK")), - mockLoggerSet: vi.fn(() => {}), - mockCaptureError: vi.fn(), -})); +const { mockRedisSet, mockRedisGet, mockRedisEval, mockLoggerSet, mockCaptureError } = + vi.hoisted(() => ({ + mockRedisSet: vi.fn(() => Promise.resolve("OK")), + mockRedisGet: vi.fn(() => Promise.resolve(null)), + mockRedisEval: vi.fn(() => Promise.resolve(1)), + mockLoggerSet: vi.fn(() => {}), + mockCaptureError: vi.fn(), + })); vi.mock("@databuddy/redis/redis", () => ({ - redis: { set: mockRedisSet }, + redis: { set: mockRedisSet, get: mockRedisGet, eval: mockRedisEval }, getRedisCache: () => ({ set: mockRedisSet }), })); vi.mock("@databuddy/redis/cacheable", () => ({ @@ -140,106 +145,185 @@ vi.mock("@lib/tracing", () => ({ captureError: mockCaptureError, })); -describe("checkDuplicate", () => { +describe("duplicate reservations", () => { beforeEach(() => { mockRedisSet.mockReset(); + mockRedisGet.mockReset(); + mockRedisEval.mockReset(); mockLoggerSet.mockReset(); mockCaptureError.mockReset(); }); - test("first event (NX returns OK) → not duplicate", async () => { + test("writes a pending reservation for a new event", async () => { mockRedisSet.mockResolvedValue("OK"); - const result = await checkDuplicate("evt_1", "track"); - expect(result).toBe(false); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toMatchObject({ + duplicate: false, + key: "dedup:track:evt_1", + token: expect.stringMatching(/^pending:/), + ttl: 86_400, + }); expect(mockRedisSet).toHaveBeenCalledWith( "dedup:track:evt_1", - "1", + expect.stringMatching(/^pending:/), "EX", 86_400, "NX" ); }); - test("duplicate event (NX returns null) → is duplicate", async () => { + test("suppresses only a confirmed delivered reservation", async () => { mockRedisSet.mockResolvedValue(null); - const result = await checkDuplicate("evt_1", "track"); - expect(result).toBe(true); + mockRedisGet.mockResolvedValue("delivered"); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: true }); + expect(mockLoggerSet).toHaveBeenCalledWith({ + dedup: { duplicate: true, eventType: "track" }, + }); }); - test("exit_ prefix → uses longer TTL (172800)", async () => { - mockRedisSet.mockResolvedValue("OK"); - await checkDuplicate("exit_abc", "track"); - expect(mockRedisSet).toHaveBeenCalledWith( - "dedup:track:exit_abc", - "1", - "EX", - 172_800, - "NX" - ); + test("retries a stale pending reservation instead of falsely suppressing it", async () => { + mockRedisSet.mockResolvedValue(null); + mockRedisGet.mockResolvedValue("pending:other-attempt"); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toMatchObject({ + duplicate: false, + key: "dedup:track:evt_1", + token: undefined, + ttl: 86_400, + }); }); - test("non-exit prefix → uses standard TTL (86400)", async () => { - mockRedisSet.mockResolvedValue("OK"); - await checkDuplicate("normal_abc", "track"); - expect(mockRedisSet).toHaveBeenCalledWith( - "dedup:track:normal_abc", - "1", - "EX", - 86_400, - "NX" - ); + test("recovers ownership when an ambiguous SET wrote this request's token", async () => { + mockRedisSet.mockResolvedValue(null); + mockRedisGet.mockImplementation(async () => mockRedisSet.mock.calls[0]?.[1]); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation.token).toBe(mockRedisSet.mock.calls[0]?.[1]); }); - test("different event types → different keys", async () => { + test("preserves the exit-event retention when its delivery id is hashed", async () => { mockRedisSet.mockResolvedValue("OK"); - await checkDuplicate("evt_1", "outgoing_link"); + + await reserveDuplicate("stable-delivery-id", "track", "exit_abc"); + expect(mockRedisSet).toHaveBeenCalledWith( - "dedup:outgoing_link:evt_1", - "1", + "dedup:track:stable-delivery-id", + expect.stringMatching(/^pending:/), "EX", - 86_400, + 172_800, "NX" ); }); - test("transient Redis error → retries once before failing open", async () => { + test("retries a Redis error before returning a retryable reservation", async () => { mockRedisSet .mockRejectedValueOnce(new Error("stale connection")) .mockResolvedValueOnce("OK"); - const result = await checkDuplicate("evt_1", "track"); + const reservation = await reserveDuplicate("evt_1", "track"); - expect(result).toBe(false); + expect(reservation.duplicate).toBe(false); expect(mockRedisSet).toHaveBeenCalledTimes(2); expect(mockCaptureError).not.toHaveBeenCalled(); }); - test("ambiguous retry null after Redis error → not duplicate", async () => { - mockRedisSet - .mockRejectedValueOnce(new Error("stale connection")) - .mockResolvedValueOnce(null); + test("fails open for a Redis outage without turning an event into a duplicate", async () => { + mockRedisSet.mockRejectedValue(new Error("Redis down")); + mockRedisGet.mockRejectedValue(new Error("Redis down")); - const result = await checkDuplicate("evt_1", "track"); + const reservation = await reserveDuplicate("evt_1", "track"); - expect(result).toBe(false); + expect(reservation).toMatchObject({ duplicate: false, ttl: 86_400 }); expect(mockRedisSet).toHaveBeenCalledTimes(2); - expect(mockLoggerSet).not.toHaveBeenCalled(); - expect(mockCaptureError).not.toHaveBeenCalled(); + expect(mockCaptureError).toHaveBeenCalledOnce(); }); - test("Redis error → returns false (fail-open)", async () => { - mockRedisSet.mockRejectedValue(new Error("Redis down")); - const result = await checkDuplicate("evt_1", "track"); - expect(result).toBe(false); - expect(mockRedisSet).toHaveBeenCalledTimes(2); - expect(mockCaptureError).toHaveBeenCalledOnce(); + test("does not release a reservation it does not own", async () => { + await releaseDuplicateReservation({ duplicate: false }); + + expect(mockRedisEval).not.toHaveBeenCalled(); }); - test("duplicate event logs dedup context", async () => { - mockRedisSet.mockResolvedValue(null); - await checkDuplicate("evt_dup", "track"); - expect(mockLoggerSet).toHaveBeenCalledWith({ - dedup: { duplicate: true, eventType: "track" }, + test("marks a confirmed delivery before suppressing future retries", async () => { + mockRedisSet.mockResolvedValue("OK"); + + await markDuplicateReservationDelivered({ + duplicate: false, + key: "dedup:track:evt_1", + ttl: 86_400, }); + + expect(mockRedisSet).toHaveBeenCalledWith( + "dedup:track:evt_1", + "delivered", + "EX", + 86_400 + ); + }); + + test("releases only the pending reservation owned by the failed request", async () => { + mockRedisEval.mockResolvedValue(1); + + await releaseDuplicateReservation({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:owner-attempt", + }); + + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + 1, + "dedup:track:evt_1", + "pending:owner-attempt" + ); + }); + + test("does not let a stale attempt release a newer reservation", async () => { + let storedToken = "pending:newer-attempt"; + mockRedisEval.mockImplementation( + async (_script, _keys, _key, expectedToken: string) => { + if (storedToken === expectedToken) { + storedToken = ""; + return 1; + } + return 0; + } + ); + + await releaseDuplicateReservation({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:stale-attempt", + }); + + expect(storedToken).toBe("pending:newer-attempt"); + }); + + test("captures release failures without hiding the original delivery failure", async () => { + mockRedisEval.mockRejectedValue(new Error("Redis down")); + + await expect( + releaseDuplicateReservation({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:owner-attempt", + }) + ).resolves.toBeUndefined(); + + expect(mockCaptureError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + message: + "Failed to release duplicate reservation after delivery failure", + }) + ); }); }); diff --git a/apps/basket/src/lib/security.ts b/apps/basket/src/lib/security.ts index 2fc24c01dc..84de738f6b 100644 --- a/apps/basket/src/lib/security.ts +++ b/apps/basket/src/lib/security.ts @@ -9,6 +9,12 @@ import { useLogger } from "evlog/elysia"; const EXIT_EVENT_TTL = 172_800; const STANDARD_EVENT_TTL = 86_400; const DEDUP_RETRY_DELAY_MS = 25; +const PENDING_DEDUP_PREFIX = "pending:"; +const DELIVERED_DEDUP_VALUE = "delivered"; +const RELEASE_PENDING_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("DEL", KEYS[1]) +end +return 0`; const RAW_VISITOR_ID_COUNTRIES = ["US"]; const COUNTRY_CODES: Record = { @@ -123,46 +129,156 @@ function wait(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -async function setDedupKey(key: string, ttl: number): Promise { +export interface DuplicateReservation { + readonly duplicate: boolean; + readonly key?: string; + /** + * Present only when this request atomically acquired the pending key. It is + * required to release the key so an older failed request cannot erase a + * newer retry's reservation. + */ + readonly token?: string; + readonly ttl?: number; +} + +type DedupReservationState = + | "acquired" + | "delivered" + | "pending" + | "unreserved"; + +async function readDedupReservationState( + key: string, + token: string +): Promise { + const value = await redis.get(key); + if (value === DELIVERED_DEDUP_VALUE) { + return "delivered"; + } + if (value === token) { + return "acquired"; + } + return value === null || value === undefined ? "unreserved" : "pending"; +} + +async function setDedupKey( + key: string, + ttl: number, + token: string +): Promise { try { - return await redis.set(key, "1", "EX", ttl, "NX"); + const result = await redis.set(key, token, "EX", ttl, "NX"); + return result === null ? readDedupReservationState(key, token) : "acquired"; } catch (firstError) { await wait(DEDUP_RETRY_DELAY_MS); try { - const retryResult = await redis.set(key, "1", "EX", ttl, "NX"); - // If the first SET succeeded but the client saw an error, retry returns null. - // Treat that ambiguous state as first delivery so ingestion fails open. - return retryResult ?? "OK"; + const retryResult = await redis.set(key, token, "EX", ttl, "NX"); + if (retryResult !== null) { + return "acquired"; + } + + // The first SET may have succeeded even though the client saw an error. + // Seeing our token again proves that this request owns the reservation. + // Any other pending token is retryable: only a confirmed delivery may + // suppress the event. + return await readDedupReservationState(key, token); } catch { - throw firstError; + try { + return await readDedupReservationState(key, token); + } catch { + throw firstError; + } } } } -export function checkDuplicate( +export function reserveDuplicate( eventId: string, - eventType: string -): Promise { - return record("checkDuplicate", async () => { + eventType: string, + sourceEventId = eventId +): Promise { + return record("reserveDuplicate", async () => { const key = `dedup:${eventType}:${eventId}`; - const ttl = eventId.startsWith("exit_") + const ttl = sourceEventId.startsWith("exit_") ? EXIT_EVENT_TTL : STANDARD_EVENT_TTL; + const token = `${PENDING_DEDUP_PREFIX}${crypto.randomUUID()}`; try { - const result = await setDedupKey(key, ttl); - const isDuplicate = result === null; - if (isDuplicate) { + const result = await setDedupKey(key, ttl, token); + if (result === "delivered") { useLogger().set({ dedup: { duplicate: true, eventType } }); + return { duplicate: true }; } - return isDuplicate; + + // A pending key may be a concurrent attempt, an interrupted request, or + // a release that timed out. Retrying the same stable delivery id is safer + // than falsely reporting success for an event that never reached storage. + return { + duplicate: false, + key, + token: result === "acquired" ? token : undefined, + ttl, + }; } catch (error) { captureError(error, { message: "Failed to check duplicate event in Redis", eventId, eventType, }); - return false; + return { duplicate: false, key, ttl }; + } + }); +} + +/** + * Only a confirmed Kafka or ClickHouse acknowledgement may turn a pending + * retry guard into a duplicate suppression key. A Redis failure here preserves + * the pending state, which deliberately favors an idempotent retry over loss. + */ +export function markDuplicateReservationDelivered( + reservation: DuplicateReservation +): Promise { + return record("markDuplicateReservationDelivered", async () => { + if (!(reservation.key && reservation.ttl)) { + return; + } + + try { + await redis.set( + reservation.key, + DELIVERED_DEDUP_VALUE, + "EX", + reservation.ttl + ); + } catch (error) { + captureError(error, { + message: "Failed to confirm duplicate reservation after delivery", + }); + } + }); +} + +export function releaseDuplicateReservation( + reservation: DuplicateReservation +): Promise { + return record("releaseDuplicateReservation", async () => { + if (!(reservation.key && reservation.token)) { + return; + } + + try { + await redis.eval( + RELEASE_PENDING_DEDUP_RESERVATION, + 1, + reservation.key, + reservation.token + ); + } catch (error) { + captureError(error, { + message: + "Failed to release duplicate reservation after delivery failure", + }); } }); } diff --git a/apps/basket/src/routes/basket.ts b/apps/basket/src/routes/basket.ts index 5c73b25ca9..488f7772e1 100644 --- a/apps/basket/src/routes/basket.ts +++ b/apps/basket/src/routes/basket.ts @@ -24,6 +24,7 @@ import { insertOutgoingLinksBatch, insertTrackEvent, insertTrackEventsBatch, + stableAnalyticsEventId, } from "@lib/event-service"; import { summarizeRejectedBody } from "@lib/rejection-summary"; import { @@ -113,6 +114,7 @@ async function processOutgoingLinkData( clientId: string, visitorCountry?: unknown ): Promise { + const eventId = parseEventId(linkData.eventId, () => randomUUIDv7()); const timestamp = parseTimestamp(linkData.timestamp); const anonymizeVisitorIds = shouldAnonymizeVisitorIds( linkData.anonymizeVisitorIds, @@ -127,7 +129,7 @@ async function processOutgoingLinkData( ); return { - id: randomUUIDv7(), + id: stableAnalyticsEventId(clientId, "outgoing_link", eventId), client_id: clientId, anonymous_id: anonymousId, session_id: validateSessionId(linkData.sessionId), @@ -169,9 +171,9 @@ const app = new Elysia() } if (eventType === "track") { - insertTrackEvent(eventData, clientId, userAgent, ip, request); + await insertTrackEvent(eventData, clientId, userAgent, ip, request); } else if (eventType === "outgoing_link") { - insertOutgoingLink(eventData, clientId, request); + await insertOutgoingLink(eventData, clientId, request); } else if (eventType === "web_vitals") { const vitalParse = individualVitalSchema.safeParse(eventData); if (!vitalParse.success) { @@ -182,7 +184,11 @@ const app = new Elysia() [vitalParse.data], request ); - insertIndividualVitals([vitalParse.data], clientId, visitorCountry); + await insertIndividualVitals( + [vitalParse.data], + clientId, + visitorCountry + ); } else if (eventType === "error") { const errorParse = errorSpanSchema.safeParse(eventData); if (!errorParse.success) { @@ -193,16 +199,18 @@ const app = new Elysia() [errorParse.data], request ); - insertErrorSpans([errorParse.data], clientId, visitorCountry); + await insertErrorSpans([errorParse.data], clientId, visitorCountry); } return createPixelResponse(); } catch (error) { - if (error instanceof EvlogError) { + if (error instanceof EvlogError && error.status < 500) { return createPixelResponse(); } log.error(error instanceof Error ? error : new Error(String(error))); - return createPixelResponse(); + // Do not report a successful image load before the event has durable + // storage; callers that can retry need an honest failure response. + return createPixelResponse(503); } }) .post("/vitals", async ({ body, query, request }) => { diff --git a/apps/basket/src/routes/integration.test.ts b/apps/basket/src/routes/integration.test.ts index 8a08c6bea3..47fe92a359 100644 --- a/apps/basket/src/routes/integration.test.ts +++ b/apps/basket/src/routes/integration.test.ts @@ -102,6 +102,7 @@ vi.mock("@lib/event-service", () => ({ insertIndividualVitals: mockInsertIndividualVitals, insertErrorSpans: mockInsertErrorSpans, insertCustomEvents: mockInsertCustomEvents, + stableAnalyticsEventId: vi.fn(() => "stable_id"), })); vi.mock("@lib/security", () => ({ @@ -529,11 +530,12 @@ describe("GET /px.jpg", () => { expect(res.headers.get("Content-Type")).toBe("image/gif"); }); - test("always returns pixel even on error", async () => { + test("returns a retryable GIF when delivery fails", async () => { mockValidateRequest.mockRejectedValueOnce(new Error("boom")); const res = await get(basketApp, "/px.jpg?name=test"); - expect(res.status).toBe(200); + expect(res.status).toBe(503); expect(res.headers.get("Content-Type")).toBe("image/gif"); + expect(res.headers.get("Retry-After")).toBe("5"); }); }); diff --git a/apps/basket/src/utils/pixel.test.ts b/apps/basket/src/utils/pixel.test.ts index 9c1856d132..c045ec64af 100644 --- a/apps/basket/src/utils/pixel.test.ts +++ b/apps/basket/src/utils/pixel.test.ts @@ -19,6 +19,13 @@ describe("createPixelResponse", () => { new Uint8Array([0x47, 0x49, 0x46]) ); }); + + test("keeps the GIF body but marks delivery failures retryable", () => { + const r = createPixelResponse(503); + expect(r.status).toBe(503); + expect(r.headers.get("Content-Type")).toBe("image/gif"); + expect(r.headers.get("Retry-After")).toBe("5"); + }); }); // ── parsePixelQuery ── diff --git a/apps/basket/src/utils/pixel.ts b/apps/basket/src/utils/pixel.ts index f0cf9e9b10..2f4a5826f2 100644 --- a/apps/basket/src/utils/pixel.ts +++ b/apps/basket/src/utils/pixel.ts @@ -13,14 +13,15 @@ const TRANSPARENT_PIXEL = Buffer.from( /** * Returns a 1x1 transparent GIF response */ -export function createPixelResponse(): Response { +export function createPixelResponse(status = 200): Response { return new Response(TRANSPARENT_PIXEL, { - status: 200, + status, headers: { "Content-Type": "image/gif", "Cache-Control": "no-cache, no-store, must-revalidate", Pragma: "no-cache", Expires: "0", + ...(status >= 500 ? { "Retry-After": "5" } : {}), }, }); } diff --git a/packages/tracker/src/plugins/pixel.ts b/packages/tracker/src/plugins/pixel.ts index b7764b8f78..ea2fb66580 100644 --- a/packages/tracker/src/plugins/pixel.ts +++ b/packages/tracker/src/plugins/pixel.ts @@ -11,6 +11,12 @@ const PIXEL_TYPE_BY_ENDPOINT: Record = { "/vitals": "web_vitals", "/errors": "error", }; +const MAX_PIXEL_RETRY_DELAY_MS = 30_000; + +interface PixelDeliveryResult { + attempts: number; + success: boolean; +} function safeStringify(value: unknown): string { const seen = new WeakSet(); @@ -49,11 +55,36 @@ function flattenIntoParams( export function initPixelTracking(tracker: BaseTracker) { tracker.options.enableBatching = false; + let deliveryGeneration = 0; + const cancelPendingRequests = tracker.api.cancelPendingRequests.bind( + tracker.api + ); + tracker.api.cancelPendingRequests = () => { + deliveryGeneration += 1; + cancelPendingRequests(); + }; + + const maxRetries = + tracker.options.enableRetries === false + ? 0 + : (tracker.options.maxRetries ?? 3); + const initialRetryDelay = Math.max( + 0, + tracker.options.initialRetryDelay ?? 500 + ); + + const waitForRetry = (retry: number): Promise => + new Promise((resolve) => { + setTimeout( + resolve, + Math.min(initialRetryDelay * 2 ** retry, MAX_PIXEL_RETRY_DELAY_MS) + ); + }); const sendOnePixel = ( eventType: string, data: Record - ): Promise<{ success: boolean }> => { + ): Promise => { const params = new URLSearchParams(); flattenIntoParams(params, data); @@ -76,21 +107,39 @@ export function initPixelTracking(tracker: BaseTracker) { url.searchParams.append(key, value); }); - return new Promise((resolve) => { - const img = new Image(); - img.onload = () => resolve({ success: true }); - img.onerror = () => resolve({ success: false }); - img.src = url.toString(); - }); + const generation = deliveryGeneration; + const load = (): Promise => + new Promise((resolve) => { + const img = new Image(); + img.onload = () => resolve(true); + img.onerror = () => resolve(false); + img.src = url.toString(); + }); + + return (async () => { + for (let retry = 0; retry <= maxRetries; retry += 1) { + if (generation !== deliveryGeneration) { + return { attempts: retry, success: false }; + } + if (await load()) { + return { attempts: retry + 1, success: true }; + } + if (retry < maxRetries && generation === deliveryGeneration) { + await waitForRetry(retry); + } + } + + return { attempts: maxRetries + 1, success: false }; + })(); }; const sendToPixel = async ( endpoint: string, data: unknown - ): Promise<{ success: boolean }> => { + ): Promise => { const eventType = PIXEL_TYPE_BY_ENDPOINT[endpoint]; if (!eventType) { - return { success: false }; + return { attempts: 1, success: false }; } if (Array.isArray(data)) { @@ -98,14 +147,17 @@ export function initPixelTracking(tracker: BaseTracker) { data.map((event) => event && typeof event === "object" ? sendOnePixel(eventType, event as Record) - : Promise.resolve({ success: false }) + : Promise.resolve({ attempts: 1, success: false }) ) ); - return { success: results.every((r) => r.success) }; + return { + attempts: Math.max(1, ...results.map((result) => result.attempts)), + success: results.every((result) => result.success), + }; } if (typeof data !== "object" || data === null) { - return { success: false }; + return { attempts: 1, success: false }; } return sendOnePixel(eventType, data as Record); }; @@ -120,7 +172,7 @@ export function initPixelTracking(tracker: BaseTracker) { ok: true, data: null, status: null, - attempts: 1, + attempts: result.attempts, transport: "beacon", }; } @@ -130,15 +182,15 @@ export function initPixelTracking(tracker: BaseTracker) { message: "Tracking pixel failed to load", status: null, retryable: true, - attempts: 1, + attempts: result.attempts, transport: "fetch", }; }; - tracker.sendBeacon = (data: unknown, endpoint = "/") => { - sendToPixel(endpoint, data); - return true; - }; + // Image loads cannot synchronously prove remote acceptance. Returning false + // keeps BaseTracker's queue for its fetch-style pixel fallback instead of + // dropping it immediately on an unverified load. + tracker.sendBeacon = () => false; tracker.sendBatchBeacon = () => false; } diff --git a/packages/tracker/tests/unit/pixel.test.ts b/packages/tracker/tests/unit/pixel.test.ts new file mode 100644 index 0000000000..949f460f46 --- /dev/null +++ b/packages/tracker/tests/unit/pixel.test.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, mock, test } from "bun:test"; +import type { BaseTracker } from "../../src/core/tracker"; +import type { TrackerOptions } from "../../src/core/types"; +import { initPixelTracking } from "../../src/plugins/pixel"; + +const originalImage = Object.getOwnPropertyDescriptor(globalThis, "Image"); + +afterEach(() => { + if (originalImage) { + Object.defineProperty(globalThis, "Image", originalImage); + return; + } + Reflect.deleteProperty(globalThis, "Image"); +}); + +function createTracker( + overrides: Partial = {} +): BaseTracker { + return { + api: { + cancelPendingRequests: mock(() => {}), + }, + options: { + clientId: "site_example", + enableRetries: true, + initialRetryDelay: 0, + maxRetries: 1, + sdk: "web", + sdkVersion: "2.0.0", + ...overrides, + }, + } as unknown as BaseTracker; +} + +function installImageOutcomes(outcomes: Array<"error" | "load">): string[] { + const requests: string[] = []; + + class MockImage { + onerror: (() => void) | null = null; + onload: (() => void) | null = null; + + set src(value: string) { + requests.push(value); + const outcome = outcomes.shift() ?? "error"; + queueMicrotask(() => { + if (outcome === "load") { + this.onload?.(); + return; + } + this.onerror?.(); + }); + } + } + + Object.defineProperty(globalThis, "Image", { + configurable: true, + value: MockImage, + }); + return requests; +} + +describe("pixel transport", () => { + test("retries an unacknowledged pixel load with the same event identity", async () => { + const requests = installImageOutcomes(["error", "load"]); + const tracker = createTracker(); + initPixelTracking(tracker); + + const result = await tracker.api.fetch("/", { + eventId: "event_1", + name: "pageview", + }); + + expect(result).toMatchObject({ ok: true, attempts: 2 }); + expect(requests).toHaveLength(2); + const first = new URL(requests[0] ?? ""); + const second = new URL(requests[1] ?? ""); + expect(first.searchParams.get("eventId")).toBe("event_1"); + expect(second.searchParams.get("eventId")).toBe("event_1"); + }); + + test("does not report an unverified image load as a beacon success", () => { + const tracker = createTracker(); + initPixelTracking(tracker); + + expect(tracker.sendBeacon({ eventId: "event_1" }, "/")).toBe(false); + }); +}); From ecbe223aa7ee60b3c7a2bb9934253d9d00538ae4 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:25:54 +0300 Subject: [PATCH 05/25] fix(basket): prevent concurrent event delivery --- .../src/lib/event-service.delivery.test.ts | 67 +++++++++++++++++++ apps/basket/src/lib/event-service.ts | 10 +++ apps/basket/src/lib/security.test.ts | 54 +++++++++++---- apps/basket/src/lib/security.ts | 35 +++++++--- 4 files changed, 143 insertions(+), 23 deletions(-) diff --git a/apps/basket/src/lib/event-service.delivery.test.ts b/apps/basket/src/lib/event-service.delivery.test.ts index afb037246b..5e4d0d0df5 100644 --- a/apps/basket/src/lib/event-service.delivery.test.ts +++ b/apps/basket/src/lib/event-service.delivery.test.ts @@ -78,6 +78,7 @@ const request = new Request("http://localhost/ingest"); const trackReservation = { duplicate: false, key: "dedup:track:evt_1", + token: "pending:reservation-token", ttl: 86_400, }; @@ -221,4 +222,70 @@ describe("core event delivery", () => { expect(mockSend).not.toHaveBeenCalled(); expect(mockRunPromise).not.toHaveBeenCalled(); }); + + test("returns retryable 503 without publishing a track event owned by another request", async () => { + mockReserveDuplicate.mockResolvedValueOnce({ + duplicate: false, + retryable: true, + }); + + await expect( + insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSend).not.toHaveBeenCalled(); + expect(mockRunPromise).not.toHaveBeenCalled(); + expect(mockMarkDuplicateReservationDelivered).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("returns retryable 503 without publishing an outgoing link owned by another request", async () => { + mockReserveDuplicate.mockResolvedValueOnce({ + duplicate: false, + retryable: true, + }); + + await expect( + insertOutgoingLink( + { + anonymousId: "anon_1", + eventId: "evt_link_1", + href: "https://example.com/pricing", + }, + "ws_test", + request + ) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSend).not.toHaveBeenCalled(); + expect(mockRunPromise).not.toHaveBeenCalled(); + expect(mockMarkDuplicateReservationDelivered).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); + + test("continues delivery when Redis is unavailable and no reservation was acquired", async () => { + const unavailableReservation = { duplicate: false }; + mockReserveDuplicate.mockResolvedValueOnce(unavailableReservation); + + await insertTrackEvent( + { anonymousId: "anon_1", eventId: "evt_1", name: "pageview" }, + "ws_test", + "Mozilla/5.0", + "1.2.3.4", + request + ); + + expect(mockSend).toHaveBeenCalledOnce(); + expect(mockRunPromise).toHaveBeenCalledOnce(); + expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( + unavailableReservation + ); + expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); + }); }); diff --git a/apps/basket/src/lib/event-service.ts b/apps/basket/src/lib/event-service.ts index d28d79e30d..e446aa5143 100644 --- a/apps/basket/src/lib/event-service.ts +++ b/apps/basket/src/lib/event-service.ts @@ -169,6 +169,11 @@ export function insertTrackEvent( if (reservation.duplicate) { return; } + if (reservation.retryable) { + throw deliveryUnavailable( + new Error("A concurrent attempt owns this analytics event") + ); + } try { const geoData = await getGeo(ip, request); @@ -248,6 +253,11 @@ export function insertOutgoingLink( if (reservation.duplicate) { return; } + if (reservation.retryable) { + throw deliveryUnavailable( + new Error("A concurrent attempt owns this analytics event") + ); + } try { log.set({ diff --git a/apps/basket/src/lib/security.test.ts b/apps/basket/src/lib/security.test.ts index ed04ade96b..c9b1287d7b 100644 --- a/apps/basket/src/lib/security.test.ts +++ b/apps/basket/src/lib/security.test.ts @@ -186,18 +186,22 @@ describe("duplicate reservations", () => { }); }); - test("retries a stale pending reservation instead of falsely suppressing it", async () => { + test("requires a retry when another request owns a pending reservation", async () => { mockRedisSet.mockResolvedValue(null); mockRedisGet.mockResolvedValue("pending:other-attempt"); const reservation = await reserveDuplicate("evt_1", "track"); - expect(reservation).toMatchObject({ - duplicate: false, - key: "dedup:track:evt_1", - token: undefined, - ttl: 86_400, - }); + expect(reservation).toEqual({ duplicate: false, retryable: true }); + }); + + test("requires a retry when a lost reservation vanishes before it can be read", async () => { + mockRedisSet.mockResolvedValue(null); + mockRedisGet.mockResolvedValue(null); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: false, retryable: true }); }); test("recovers ownership when an ambiguous SET wrote this request's token", async () => { @@ -241,7 +245,7 @@ describe("duplicate reservations", () => { const reservation = await reserveDuplicate("evt_1", "track"); - expect(reservation).toMatchObject({ duplicate: false, ttl: 86_400 }); + expect(reservation).toEqual({ duplicate: false }); expect(mockRedisSet).toHaveBeenCalledTimes(2); expect(mockCaptureError).toHaveBeenCalledOnce(); }); @@ -252,23 +256,47 @@ describe("duplicate reservations", () => { expect(mockRedisEval).not.toHaveBeenCalled(); }); - test("marks a confirmed delivery before suppressing future retries", async () => { - mockRedisSet.mockResolvedValue("OK"); - + test("marks only an owned pending reservation as delivered", async () => { + mockRedisEval.mockResolvedValue("OK"); await markDuplicateReservationDelivered({ duplicate: false, key: "dedup:track:evt_1", + token: "pending:owner-attempt", ttl: 86_400, }); - expect(mockRedisSet).toHaveBeenCalledWith( + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + 1, "dedup:track:evt_1", + "pending:owner-attempt", "delivered", - "EX", 86_400 ); }); + test("does not let a stale delivery promote a newer reservation", async () => { + let storedToken = "pending:newer-attempt"; + mockRedisEval.mockImplementation( + async (_script, _keys, _key, expectedToken: string, deliveredValue: string) => { + if (storedToken === expectedToken) { + storedToken = deliveredValue; + return "OK"; + } + return 0; + } + ); + + await markDuplicateReservationDelivered({ + duplicate: false, + key: "dedup:track:evt_1", + token: "pending:stale-attempt", + ttl: 86_400, + }); + + expect(storedToken).toBe("pending:newer-attempt"); + }); + test("releases only the pending reservation owned by the failed request", async () => { mockRedisEval.mockResolvedValue(1); diff --git a/apps/basket/src/lib/security.ts b/apps/basket/src/lib/security.ts index 84de738f6b..f81af24d8f 100644 --- a/apps/basket/src/lib/security.ts +++ b/apps/basket/src/lib/security.ts @@ -15,6 +15,10 @@ const RELEASE_PENDING_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV return redis.call("DEL", KEYS[1]) end return 0`; +const MARK_PENDING_DEDUP_RESERVATION_DELIVERED = `if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("SET", KEYS[1], ARGV[2], "EX", ARGV[3]) +end +return 0`; const RAW_VISITOR_ID_COUNTRIES = ["US"]; const COUNTRY_CODES: Record = { @@ -132,6 +136,12 @@ function wait(ms: number): Promise { export interface DuplicateReservation { readonly duplicate: boolean; readonly key?: string; + /** + * Redis confirmed that this request did not acquire the reservation. The + * caller must retry instead of publishing alongside its current owner. + * Omitted when Redis is unavailable so ingestion remains fail-open. + */ + readonly retryable?: true; /** * Present only when this request atomically acquired the pending key. It is * required to release the key so an older failed request cannot erase a @@ -211,13 +221,16 @@ export function reserveDuplicate( return { duplicate: true }; } - // A pending key may be a concurrent attempt, an interrupted request, or - // a release that timed out. Retrying the same stable delivery id is safer - // than falsely reporting success for an event that never reached storage. + // This request did not acquire a pending key. Publishing without ownership + // can duplicate a concurrent delivery or overwrite its confirmed state. + if (result === "pending" || result === "unreserved") { + return { duplicate: false, retryable: true }; + } + return { duplicate: false, key, - token: result === "acquired" ? token : undefined, + token, ttl, }; } catch (error) { @@ -226,29 +239,31 @@ export function reserveDuplicate( eventId, eventType, }); - return { duplicate: false, key, ttl }; + return { duplicate: false }; } }); } /** * Only a confirmed Kafka or ClickHouse acknowledgement may turn a pending - * retry guard into a duplicate suppression key. A Redis failure here preserves - * the pending state, which deliberately favors an idempotent retry over loss. + * retry guard into a duplicate suppression key. The promotion is conditional + * on the owner's token so a stale request cannot overwrite a newer reservation. */ export function markDuplicateReservationDelivered( reservation: DuplicateReservation ): Promise { return record("markDuplicateReservationDelivered", async () => { - if (!(reservation.key && reservation.ttl)) { + if (!(reservation.key && reservation.token && reservation.ttl)) { return; } try { - await redis.set( + await redis.eval( + MARK_PENDING_DEDUP_RESERVATION_DELIVERED, + 1, reservation.key, + reservation.token, DELIVERED_DEDUP_VALUE, - "EX", reservation.ttl ); } catch (error) { From 70d4515009d5d708d93897912acd64e220630f8d Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:26:00 +0300 Subject: [PATCH 06/25] test(links): isolate logging mocks --- .agents/skills/databuddy-internal/SKILL.md | 1 + apps/links/src/routes/redirect.route.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/.agents/skills/databuddy-internal/SKILL.md b/.agents/skills/databuddy-internal/SKILL.md index 48709518b5..a9de56cf8f 100644 --- a/.agents/skills/databuddy-internal/SKILL.md +++ b/.agents/skills/databuddy-internal/SKILL.md @@ -60,6 +60,7 @@ Keep additions **minimal**: one bullet, a new `rg` hint, or a routing note—eno - `apps/docs`: Next.js + Fumadocs docs app on port `3005` - When a user drops a prototype, remove only prototype-specific wiring and preserve the existing product surfaces it temporarily reused. - `apps/links`: redirect/link service +- `apps/links` tests share Bun `mock.module` state; mocks of `../lib/logging` must include every export used by later test files, including `setAttributes` for producer imports. - `apps/uptime`: uptime monitoring service - `apps/uptime` BullMQ worker concurrency defaults high for Bun async I/O; do not lower it just because `10_000` looks large. Verify downstream saturation or lock/timeout evidence first. - Public status pages render from `apps/status`; `apps/dashboard` owns status-page management/config UI only. When cleaning public status UX, update shared `@databuddy/ui/uptime` pieces or `apps/status` wrappers instead of redesigning dashboard-only route remnants. diff --git a/apps/links/src/routes/redirect.route.test.ts b/apps/links/src/routes/redirect.route.test.ts index 8c73b101f3..859c649f8b 100644 --- a/apps/links/src/routes/redirect.route.test.ts +++ b/apps/links/src/routes/redirect.route.test.ts @@ -74,6 +74,7 @@ mock.module("../lib/logging", () => ({ captureError: mock(() => {}), mergeWideEvent: mock(() => {}), record: async (_name: string, run: () => Promise | T) => run(), + setAttributes: mock(() => {}), })); mock.module("../lib/producer", () => ({ From c64d70d4aed83fc5b84b659ba470649312bf7950 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:01:24 +0300 Subject: [PATCH 07/25] test(ai): avoid stale Redis barrel mocks --- packages/ai/src/ai/agents/cache.test.ts | 119 +---------------- .../ai/src/ai/mcp/conversation-store.test.ts | 120 +----------------- 2 files changed, 8 insertions(+), 231 deletions(-) diff --git a/packages/ai/src/ai/agents/cache.test.ts b/packages/ai/src/ai/agents/cache.test.ts index 51227ffc0c..33105755d6 100644 --- a/packages/ai/src/ai/agents/cache.test.ts +++ b/packages/ai/src/ai/agents/cache.test.ts @@ -1,3 +1,4 @@ +import * as actualRedis from "@databuddy/redis"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; interface RedisEntry { @@ -44,9 +45,6 @@ const mockEnrichAgentContext = vi.fn( }) => `fresh:${opts.organizationId ?? opts.userId}:${opts.websiteId}` ); -const passthroughCacheable = unknown>(fn: T) => - fn; - vi.mock("@databuddy/auth", () => ({ auth: {}, websitesApi: { @@ -54,120 +52,11 @@ vi.mock("@databuddy/auth", () => ({ }, })); +const redisModule = { ...actualRedis }; + vi.mock("@databuddy/redis", () => ({ - AGENT_CONTEXT_SNAPSHOT_PREFIX: "agent:context-snapshot", - UPTIME_CHECK_JOB_NAME: "uptime-check", - UPTIME_JOB_OPTIONS: {}, - UPTIME_JOB_TIMEOUT_MS: 60_000, - UPTIME_QUEUE_NAME: "uptime-checks", - INSIGHTS_DISPATCH_JOB_NAME: "insights-dispatch", - INSIGHTS_GENERATE_WEBSITE_JOB_NAME: "insights-generate-website", - INSIGHTS_JOB_OPTIONS: {}, - INSIGHTS_JOB_TIMEOUT_MS: 120_000, - INSIGHTS_QUEUE_ENV_PREFIX: "INSIGHTS", - INSIGHTS_QUEUE_NAME: "insights-generation", - activeStreamKey: (id: string) => `active:${id}`, - appendStreamChunk: vi.fn(async () => undefined), - cacheNamespaces: { - agentTelemetryWebsiteExists: "agent-telemetry:website-exists", - apiKeyByHash: "api-key-by-hash", - apiKeyOwnerId: "api_key_owner_id", - billingOwner: "rpc:billing_owner", - flag: "flag", - flagsClient: "flags-client", - flagsDefinitions: "flags-definitions", - flagsUser: "flags-user", - mcpInsights: "mcp:insights", - memberRole: "rpc:member_role", - organizationOwner: "rpc:org_owner", - organizationRole: "rpc:org_role", - slackChannelBinding: "slack-channel-binding", - slackIntegrationByTeam: "slack-integration-by-team", - statusPage: "status-page", - userPreferences: "user-prefs", - websiteById: "website_by_id", - websiteCache: "website-cache", - websiteDomain: "website-domain", - websiteDomainsBatch: "website-domains-batch", - websiteWithOwner: "website_with_owner_v2", - }, - cacheTags: { - billingOwner: (ownerId: string) => `billing-owner:${ownerId}`, - flagClient: (clientId: string) => `flag-client:${clientId}`, - flagKey: (clientId: string, flagKey: string) => - `flag-key:${clientId}:${flagKey}`, - flagUser: (clientId: string, userId: string) => - `flag-user:${clientId}:${userId}`, - organization: (organizationId: string) => `organization:${organizationId}`, - website: (websiteId: string) => `website:${websiteId}`, - }, - cacheable: passthroughCacheable, - clearActiveStream: vi.fn(async () => undefined), - closeInsightsQueue: vi.fn(async () => undefined), - closeUptimeQueue: vi.fn(async () => undefined), - createDrizzleCache: () => ({}), - getActiveStream: vi.fn(async () => null), - getAgentContextSnapshotKey: getSnapshotKey, - getBullMQConnectionOptions: vi.fn(() => ({})), - getBullMQWorkerConnectionOptions: vi.fn(() => ({})), - getCachedLink: vi.fn(async () => null), - getCacheableKey: vi.fn( - (prefix: string, args: unknown[]) => `${prefix}:${JSON.stringify(args)}` - ), - getLinkCacheKey: vi.fn((slug: string) => `link:${slug}`), - getRateLimitHeaders: vi.fn(() => ({})), - getInsightsQueue: vi.fn(() => ({})), + ...redisModule, getRedisCache: () => mockRedisClient, - getUptimeQueue: vi.fn(() => ({})), - invalidateAgentContextSnapshot: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForOwner: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForWebsite: vi.fn(async () => 0), - invalidateCacheableKey: vi.fn(async () => 0), - invalidateCacheablePattern: vi.fn(async () => 0), - invalidateCacheablePrefix: vi.fn(async () => 0), - invalidateCacheableTag: vi.fn(async () => 0), - invalidateCacheableTags: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateCacheableWithArgs: vi.fn(async () => 0), - invalidateBillingOwnerCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateFlagReadCaches: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateInsightsCachesForOrganization: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateLinkCache: vi.fn(async () => undefined), - invalidateLinkCaches: vi.fn(async () => undefined), - invalidateOrganizationMembershipCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateSlackChannelBindingCache: vi.fn(async () => undefined), - invalidateSlackIntegrationCache: vi.fn(async () => undefined), - invalidateStatusPageCache: vi.fn(async () => 0), - invalidateUserPreferencesCache: vi.fn(async () => undefined), - invalidateWebsiteReadCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - enqueueInsightsResume: vi.fn(async () => "queued"), - insightsResumeJobId: (replyId: string) => `insights-reply-${replyId}`, - insightsWebsiteJobId: (runId: string, websiteId: string) => - `insights-website-${runId}-${websiteId}`, - isClickRecorded: vi.fn(async () => false), - markStreamDone: vi.fn(async () => undefined), - ratelimit: vi.fn(async () => ({ success: true })), - readStreamHistory: vi.fn(async () => []), - redis: mockRedisClient, - setActiveStream: vi.fn(async () => undefined), - setCachedLink: vi.fn(async () => undefined), - setCachedLinkNotFound: vi.fn(async () => undefined), - shouldRecordClick: vi.fn(async () => true), - shutdownRedis: vi.fn(async () => undefined), - streamBufferKey: (id: string) => `stream:${id}`, - uptimeImmediateJobId: (id: string) => `uptime:immediate:${id}`, - uptimeSchedulerId: (id: string) => `uptime:scheduler:${id}`, })); vi.mock("../../lib/supermemory", () => ({ diff --git a/packages/ai/src/ai/mcp/conversation-store.test.ts b/packages/ai/src/ai/mcp/conversation-store.test.ts index db6f8614f0..4edcb70fe9 100644 --- a/packages/ai/src/ai/mcp/conversation-store.test.ts +++ b/packages/ai/src/ai/mcp/conversation-store.test.ts @@ -1,3 +1,4 @@ +import * as actualRedis from "@databuddy/redis"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const redisStore = new Map(); @@ -21,129 +22,16 @@ const mockRedisClient = { }), }; +const redisModule = { ...actualRedis }; + vi.mock("@databuddy/redis", () => ({ - AGENT_CONTEXT_SNAPSHOT_PREFIX: "agent:context-snapshot", - UPTIME_CHECK_JOB_NAME: "uptime-check", - UPTIME_JOB_OPTIONS: {}, - UPTIME_JOB_TIMEOUT_MS: 60_000, - UPTIME_QUEUE_NAME: "uptime-checks", - INSIGHTS_DISPATCH_JOB_NAME: "insights-dispatch", - INSIGHTS_GENERATE_WEBSITE_JOB_NAME: "insights-generate-website", - INSIGHTS_JOB_OPTIONS: {}, - INSIGHTS_JOB_TIMEOUT_MS: 120_000, - INSIGHTS_QUEUE_ENV_PREFIX: "INSIGHTS", - INSIGHTS_QUEUE_NAME: "insights-generation", - activeStreamKey: (id: string) => `active:${id}`, - appendStreamChunk: vi.fn(async () => undefined), - cacheNamespaces: { - agentTelemetryWebsiteExists: "agent-telemetry:website-exists", - apiKeyByHash: "api-key-by-hash", - apiKeyOwnerId: "api_key_owner_id", - billingOwner: "rpc:billing_owner", - flag: "flag", - flagsClient: "flags-client", - flagsDefinitions: "flags-definitions", - flagsUser: "flags-user", - mcpInsights: "mcp:insights", - memberRole: "rpc:member_role", - organizationOwner: "rpc:org_owner", - organizationRole: "rpc:org_role", - slackChannelBinding: "slack-channel-binding", - slackIntegrationByTeam: "slack-integration-by-team", - statusPage: "status-page", - userPreferences: "user-prefs", - websiteById: "website_by_id", - websiteCache: "website-cache", - websiteDomain: "website-domain", - websiteDomainsBatch: "website-domains-batch", - websiteWithOwner: "website_with_owner_v2", - }, - cacheTags: { - billingOwner: (ownerId: string) => `billing-owner:${ownerId}`, - flagClient: (clientId: string) => `flag-client:${clientId}`, - flagKey: (clientId: string, flagKey: string) => - `flag-key:${clientId}:${flagKey}`, - flagUser: (clientId: string, userId: string) => - `flag-user:${clientId}:${userId}`, - organization: (organizationId: string) => `organization:${organizationId}`, - website: (websiteId: string) => `website:${websiteId}`, - }, - cacheable: unknown>(fn: T) => fn, - clearActiveStream: vi.fn(async () => undefined), - closeInsightsQueue: vi.fn(async () => undefined), - closeUptimeQueue: vi.fn(async () => undefined), - createDrizzleCache: () => ({}), - getActiveStream: vi.fn(async () => null), - getAgentContextSnapshotKey: ( - userId: string, - websiteId: string, - organizationId?: string | null - ) => `agent:context-snapshot:${organizationId ?? userId}:${websiteId}`, - getBullMQConnectionOptions: vi.fn(() => ({})), - getBullMQWorkerConnectionOptions: vi.fn(() => ({})), - getCachedLink: vi.fn(async () => null), - getCacheableKey: vi.fn( - (prefix: string, args: unknown[]) => `${prefix}:${JSON.stringify(args)}` - ), - getLinkCacheKey: vi.fn((slug: string) => `link:${slug}`), - getRateLimitHeaders: vi.fn(() => ({})), + ...redisModule, getRedisCache: () => { if (redisUnavailable) { throw new Error("redis unavailable"); } return mockRedisClient; }, - getInsightsQueue: vi.fn(() => ({})), - getUptimeQueue: vi.fn(() => ({})), - invalidateAgentContextSnapshot: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForOwner: vi.fn(async () => 0), - invalidateAgentContextSnapshotsForWebsite: vi.fn(async () => 0), - invalidateCacheableKey: vi.fn(async () => 0), - invalidateCacheablePattern: vi.fn(async () => 0), - invalidateCacheablePrefix: vi.fn(async () => 0), - invalidateCacheableTag: vi.fn(async () => 0), - invalidateCacheableTags: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateCacheableWithArgs: vi.fn(async () => 0), - invalidateBillingOwnerCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateFlagReadCaches: vi.fn(async () => ({ attempted: 0, failed: 0 })), - invalidateInsightsCachesForOrganization: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateLinkCache: vi.fn(async () => undefined), - invalidateLinkCaches: vi.fn(async () => undefined), - invalidateOrganizationMembershipCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - invalidateSlackChannelBindingCache: vi.fn(async () => undefined), - invalidateSlackIntegrationCache: vi.fn(async () => undefined), - invalidateStatusPageCache: vi.fn(async () => 0), - invalidateUserPreferencesCache: vi.fn(async () => undefined), - invalidateWebsiteReadCaches: vi.fn(async () => ({ - attempted: 0, - failed: 0, - })), - enqueueInsightsResume: vi.fn(async () => "queued"), - insightsResumeJobId: (replyId: string) => `insights-reply-${replyId}`, - insightsWebsiteJobId: (runId: string, websiteId: string) => - `insights-website-${runId}-${websiteId}`, - isClickRecorded: vi.fn(async () => false), - markStreamDone: vi.fn(async () => undefined), - ratelimit: vi.fn(async () => ({ success: true })), - readStreamHistory: vi.fn(async () => []), - redis: mockRedisClient, - setActiveStream: vi.fn(async () => undefined), - setCachedLink: vi.fn(async () => undefined), - setCachedLinkNotFound: vi.fn(async () => undefined), - shouldRecordClick: vi.fn(async () => true), - shutdownRedis: vi.fn(async () => undefined), - streamBufferKey: (id: string) => `stream:${id}`, - uptimeImmediateJobId: (id: string) => `uptime:immediate:${id}`, - uptimeSchedulerId: (id: string) => `uptime:scheduler:${id}`, })); const { appendToConversation, getConversationHistory } = await import( From 53f4606ae725ef5501a73d8531047b1b14ed728b Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:01:35 +0300 Subject: [PATCH 08/25] fix(basket): preserve retryable analytics delivery --- .../src/lib/event-service.delivery.test.ts | 91 ++++++++++++++- apps/basket/src/lib/event-service.ts | 108 +++++++++++++++--- apps/basket/src/routes/basket.ts | 54 +++++---- apps/basket/src/routes/integration.test.ts | 64 ++++++++++- apps/basket/src/utils/pixel.test.ts | 5 +- apps/basket/src/utils/pixel.ts | 19 ++- 6 files changed, 293 insertions(+), 48 deletions(-) diff --git a/apps/basket/src/lib/event-service.delivery.test.ts b/apps/basket/src/lib/event-service.delivery.test.ts index 5e4d0d0df5..dda91d9562 100644 --- a/apps/basket/src/lib/event-service.delivery.test.ts +++ b/apps/basket/src/lib/event-service.delivery.test.ts @@ -11,6 +11,7 @@ const { mockReserveDuplicate, mockRunPromise, mockSend, + mockSendBatch, mockShouldAnonymizeVisitorIds, } = vi.hoisted(() => ({ mockApplyVisitorIdPrivacy: vi.fn((value: unknown) => String(value ?? "")), @@ -36,13 +37,14 @@ const { ), mockRunPromise: vi.fn(() => Promise.resolve()), mockSend: vi.fn(() => ({ _tag: "MockEffect" })), + mockSendBatch: vi.fn(() => ({ _tag: "MockEffect" })), mockShouldAnonymizeVisitorIds: vi.fn(() => false), })); vi.mock("@lib/producer", () => ({ runPromise: mockRunPromise, send: mockSend, - sendBatch: vi.fn(() => ({ _tag: "MockEffect" })), + sendBatch: mockSendBatch, })); vi.mock("@lib/security", () => ({ @@ -71,8 +73,13 @@ vi.mock("evlog/elysia", () => ({ useLogger: () => ({ set: mockLoggerSet }), })); -const { insertOutgoingLink, insertTrackEvent, stableAnalyticsEventId } = - await import("./event-service"); +const { + buildTrackEvent, + insertOutgoingLink, + insertTrackEvent, + insertTrackEventsBatch, + stableAnalyticsEventId, +} = await import("./event-service"); const request = new Request("http://localhost/ingest"); const trackReservation = { @@ -82,6 +89,23 @@ const trackReservation = { ttl: 86_400, }; +function trackBatchItem(eventId: string) { + return { + event: buildTrackEvent( + { anonymousId: "anon_1", eventId, name: "pageview" }, + { + anonymousId: "anon_1", + clientId: "ws_test", + eventId, + geo: { anonymizedIP: "anonymous-ip" }, + now: 1, + ua: {}, + } + ), + sourceEventId: eventId, + }; +} + describe("core event delivery", () => { beforeEach(() => { vi.clearAllMocks(); @@ -95,6 +119,7 @@ describe("core event delivery", () => { mockReserveDuplicate.mockResolvedValue(trackReservation); mockRunPromise.mockResolvedValue(undefined); mockSend.mockReturnValue({ _tag: "MockEffect" }); + mockSendBatch.mockReturnValue({ _tag: "MockEffect" }); mockShouldAnonymizeVisitorIds.mockReturnValue(false); }); @@ -288,4 +313,64 @@ describe("core event delivery", () => { ); expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); }); + + test("delivers one logical batch event and promotes its reservation after acknowledgement", async () => { + const first = trackBatchItem("evt_1"); + const duplicate = trackBatchItem("evt_1"); + + await insertTrackEventsBatch([first, duplicate]); + + expect(mockReserveDuplicate).toHaveBeenCalledOnce(); + expect(mockReserveDuplicate).toHaveBeenCalledWith( + first.event.id, + "track", + "evt_1" + ); + expect(mockSendBatch).toHaveBeenCalledWith("analytics-events", [first.event]); + expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("releases acquired batch reservations when another request owns an item", async () => { + mockReserveDuplicate + .mockResolvedValueOnce(trackReservation) + .mockResolvedValueOnce({ duplicate: false, retryable: true }); + + await expect( + insertTrackEventsBatch([trackBatchItem("evt_1"), trackBatchItem("evt_2")]) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSendBatch).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("releases acquired batch reservations when reservation lookup fails", async () => { + mockReserveDuplicate + .mockResolvedValueOnce(trackReservation) + .mockRejectedValueOnce(new Error("Redis lookup failed")); + + await expect( + insertTrackEventsBatch([trackBatchItem("evt_1"), trackBatchItem("evt_2")]) + ).rejects.toMatchObject({ status: 503 }); + + expect(mockSendBatch).not.toHaveBeenCalled(); + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); + + test("releases batch reservations when durable delivery fails", async () => { + mockRunPromise.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + + await expect(insertTrackEventsBatch([trackBatchItem("evt_1")])).rejects.toMatchObject({ + status: 503, + }); + + expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith( + trackReservation + ); + }); }); diff --git a/apps/basket/src/lib/event-service.ts b/apps/basket/src/lib/event-service.ts index e446aa5143..39e10dd00b 100644 --- a/apps/basket/src/lib/event-service.ts +++ b/apps/basket/src/lib/event-service.ts @@ -51,6 +51,11 @@ export interface TrackEventContext { }; } +export interface BatchEvent { + event: T; + sourceEventId: string; +} + function deliveryUnavailable(cause: unknown) { return createError({ code: "basket.DELIVERY_UNAVAILABLE", @@ -216,11 +221,12 @@ export function insertTrackEvent( }); await runPromise(send("analytics-events", trackEvent)); - await markDuplicateReservationDelivered(reservation); } catch (error) { await releaseDuplicateReservation(reservation); throw deliveryUnavailable(error); } + + await markDuplicateReservationDelivered(reservation); }); } @@ -296,22 +302,94 @@ export function insertOutgoingLink( }; await runPromise(send("analytics-outgoing-links", outgoingLinkEvent)); - await markDuplicateReservationDelivered(reservation); } catch (error) { await releaseDuplicateReservation(reservation); throw deliveryUnavailable(error); } + + await markDuplicateReservationDelivered(reservation); }); } -export function insertTrackEventsBatch(events: EventsInsert[]): Promise { - return record("insertTrackEventsBatch", async () => { - if (events.length === 0) { - return; - } +async function deliverBatch( + eventType: "outgoing_link" | "track", + topic: string, + items: BatchEvent[] +): Promise { + const uniqueItems = Array.from( + new Map(items.map((item) => [item.event.id, item])).values() + ); + if (uniqueItems.length === 0) { + return; + } + + const reservationResults = await Promise.allSettled( + uniqueItems.map(async (item) => ({ + item, + reservation: await reserveDuplicate( + item.event.id, + eventType, + item.sourceEventId + ), + })) + ); + const reservations = reservationResults.flatMap((result) => + result.status === "fulfilled" ? [result.value] : [] + ); + const reservationFailure = reservationResults.find( + (result): result is PromiseRejectedResult => result.status === "rejected" + ); + const retryableReservation = reservations.find( + ({ reservation }) => reservation.retryable + ); + if (reservationFailure || retryableReservation) { + await Promise.allSettled( + reservations.map(({ reservation }) => + releaseDuplicateReservation(reservation) + ) + ); + throw deliveryUnavailable( + reservationFailure?.reason ?? + new Error("A concurrent attempt owns this analytics event") + ); + } + + const accepted = reservations.filter( + ({ reservation }) => !reservation.duplicate + ); + if (accepted.length === 0) { + return; + } + + try { + await runPromise( + sendBatch( + topic, + accepted.map(({ item }) => item.event) + ) + ); + } catch (error) { + await Promise.allSettled( + accepted.map(({ reservation }) => + releaseDuplicateReservation(reservation) + ) + ); + throw deliveryUnavailable(error); + } + + await Promise.allSettled( + accepted.map(({ reservation }) => + markDuplicateReservationDelivered(reservation) + ) + ); +} - await runPromise(sendBatch("analytics-events", events)); - }); +export function insertTrackEventsBatch( + events: BatchEvent[] +): Promise { + return record("insertTrackEventsBatch", () => + deliverBatch("track", "analytics-events", events) + ); } export function insertErrorSpans( @@ -399,15 +477,11 @@ export function insertIndividualVitals( } export function insertOutgoingLinksBatch( - events: OutgoingLinksInsert[] + events: BatchEvent[] ): Promise { - return record("insertOutgoingLinksBatch", async () => { - if (events.length === 0) { - return; - } - - await runPromise(sendBatch("analytics-outgoing-links", events)); - }); + return record("insertOutgoingLinksBatch", () => + deliverBatch("outgoing_link", "analytics-outgoing-links", events) + ); } export function insertCustomEvents( diff --git a/apps/basket/src/routes/basket.ts b/apps/basket/src/routes/basket.ts index 488f7772e1..e8afd2dd2d 100644 --- a/apps/basket/src/routes/basket.ts +++ b/apps/basket/src/routes/basket.ts @@ -25,6 +25,7 @@ import { insertTrackEvent, insertTrackEventsBatch, stableAnalyticsEventId, + type BatchEvent, } from "@lib/event-service"; import { summarizeRejectedBody } from "@lib/rejection-summary"; import { @@ -98,14 +99,17 @@ function processTrackEventData( salt ); - return buildTrackEvent(trackData, { - clientId, - eventId, - anonymousId, - geo: geoData, - ua, - now: Date.now(), - }); + return { + event: buildTrackEvent(trackData, { + clientId, + eventId, + anonymousId, + geo: geoData, + ua, + now: Date.now(), + }), + sourceEventId: eventId, + }; }); } @@ -113,7 +117,7 @@ async function processOutgoingLinkData( linkData: OutgoingLinkInput, clientId: string, visitorCountry?: unknown -): Promise { +): Promise> { const eventId = parseEventId(linkData.eventId, () => randomUUIDv7()); const timestamp = parseTimestamp(linkData.timestamp); const anonymizeVisitorIds = shouldAnonymizeVisitorIds( @@ -129,14 +133,17 @@ async function processOutgoingLinkData( ); return { - id: stableAnalyticsEventId(clientId, "outgoing_link", eventId), - client_id: clientId, - anonymous_id: anonymousId, - session_id: validateSessionId(linkData.sessionId), - href: sanitizeString(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), - text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), - properties: parseProperties(linkData.properties), - timestamp, + event: { + id: stableAnalyticsEventId(clientId, "outgoing_link", eventId), + client_id: clientId, + anonymous_id: anonymousId, + session_id: validateSessionId(linkData.sessionId), + href: sanitizeString(linkData.href, VALIDATION_LIMITS.PATH_MAX_LENGTH), + text: sanitizeString(linkData.text, VALIDATION_LIMITS.TEXT_MAX_LENGTH), + properties: parseProperties(linkData.properties), + timestamp, + }, + sourceEventId: eventId, }; } @@ -492,8 +499,8 @@ const app = new Elysia() const { clientId, userAgent, ip } = validation; log.set({ clientId }); - const trackEvents: EventsInsert[] = []; - const outgoingLinkEvents: OutgoingLinksInsert[] = []; + const trackEvents: BatchEvent[] = []; + const outgoingLinkEvents: BatchEvent[] = []; const results: Record[] = []; let batchVisitorCountry: string | undefined; let hasResolvedBatchVisitorCountry = false; @@ -622,10 +629,17 @@ const app = new Elysia() } } - await Promise.all([ + const deliveryResults = await Promise.allSettled([ insertTrackEventsBatch(trackEvents), insertOutgoingLinksBatch(outgoingLinkEvents), ]); + const deliveryFailure = deliveryResults.find( + (result): result is PromiseRejectedResult => + result.status === "rejected" + ); + if (deliveryFailure) { + throw deliveryFailure.reason; + } log.set({ processed: results.length, diff --git a/apps/basket/src/routes/integration.test.ts b/apps/basket/src/routes/integration.test.ts index 47fe92a359..6c8bf5f7a7 100644 --- a/apps/basket/src/routes/integration.test.ts +++ b/apps/basket/src/routes/integration.test.ts @@ -1,3 +1,4 @@ +import { createError } from "evlog"; import { vi, beforeEach, describe, expect, test } from "vitest"; const { @@ -444,6 +445,11 @@ describe("POST /events", () => { // ── POST /batch ── describe("POST /batch", () => { + beforeEach(() => { + mockInsertOutgoingLinksBatch.mockClear(); + mockInsertTrackEventsBatch.mockClear(); + }); + test("batch of track events → 200", async () => { const res = await post(basketApp, "/batch", [ { @@ -463,6 +469,59 @@ describe("POST /batch", () => { const body = await json(res); expect(body.batch).toBe(true); expect(body.processed).toBe(2); + expect(mockInsertTrackEventsBatch).toHaveBeenCalledWith([ + { + event: expect.objectContaining({ id: "built_id" }), + sourceEventId: "evt_1", + }, + { + event: expect.objectContaining({ id: "built_id" }), + sourceEventId: "evt_2", + }, + ]); + }); + + test("waits for both delivery topics before returning a retryable failure", async () => { + let releaseOutgoingDelivery: (() => void) | undefined; + let responseSettled = false; + mockInsertTrackEventsBatch.mockRejectedValueOnce( + createError({ + message: "Analytics delivery temporarily unavailable", + status: 503, + }) + ); + mockInsertOutgoingLinksBatch.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseOutgoingDelivery = resolve; + }) + ); + + const response = post(basketApp, "/batch", [ + { + type: "track", + eventId: "evt_1", + name: "pageview", + path: "https://example.com/a", + }, + { + type: "outgoing_link", + eventId: "evt_link_1", + href: "https://external.com", + }, + ]); + void response.then(() => { + responseSettled = true; + }); + + await vi.waitFor(() => + expect(mockInsertOutgoingLinksBatch).toHaveBeenCalledOnce() + ); + expect(responseSettled).toBe(false); + + releaseOutgoingDelivery?.(); + const result = await response; + expect(result.status).toBe(503); }); test("not an array → 400", async () => { @@ -530,12 +589,13 @@ describe("GET /px.jpg", () => { expect(res.headers.get("Content-Type")).toBe("image/gif"); }); - test("returns a retryable GIF when delivery fails", async () => { + test("returns an empty retryable response when delivery fails", async () => { mockValidateRequest.mockRejectedValueOnce(new Error("boom")); const res = await get(basketApp, "/px.jpg?name=test"); expect(res.status).toBe(503); - expect(res.headers.get("Content-Type")).toBe("image/gif"); + expect(res.headers.get("Content-Type")).toBeNull(); expect(res.headers.get("Retry-After")).toBe("5"); + expect(new Uint8Array(await res.arrayBuffer())).toHaveLength(0); }); }); diff --git a/apps/basket/src/utils/pixel.test.ts b/apps/basket/src/utils/pixel.test.ts index c045ec64af..cae9b1a25e 100644 --- a/apps/basket/src/utils/pixel.test.ts +++ b/apps/basket/src/utils/pixel.test.ts @@ -20,11 +20,12 @@ describe("createPixelResponse", () => { ); }); - test("keeps the GIF body but marks delivery failures retryable", () => { + test("returns an empty retryable response so image transport can retry", async () => { const r = createPixelResponse(503); expect(r.status).toBe(503); - expect(r.headers.get("Content-Type")).toBe("image/gif"); + expect(r.headers.get("Content-Type")).toBeNull(); expect(r.headers.get("Retry-After")).toBe("5"); + expect(new Uint8Array(await r.arrayBuffer())).toHaveLength(0); }); }); diff --git a/apps/basket/src/utils/pixel.ts b/apps/basket/src/utils/pixel.ts index 2f4a5826f2..91a5d0d259 100644 --- a/apps/basket/src/utils/pixel.ts +++ b/apps/basket/src/utils/pixel.ts @@ -14,14 +14,25 @@ const TRANSPARENT_PIXEL = Buffer.from( * Returns a 1x1 transparent GIF response */ export function createPixelResponse(status = 200): Response { + const headers = { + "Cache-Control": "no-cache, no-store, must-revalidate", + Pragma: "no-cache", + Expires: "0", + ...(status >= 500 ? { "Retry-After": "5" } : {}), + }; + + // Browsers report a decodable GIF as an image load even when its HTTP status + // is 5xx. Return no image body for failures so pixel transport receives an + // error event and retains the event for retry. + if (status >= 400) { + return new Response(null, { status, headers }); + } + return new Response(TRANSPARENT_PIXEL, { status, headers: { "Content-Type": "image/gif", - "Cache-Control": "no-cache, no-store, must-revalidate", - Pragma: "no-cache", - Expires: "0", - ...(status >= 500 ? { "Retry-After": "5" } : {}), + ...headers, }, }); } From 0484991b2da6978c96f878464de2e0d2e8f9b60f Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:01:42 +0300 Subject: [PATCH 09/25] refactor(basket): remove ephemeral delivery buffer --- apps/basket/src/index.ts | 6 +- apps/basket/src/lib/producer.delivery.test.ts | 98 +----- apps/basket/src/lib/producer.kafka.test.ts | 3 +- apps/basket/src/lib/producer.test.ts | 15 +- apps/basket/src/lib/producer.ts | 283 +----------------- 5 files changed, 39 insertions(+), 366 deletions(-) diff --git a/apps/basket/src/index.ts b/apps/basket/src/index.ts index c89df0b823..0d87a76a4c 100644 --- a/apps/basket/src/index.ts +++ b/apps/basket/src/index.ts @@ -84,8 +84,7 @@ async function gracefulShutdown(signal: string, exitCode = 0) { error_message: error instanceof Error ? error.message : String(error), }); const { shutdownRedis } = await import("@databuddy/redis"); - // A buffered fallback can still need ClickHouse after Kafka is unavailable. - // Drain it before flushing its failure telemetry or tearing down services. + // Wait for acknowledged delivery before tearing down its dependencies. try { await runPromise(disconnect); } catch (error) { @@ -94,9 +93,8 @@ async function gracefulShutdown(signal: string, exitCode = 0) { log.error({ lifecycle: "producerDrain", error_message: - "Basket producer drain timed out with undelivered events", + "Basket producer drain timed out waiting for in-flight delivery", in_flight: error.inFlight, - residual_count: error.residualCount, drain_timeout_ms: error.deadlineMs, }); } else { diff --git a/apps/basket/src/lib/producer.delivery.test.ts b/apps/basket/src/lib/producer.delivery.test.ts index 554ecc1c39..3c67a47bf1 100644 --- a/apps/basket/src/lib/producer.delivery.test.ts +++ b/apps/basket/src/lib/producer.delivery.test.ts @@ -31,9 +31,6 @@ const { createProducerEffects } = await import("./producer"); const topicMap = { "analytics-events": "analytics.events" }; const baseConfig: ProducerConfig = { - bufferHardMax: 2, - bufferInterval: 60_000, - bufferMax: 1, broker: undefined, chunkSize: 100, kafkaTimeout: 1_000, @@ -42,7 +39,6 @@ const baseConfig: ProducerConfig = { producerRetryDelay: 1, reconnectCooldown: 1, selfHost: true, - shutdownDrainRetryDelay: 1, shutdownDrainTimeout: 50, username: undefined, }; @@ -82,7 +78,7 @@ describe("producer delivery guarantees", () => { }) ); const stats = await Effect.runPromise(effects.stats); - expect(stats).toMatchObject({ bufferSize: 0, flushed: 1 }); + expect(stats).toMatchObject({ inFlight: 0, sent: 1 }); }); test("returns a retryable error instead of acknowledging a failed direct fallback", async () => { @@ -98,77 +94,7 @@ describe("producer delivery guarantees", () => { }); const stats = await Effect.runPromise(effects.stats); - expect(stats).toMatchObject({ bufferSize: 0, errors: 1 }); - }); - - test("rejects buffered delivery atomically when the in-memory buffer is full", async () => { - const effects = await makeEffects(() => Promise.resolve()); - - await Effect.runPromise( - effects.sendManyBuffered("analytics-events", [event("event_1"), event("event_2")]) - ); - await expect( - Effect.runPromise( - effects.sendOneBuffered("analytics-events", event("event_3")) - ) - ).rejects.toMatchObject({ - _tag: "BufferOverflowError", - bufferHardMax: 2, - bufferLength: 2, - eventCount: 1, - retryable: true, - }); - - const stats = await Effect.runPromise(effects.stats); - expect(stats).toMatchObject({ bufferSize: 2, dropped: 0 }); - }); - - test("drains every buffered event before shutdown completes", async () => { - const insert = vi.fn(() => Promise.resolve()); - const effects = await makeEffects(insert, { bufferHardMax: 3 }); - - await Effect.runPromise( - effects.sendManyBuffered("analytics-events", [ - event("event_1"), - event("event_2"), - event("event_3"), - ]) - ); - await Effect.runPromise(effects.shutDown); - - expect(insert).toHaveBeenCalledTimes(3); - const stats = await Effect.runPromise(effects.stats); - expect(stats).toMatchObject({ bufferSize: 0, flushed: 3 }); - }); - - test("rejects new events with a retryable error once shutdown begins", async () => { - let releaseInsert: (() => void) | undefined; - const effects = await makeEffects( - () => - new Promise((resolve) => { - releaseInsert = resolve; - }), - { shutdownDrainTimeout: 500 } - ); - - await Effect.runPromise( - effects.sendOneBuffered("analytics-events", event("event_1")) - ); - const shutdown = Effect.runPromise(effects.shutDown); - await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); - - await expect( - Effect.runPromise( - effects.sendOneBuffered("analytics-events", event("event_2")) - ) - ).rejects.toMatchObject({ - _tag: "ProducerShuttingDownError", - eventCount: 1, - retryable: true, - }); - - releaseInsert?.(); - await shutdown; + expect(stats).toMatchObject({ errors: 1, inFlight: 0, sent: 0 }); }); test("keeps an active direct delivery counted when shutdown rejects another send", async () => { @@ -202,23 +128,27 @@ describe("producer delivery guarantees", () => { await shutdown; }); - test("reports retained residual events when shutdown cannot drain before its deadline", async () => { + test("reports an in-flight direct delivery when shutdown reaches its deadline", async () => { + let releaseInsert: (() => void) | undefined; const effects = await makeEffects( - () => Promise.reject(new Error("ClickHouse unavailable")), + () => + new Promise((resolve) => { + releaseInsert = resolve; + }), { shutdownDrainTimeout: 20 } ); - await Effect.runPromise( - effects.sendOneBuffered("analytics-events", event("event_1")) + const activeDelivery = Effect.runPromise( + effects.sendOne("analytics-events", event("event_1")) ); + await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); await expect(Effect.runPromise(effects.shutDown)).rejects.toMatchObject({ _tag: "ShutdownDrainError", - inFlight: 0, - residualCount: 1, + inFlight: 1, retryable: true, }); - const stats = await Effect.runPromise(effects.stats); - expect(stats.bufferSize).toBe(1); + releaseInsert?.(); + await activeDelivery; }); }); diff --git a/apps/basket/src/lib/producer.kafka.test.ts b/apps/basket/src/lib/producer.kafka.test.ts index 13c32bb972..9e8b38ef22 100644 --- a/apps/basket/src/lib/producer.kafka.test.ts +++ b/apps/basket/src/lib/producer.kafka.test.ts @@ -117,8 +117,7 @@ describe("producer Kafka send failure handling", () => { expect(mockConnect).toHaveBeenCalledTimes(1); expect(mockSend).toHaveBeenCalledTimes(1); expect(mockClickHouseInsert).toHaveBeenCalledTimes(2); - expect(stats?.bufferSize).toBe(0); - expect(stats?.flushed).toBe(2); + expect(stats?.sent).toBe(2); expect(stats?.connected).toBe(false); expect(stats?.failed).toBe(true); expect(stats?.failedCount).toBe(1); diff --git a/apps/basket/src/lib/producer.test.ts b/apps/basket/src/lib/producer.test.ts index 8ad1747984..09532266fa 100644 --- a/apps/basket/src/lib/producer.test.ts +++ b/apps/basket/src/lib/producer.test.ts @@ -34,7 +34,6 @@ const { getStats, runPromise, send, - sendBuffered, } = await import("./producer"); beforeEach(async () => { @@ -52,9 +51,9 @@ afterAll(async () => { }); describe("producer fallback topics", () => { - test("blocked traffic is buffered for ClickHouse fallback", async () => { + test("blocked traffic is delivered through the direct ClickHouse fallback", async () => { await runPromise( - sendBuffered("analytics-blocked-traffic", { + send("analytics-blocked-traffic", { id: "blocked_1", client_id: "ws_1", timestamp: Date.now(), @@ -63,7 +62,15 @@ describe("producer fallback topics", () => { const stats = await runPromise(getStats); expect(stats?.errors).toBe(0); - expect(stats?.bufferSize).toBe(1); + expect(stats?.sent).toBe(1); + expect(mockClickHouseInsert).toHaveBeenCalledWith( + expect.objectContaining({ + table: "analytics.blocked_traffic", + values: [ + expect.objectContaining({ id: "blocked_1", client_id: "ws_1" }), + ], + }) + ); }); test("unknown topics include the missing topic in error context", async () => { diff --git a/apps/basket/src/lib/producer.ts b/apps/basket/src/lib/producer.ts index b681624b58..522bb8a3e4 100644 --- a/apps/basket/src/lib/producer.ts +++ b/apps/basket/src/lib/producer.ts @@ -2,7 +2,7 @@ import type { ClickHouseClient } from "@clickhouse/client"; import { clickHouse, TABLE_NAMES } from "@databuddy/db/clickhouse"; import { readBooleanEnv } from "@databuddy/env/boolean"; import { captureError, record } from "@lib/tracing"; -import { Data, Effect, Layer, ManagedRuntime, Ref, Schedule } from "effect"; +import { Data, Effect, Layer, ManagedRuntime, Ref } from "effect"; import { createError } from "evlog"; import { CompressionTypes, Kafka, type Producer } from "kafkajs"; @@ -19,15 +19,6 @@ export class KafkaSendError extends Data.TaggedError("KafkaSendError")<{ readonly topic: string; readonly cause?: Error; }> {} -export class BufferOverflowError extends Data.TaggedError( - "BufferOverflowError" -)<{ - readonly bufferHardMax: number; - readonly bufferLength: number; - readonly eventCount: number; - readonly retryable: true; - readonly topic: string; -}> {} export class ProducerShuttingDownError extends Data.TaggedError( "ProducerShuttingDownError" )<{ @@ -51,42 +42,25 @@ export class ClickHouseFallbackError extends Data.TaggedError( readonly table: string; readonly topic: string; }> {} -export class FlushError extends Data.TaggedError("FlushError")<{ - readonly table: string; - readonly cause?: Error; -}> {} export class ShutdownDrainError extends Data.TaggedError("ShutdownDrainError")<{ readonly deadlineMs: number; readonly inFlight: number; - readonly residualCount: number; readonly retryable: true; }> {} export type ProducerError = | KafkaConnectionError | KafkaSendError - | BufferOverflowError - | FlushError | ProducerShuttingDownError | ProducerUnavailableError | UnknownKafkaTopicError | ClickHouseFallbackError; -interface BufferedEvent { - event: unknown; - table: string; -} - interface ProducerState { - buffer: BufferedEvent[]; - buffered: number; connected: boolean; connectionFailed: boolean; - dropped: number; errors: number; failedCount: number; - flushed: number; - flushing: boolean; inFlight: number; lastErrorTime: number | null; lastRetry: number; @@ -97,9 +71,6 @@ interface ProducerState { export interface ProducerConfig { broker?: string; - bufferHardMax: number; - bufferInterval: number; - bufferMax: number; chunkSize: number; kafkaTimeout: number; maxProducerRetries: number; @@ -107,42 +78,27 @@ export interface ProducerConfig { producerRetryDelay: number; reconnectCooldown: number; selfHost: boolean; - shutdownDrainRetryDelay: number; shutdownDrainTimeout: number; username?: string; } export interface ProducerEffects { - flush: Effect.Effect; sendMany: ( topic: string, events: unknown[] ) => Effect.Effect; - sendManyBuffered: ( - topic: string, - events: unknown[] - ) => Effect.Effect; sendOne: ( topic: string, event: unknown, key?: string ) => Effect.Effect; - sendOneBuffered: ( - topic: string, - event: unknown, - key?: string - ) => Effect.Effect; shutDown: Effect.Effect; stats: Effect.Effect; } const INITIAL_STATE: ProducerState = { - buffer: [], sent: 0, failedCount: 0, - buffered: 0, - flushed: 0, - dropped: 0, errors: 0, lastErrorTime: null, connected: false, @@ -150,7 +106,6 @@ const INITIAL_STATE: ProducerState = { lastRetry: 0, producerInitialized: false, shuttingDown: false, - flushing: false, inFlight: 0, }; @@ -158,21 +113,6 @@ function toError(err: unknown): Error { return err instanceof Error ? err : new Error(String(err)); } -function groupBufferedEvents( - items: BufferedEvent[] -): Map { - const grouped = new Map(); - for (const item of items) { - const tableEvents = grouped.get(item.table); - if (tableEvents) { - tableEvents.push(item); - continue; - } - grouped.set(item.table, [item]); - } - return grouped; -} - async function insertClickHouseChunks( ch: ClickHouseClient, table: string, @@ -251,68 +191,6 @@ function makeProducerEffects( ); }); - const flush: Effect.Effect = Effect.gen(function* () { - const items = yield* Ref.modify(ref, (s) => { - if (s.buffer.length === 0 || s.flushing) { - return [null, s] as const; - } - - const batchSize = Math.min(s.buffer.length, config.bufferMax); - return [s.buffer.slice(0, batchSize), { ...s, flushing: true }] as const; - }); - - if (!items) { - return true; - } - - const grouped = groupBufferedEvents(items); - return yield* Effect.forEach( - Array.from(grouped.entries()), - ([table, entries]) => { - const events = entries.map((entry) => entry.event); - const flushedEntries = new Set(entries); - return Effect.tryPromise({ - try: () => - record("clickhouseFallbackInsert", () => - insertClickHouseChunks(ch, table, events, config.chunkSize) - ), - catch: (e) => new FlushError({ table, cause: toError(e) }), - }).pipe( - Effect.tap(() => - Ref.update(ref, (s) => ({ - ...s, - buffer: s.buffer.filter((entry) => !flushedEntries.has(entry)), - flushed: s.flushed + entries.length, - })) - ), - Effect.as(true), - Effect.catchTag("FlushError", (error) => - Ref.update(ref, (s) => ({ - ...s, - errors: s.errors + 1, - lastErrorTime: Date.now(), - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(error.cause, { - message: - "ClickHouse fallback insert failed; retaining events for retry", - table, - }) - ) - ), - Effect.as(false) - ) - ) - ); - }, - { concurrency: "unbounded" } - ).pipe( - Effect.map((results) => results.every(Boolean)), - Effect.ensuring(Ref.update(ref, (s) => ({ ...s, flushing: false }))) - ); - }); - const resolveTopic = ( topic: string ): Effect.Effect => @@ -359,7 +237,7 @@ function makeProducerEffects( topic, }), }).pipe( - Effect.tap(() => inc("flushed", events.length)), + Effect.tap(() => inc("sent", events.length)), Effect.catchTag("ClickHouseFallbackError", (error) => Ref.update(ref, (s) => ({ ...s, @@ -382,62 +260,6 @@ function makeProducerEffects( ); }); - const bufferAll = ( - topic: string, - events: unknown[] - ): Effect.Effect => - Effect.gen(function* () { - const table = yield* resolveTopic(topic); - - type BufferAppendResult = - | { bufferLength: number; overflow: true } - | { overflow: false }; - const result = yield* Ref.modify( - ref, - (s): [BufferAppendResult, ProducerState] => { - if (s.buffer.length + events.length > config.bufferHardMax) { - return [ - { bufferLength: s.buffer.length, overflow: true }, - { - ...s, - errors: s.errors + 1, - lastErrorTime: Date.now(), - }, - ]; - } - - return [ - { overflow: false }, - { - ...s, - buffer: [ - ...s.buffer, - ...events.map((event) => ({ table, event })), - ], - buffered: s.buffered + events.length, - }, - ]; - } - ); - - if (result.overflow) { - const error = new BufferOverflowError({ - bufferHardMax: config.bufferHardMax, - bufferLength: result.bufferLength, - eventCount: events.length, - retryable: true, - topic, - }); - yield* Effect.sync(() => - captureError(error, { - message: "ClickHouse fallback buffer is full", - topic, - }) - ); - return yield* Effect.fail(error); - } - }); - const acquireSendSlot = (eventCount: number) => Ref.modify(ref, (s) => { if (s.shuttingDown) { @@ -466,8 +288,7 @@ function makeProducerEffects( const sendViaKafka = ( topic: string, messages: Array<{ value: string; key?: string }>, - fallbackEvents: unknown[], - fallback: "buffer" | "direct" + fallbackEvents: unknown[] ): Effect.Effect => acquireSendSlot(fallbackEvents.length).pipe( Effect.flatMap(() => @@ -515,12 +336,7 @@ function makeProducerEffects( } } - if (fallback === "direct") { - yield* persistDirectly(topic, fallbackEvents); - return; - } - - yield* bufferAll(topic, fallbackEvents); + yield* persistDirectly(topic, fallbackEvents); }).pipe(Effect.ensuring(releaseSendSlot(fallbackEvents.length))) ) ); @@ -538,8 +354,7 @@ function makeProducerEffects( key: key || (event as { client_id?: string }).client_id, }, ], - [event], - "direct" + [event] ); const sendMany = ( @@ -557,45 +372,7 @@ function makeProducerEffects( (e as { client_id?: string }).client_id || (e as { event_id?: string }).event_id, })), - events, - "direct" - ); - }; - - const sendOneBuffered = ( - topic: string, - event: unknown, - key?: string - ): Effect.Effect => - sendViaKafka( - topic, - [ - { - value: stringifyEvent(event), - key: key || (event as { client_id?: string }).client_id, - }, - ], - [event], - "buffer" - ); - - const sendManyBuffered = ( - topic: string, - events: unknown[] - ): Effect.Effect => { - if (events.length === 0) { - return Effect.void; - } - return sendViaKafka( - topic, - events.map((e) => ({ - value: stringifyEvent(e), - key: - (e as { client_id?: string }).client_id || - (e as { event_id?: string }).event_id, - })), - events, - "buffer" + events ); }; @@ -626,28 +403,16 @@ function makeProducerEffects( ); }); - const drainUntilEmpty = Effect.gen(function* () { - while (true) { - const state = yield* Ref.get(ref); - if (state.inFlight > 0 || state.flushing) { - yield* Effect.sleep("10 millis"); - continue; - } - if (state.buffer.length === 0) { - return; - } - - const didFlush = yield* flush; - if (!didFlush) { - yield* Effect.sleep(`${config.shutdownDrainRetryDelay} millis`); - } + const drainInFlight = Effect.gen(function* () { + while ((yield* Ref.get(ref)).inFlight > 0) { + yield* Effect.sleep("10 millis"); } }); const shutDown: Effect.Effect = Effect.gen( function* () { yield* Ref.update(ref, (s) => ({ ...s, shuttingDown: true })); - yield* drainUntilEmpty.pipe( + yield* drainInFlight.pipe( Effect.timeout(`${config.shutdownDrainTimeout} millis`), Effect.catch(() => Ref.get(ref).pipe( @@ -656,7 +421,6 @@ function makeProducerEffects( new ShutdownDrainError({ deadlineMs: config.shutdownDrainTimeout, inFlight: state.inFlight, - residualCount: state.buffer.length + state.inFlight, retryable: true, }) ) @@ -671,8 +435,6 @@ function makeProducerEffects( const stats: Effect.Effect = Ref.get(ref).pipe( Effect.map( ({ - buffer, - flushing: _f, shuttingDown: _s, connectionFailed, producerInitialized: _p, @@ -680,18 +442,14 @@ function makeProducerEffects( }) => ({ ...rest, failed: connectionFailed, - bufferSize: buffer.length, kafkaEnabled: enabled, }) ) ); return { - flush, sendOne, sendMany, - sendOneBuffered, - sendManyBuffered, shutDown, stats, }; @@ -703,7 +461,7 @@ export const createProducerEffects = ( ch: ClickHouseClient, topicMap: Record ): Effect.Effect => - Ref.make({ ...INITIAL_STATE, buffer: [] }).pipe( + Ref.make(INITIAL_STATE).pipe( Effect.map((ref) => makeProducerEffects(config, kafka, ch, topicMap, ref)) ); @@ -748,14 +506,10 @@ function initializeKafka(config: ProducerConfig): Producer | null { } export interface ProducerStatsSnapshot { - buffered: number; - bufferSize: number; connected: boolean; - dropped: number; errors: number; failed: boolean; failedCount: number; - flushed: number; inFlight: number; kafkaEnabled: boolean; lastErrorTime: number | null; @@ -772,11 +526,7 @@ const CONFIG: ProducerConfig = { kafkaTimeout: 10_000, maxProducerRetries: 3, producerRetryDelay: 300, - bufferInterval: 5000, - bufferMax: 1000, - bufferHardMax: 10_000, chunkSize: 5000, - shutdownDrainRetryDelay: 250, shutdownDrainTimeout: 8000, }; @@ -802,11 +552,6 @@ const ProducerLive = Layer.effectDiscard( TOPIC_MAP ); fx = effects; - yield* effects.flush.pipe( - Effect.catch(() => Effect.void), - Effect.repeat(Schedule.spaced(CONFIG.bufferInterval)), - Effect.forkScoped - ); }) ); @@ -833,12 +578,6 @@ export const send = (topic: string, event: unknown, key?: string) => export const sendBatch = (topic: string, events: unknown[]) => withActiveFx((f) => f.sendMany(topic, events)); -export const sendBuffered = (topic: string, event: unknown, key?: string) => - withActiveFx((f) => f.sendOneBuffered(topic, event, key)); - -export const sendBufferedBatch = (topic: string, events: unknown[]) => - withActiveFx((f) => f.sendManyBuffered(topic, events)); - export const disconnect = withFx((f) => f.shutDown); export const getStats = withFx((f) => f.stats); From d0e88bdd06b5524e5a86a9be2616f24283655b0d Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:01:52 +0300 Subject: [PATCH 10/25] fix(uptime): refresh durable scheduler jobs --- apps/uptime/src/sync-schedulers.test.ts | 74 +++++++++++++++++++++++++ apps/uptime/src/sync-schedulers.ts | 29 ++-------- 2 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 apps/uptime/src/sync-schedulers.test.ts diff --git a/apps/uptime/src/sync-schedulers.test.ts b/apps/uptime/src/sync-schedulers.test.ts new file mode 100644 index 0000000000..c306871efd --- /dev/null +++ b/apps/uptime/src/sync-schedulers.test.ts @@ -0,0 +1,74 @@ +import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import * as actualDb from "@databuddy/db"; +import * as actualSchema from "@databuddy/db/schema"; +import * as actualRedis from "@databuddy/redis"; +import * as actualEvlog from "evlog"; + +const monitors = [{ granularity: "five_minutes", id: "schedule-1" }]; +const upsertJobScheduler = mock(async () => undefined); +const getJobScheduler = mock(async () => ({ id: "uptime-schedule-1" })); +const dbSelect = mock(() => ({ + from: () => ({ where: async () => monitors }), +})); +const logInfo = mock(() => {}); + +mock.module("@databuddy/db", () => ({ + ...actualDb, + db: { select: dbSelect }, + eq: mock(() => undefined), +})); +mock.module("@databuddy/db/schema", () => ({ + ...actualSchema, + uptimeSchedules: { + granularity: "granularity", + id: "id", + isPaused: "isPaused", + }, +})); +mock.module("@databuddy/redis", () => ({ + ...actualRedis, + getUptimeQueue: () => ({ getJobScheduler, upsertJobScheduler }), + UPTIME_CHECK_JOB_NAME: "uptime-check", + UPTIME_JOB_OPTIONS: { attempts: 1_000_000 }, + uptimeSchedulerId: (scheduleId: string) => `uptime-${scheduleId}`, +})); +mock.module("evlog", () => ({ + ...actualEvlog, + log: { error: mock(() => {}), info: logInfo }, +})); + +const { syncSchedulers } = await import("./sync-schedulers"); + +beforeEach(() => { + dbSelect.mockClear(); + getJobScheduler.mockClear(); + logInfo.mockClear(); + upsertJobScheduler.mockClear(); +}); + +afterAll(() => { + mock.module("@databuddy/db", () => actualDb); + mock.module("@databuddy/db/schema", () => actualSchema); + mock.module("@databuddy/redis", () => actualRedis); + mock.module("evlog", () => actualEvlog); +}); + +describe("syncSchedulers", () => { + test("updates an existing scheduler with the current durable job options", async () => { + await syncSchedulers(); + + expect(getJobScheduler).not.toHaveBeenCalled(); + expect(upsertJobScheduler).toHaveBeenCalledWith( + "uptime-schedule-1", + { pattern: "*/5 * * * *" }, + { + data: { scheduleId: "schedule-1", trigger: "scheduled" }, + name: "uptime-check", + opts: { attempts: 1_000_000 }, + } + ); + expect(logInfo).toHaveBeenCalledWith( + expect.objectContaining({ failed: 0, total: 1, upserted: 1 }) + ); + }); +}); diff --git a/apps/uptime/src/sync-schedulers.ts b/apps/uptime/src/sync-schedulers.ts index da55ad9e23..4747f5a68a 100644 --- a/apps/uptime/src/sync-schedulers.ts +++ b/apps/uptime/src/sync-schedulers.ts @@ -31,15 +31,6 @@ const syncMonitor = ( ) => Effect.gen(function* () { const schedulerId = uptimeSchedulerId(monitor.id); - - const existing = yield* Effect.tryPromise({ - try: () => queue.getJobScheduler(schedulerId), - catch: (cause) => cause, - }); - if (existing) { - return "skipped" as const; - } - const pattern = CRON_GRANULARITIES[monitor.granularity]; if (!pattern) { return yield* Effect.fail( @@ -66,8 +57,6 @@ const syncMonitor = ( ), catch: (cause) => cause, }); - - return "created" as const; }); const syncAll = Effect.gen(function* () { @@ -85,17 +74,12 @@ const syncAll = Effect.gen(function* () { catch: (cause) => cause, }); - const created = yield* Ref.make(0); - const skipped = yield* Ref.make(0); + const upserted = yield* Ref.make(0); const failed = yield* Ref.make(0); for (const monitor of monitors) { yield* syncMonitor(monitor, queue).pipe( - Effect.tap((result) => - result === "created" - ? Ref.update(created, (n) => n + 1) - : Ref.update(skipped, (n) => n + 1) - ), + Effect.tap(() => Ref.update(upserted, (n) => n + 1)), Effect.catch((error) => { if (error instanceof UnknownGranularity) { log.error({ @@ -116,17 +100,12 @@ const syncAll = Effect.gen(function* () { ); } - const [c, s, f] = yield* Effect.all([ - Ref.get(created), - Ref.get(skipped), - Ref.get(failed), - ]); + const [u, f] = yield* Effect.all([Ref.get(upserted), Ref.get(failed)]); log.info({ sync: "scheduler", total: monitors.length, - created: c, - skipped: s, + upserted: u, failed: f, }); }); From 9cb4cc7e5fbf25ea434a1faaed1b29c1f8d643db Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:02:00 +0300 Subject: [PATCH 11/25] chore(ingest): remove dead Vector sinks --- infra/ingest/vector.yaml | 54 ---------------------------------------- 1 file changed, 54 deletions(-) diff --git a/infra/ingest/vector.yaml b/infra/ingest/vector.yaml index a848c5a6fa..6a524f266d 100644 --- a/infra/ingest/vector.yaml +++ b/infra/ingest/vector.yaml @@ -35,9 +35,7 @@ transforms: - redpanda route: events: '.topic == "analytics-events"' - errors: '.topic == "analytics-errors"' error_spans: '.topic == "analytics-error-spans"' - web_vitals: '.topic == "analytics-web-vitals"' vitals_spans: '.topic == "analytics-vitals-spans"' custom_events: '.topic == "analytics-custom-events"' ai_traffic_spans: '.topic == "analytics-ai-traffic-spans"' @@ -73,32 +71,6 @@ sinks: encoding: timestamp_format: unix_ms - clickhouse_errors: - type: clickhouse - inputs: - - route_analytics.errors - endpoint: "${CLICKHOUSE_URL}" - database: analytics - table: errors - auth: - strategy: basic - user: "${CLICKHOUSE_USER}" - password: "${CLICKHOUSE_PASSWORD}" - batch: - max_events: 5000 - max_bytes: 5000000 - timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block - date_time_best_effort: true - skip_unknown_fields: true - encoding: - timestamp_format: unix_ms - clickhouse_error_spans: type: clickhouse inputs: @@ -125,32 +97,6 @@ sinks: encoding: timestamp_format: unix_ms - clickhouse_web_vitals: - type: clickhouse - inputs: - - route_analytics.web_vitals - endpoint: "${CLICKHOUSE_URL}" - database: analytics - table: web_vitals - auth: - strategy: basic - user: "${CLICKHOUSE_USER}" - password: "${CLICKHOUSE_PASSWORD}" - batch: - max_events: 5000 - max_bytes: 5000000 - timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block - date_time_best_effort: true - skip_unknown_fields: true - encoding: - timestamp_format: unix_ms - clickhouse_vitals_spans: type: clickhouse inputs: From f162c309c4303c7bedda4f0cb01600f641665a2d Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:33:26 +0300 Subject: [PATCH 12/25] fix(basket): honor legacy delivery deduplication --- apps/basket/src/lib/security.test.ts | 9 +++++++++ apps/basket/src/lib/security.ts | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/basket/src/lib/security.test.ts b/apps/basket/src/lib/security.test.ts index c9b1287d7b..fcd3706bde 100644 --- a/apps/basket/src/lib/security.test.ts +++ b/apps/basket/src/lib/security.test.ts @@ -186,6 +186,15 @@ describe("duplicate reservations", () => { }); }); + test("suppresses a delivered reservation written by the legacy key format", async () => { + mockRedisSet.mockResolvedValue(null); + mockRedisGet.mockResolvedValue("1"); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: true }); + }); + test("requires a retry when another request owns a pending reservation", async () => { mockRedisSet.mockResolvedValue(null); mockRedisGet.mockResolvedValue("pending:other-attempt"); diff --git a/apps/basket/src/lib/security.ts b/apps/basket/src/lib/security.ts index f81af24d8f..d56247b3dd 100644 --- a/apps/basket/src/lib/security.ts +++ b/apps/basket/src/lib/security.ts @@ -11,6 +11,8 @@ const STANDARD_EVENT_TTL = 86_400; const DEDUP_RETRY_DELAY_MS = 25; const PENDING_DEDUP_PREFIX = "pending:"; const DELIVERED_DEDUP_VALUE = "delivered"; +// Kept until all keys written by the pre-reservation implementation expire. +const LEGACY_DELIVERED_DEDUP_VALUE = "1"; const RELEASE_PENDING_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) end @@ -162,7 +164,10 @@ async function readDedupReservationState( token: string ): Promise { const value = await redis.get(key); - if (value === DELIVERED_DEDUP_VALUE) { + if ( + value === DELIVERED_DEDUP_VALUE || + value === LEGACY_DELIVERED_DEDUP_VALUE + ) { return "delivered"; } if (value === token) { From 9724ec35411f656c0648e2d2447520c3771264a8 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:33:35 +0300 Subject: [PATCH 13/25] fix(tracker): verify analytics delivery outcomes --- packages/tracker/src/core/client.ts | 24 -------- packages/tracker/src/plugins/pixel.ts | 43 ++++++++++++- .../tracker/tests/unit/http-client.test.ts | 36 +++++++++++ packages/tracker/tests/unit/pixel.test.ts | 60 ++++++++++++++++++- 4 files changed, 134 insertions(+), 29 deletions(-) diff --git a/packages/tracker/src/core/client.ts b/packages/tracker/src/core/client.ts index 9e47db6eab..e72a0f60f7 100644 --- a/packages/tracker/src/core/client.ts +++ b/packages/tracker/src/core/client.ts @@ -116,30 +116,6 @@ export class HttpClient { return this.abortedResult(retryCount); } - if ( - retryCount === 0 && - typeof navigator !== "undefined" && - navigator.sendBeacon && - options.keepalive - ) { - try { - const blob = new Blob([JSON.stringify(data ?? {})], { - type: "application/json", - }); - if (navigator.sendBeacon(url, blob)) { - return { - ok: true, - data: null, - status: null, - attempts: 1, - transport: "beacon", - }; - } - } catch { - // Fall through to fetch so callers receive a verifiable outcome. - } - } - try { const response = await fetch(url, { method: "POST", diff --git a/packages/tracker/src/plugins/pixel.ts b/packages/tracker/src/plugins/pixel.ts index ea2fb66580..2fc9b40b2d 100644 --- a/packages/tracker/src/plugins/pixel.ts +++ b/packages/tracker/src/plugins/pixel.ts @@ -12,6 +12,7 @@ const PIXEL_TYPE_BY_ENDPOINT: Record = { "/errors": "error", }; const MAX_PIXEL_RETRY_DELAY_MS = 30_000; +const PIXEL_LOAD_TIMEOUT_MS = 10_000; interface PixelDeliveryResult { attempts: number; @@ -56,18 +57,27 @@ function flattenIntoParams( export function initPixelTracking(tracker: BaseTracker) { tracker.options.enableBatching = false; let deliveryGeneration = 0; + const activeImageCancels = new Set<() => void>(); const cancelPendingRequests = tracker.api.cancelPendingRequests.bind( tracker.api ); tracker.api.cancelPendingRequests = () => { deliveryGeneration += 1; + for (const cancel of activeImageCancels) { + cancel(); + } cancelPendingRequests(); }; + const configuredMaxRetries = tracker.options.maxRetries; const maxRetries = tracker.options.enableRetries === false ? 0 - : (tracker.options.maxRetries ?? 3); + : typeof configuredMaxRetries === "number" && + Number.isFinite(configuredMaxRetries) && + configuredMaxRetries >= 0 + ? Math.floor(configuredMaxRetries) + : 3; const initialRetryDelay = Math.max( 0, tracker.options.initialRetryDelay ?? 500 @@ -111,8 +121,35 @@ export function initPixelTracking(tracker: BaseTracker) { const load = (): Promise => new Promise((resolve) => { const img = new Image(); - img.onload = () => resolve(true); - img.onerror = () => resolve(false); + let settled = false; + let timeout: ReturnType | undefined; + const finish = (success: boolean) => { + if (settled) { + return; + } + settled = true; + if (timeout !== undefined) { + clearTimeout(timeout); + } + activeImageCancels.delete(cancel); + img.onload = null; + img.onerror = null; + resolve(success); + }; + const cancel = () => { + if (settled) { + return; + } + img.onload = null; + img.onerror = null; + img.src = ""; + finish(false); + }; + + activeImageCancels.add(cancel); + timeout = setTimeout(cancel, PIXEL_LOAD_TIMEOUT_MS); + img.onload = () => finish(true); + img.onerror = () => finish(false); img.src = url.toString(); }); diff --git a/packages/tracker/tests/unit/http-client.test.ts b/packages/tracker/tests/unit/http-client.test.ts index c32045c0a1..603fd3f3ab 100644 --- a/packages/tracker/tests/unit/http-client.test.ts +++ b/packages/tracker/tests/unit/http-client.test.ts @@ -3,6 +3,7 @@ import { HttpClient, type HttpResult } from "../../src/core/client"; import { BaseTracker } from "../../src/core/tracker"; const originalFetch = globalThis.fetch; +const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); class DeliveryTestTracker extends BaseTracker { private deliveryBlocked = false; @@ -27,6 +28,11 @@ afterEach(() => { jest.useRealTimers(); } globalThis.fetch = originalFetch; + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + return; + } + Reflect.deleteProperty(globalThis, "navigator"); }); async function flushMicrotasks(): Promise { @@ -201,6 +207,36 @@ describe("HttpClient", () => { }); describe("BaseTracker delivery outcomes", () => { + test("requeues a retryable batch despite an accepted Beacon", async () => { + jest.useFakeTimers(); + const sendBeacon = mock(() => true); + Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { sendBeacon }, + }); + const fetchMock = mock(async () => + Response.json({ error: "Temporarily unavailable" }, { status: 503 }) + ); + globalThis.fetch = fetchMock as typeof fetch; + const tracker = new DeliveryTestTracker({ + clientId: "site_example", + enableRetries: false, + }); + const event = { eventId: "event_1", timestamp: 1 }; + await tracker.addToBatch(event); + + const result = await tracker.flushBatch(); + + expect(sendBeacon).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + ok: false, + status: "failed", + retryable: true, + }); + expect(tracker.batchQueue).toEqual([event]); + }); + test("keeps a retryable failed batch queued", async () => { jest.useFakeTimers(); const tracker = new DeliveryTestTracker({ clientId: "site_example" }); diff --git a/packages/tracker/tests/unit/pixel.test.ts b/packages/tracker/tests/unit/pixel.test.ts index 949f460f46..f784f61d86 100644 --- a/packages/tracker/tests/unit/pixel.test.ts +++ b/packages/tracker/tests/unit/pixel.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, mock, test } from "bun:test"; +import { afterEach, describe, expect, jest, mock, test } from "bun:test"; import type { BaseTracker } from "../../src/core/tracker"; import type { TrackerOptions } from "../../src/core/types"; import { initPixelTracking } from "../../src/plugins/pixel"; @@ -6,6 +6,10 @@ import { initPixelTracking } from "../../src/plugins/pixel"; const originalImage = Object.getOwnPropertyDescriptor(globalThis, "Image"); afterEach(() => { + if (jest.isFakeTimers()) { + jest.clearAllTimers(); + jest.useRealTimers(); + } if (originalImage) { Object.defineProperty(globalThis, "Image", originalImage); return; @@ -32,7 +36,9 @@ function createTracker( } as unknown as BaseTracker; } -function installImageOutcomes(outcomes: Array<"error" | "load">): string[] { +function installImageOutcomes( + outcomes: Array<"error" | "load" | "pending"> +): string[] { const requests: string[] = []; class MockImage { @@ -41,7 +47,13 @@ function installImageOutcomes(outcomes: Array<"error" | "load">): string[] { set src(value: string) { requests.push(value); + if (!value) { + return; + } const outcome = outcomes.shift() ?? "error"; + if (outcome === "pending") { + return; + } queueMicrotask(() => { if (outcome === "load") { this.onload?.(); @@ -84,4 +96,48 @@ describe("pixel transport", () => { expect(tracker.sendBeacon({ eventId: "event_1" }, "/")).toBe(false); }); + + test("cancels an active image request when tracking is cleared", async () => { + const requests = installImageOutcomes(["pending"]); + const tracker = createTracker(); + initPixelTracking(tracker); + + const delivery = tracker.api.fetch("/", { eventId: "event_1" }); + tracker.api.cancelPendingRequests(); + + expect(await delivery).toMatchObject({ + ok: false, + attempts: 1, + }); + expect(requests).toHaveLength(2); + expect(requests[1]).toBe(""); + }); + + test("bounds a pixel image load that never completes", async () => { + jest.useFakeTimers(); + const requests = installImageOutcomes(["pending"]); + const tracker = createTracker({ maxRetries: 0 }); + initPixelTracking(tracker); + + const delivery = tracker.api.fetch("/", { eventId: "event_1" }); + jest.advanceTimersByTime(10_000); + + expect(await delivery).toMatchObject({ + ok: false, + attempts: 1, + }); + expect(requests).toHaveLength(2); + expect(requests[1]).toBe(""); + }); + + test("uses the retry fallback for an invalid maxRetries value", async () => { + const requests = installImageOutcomes(["error", "error", "error", "error"]); + const tracker = createTracker({ maxRetries: Number.NaN }); + initPixelTracking(tracker); + + const result = await tracker.api.fetch("/", { eventId: "event_1" }); + + expect(result).toMatchObject({ ok: false, attempts: 4 }); + expect(requests).toHaveLength(4); + }); }); From b83be11b612372bd9b3e49da2a96c1e579f8356e Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:33:41 +0300 Subject: [PATCH 14/25] fix(uptime): validate durable delivery payloads --- apps/uptime/package.json | 3 +- apps/uptime/src/sync-schedulers.test.ts | 7 +--- apps/uptime/src/types.ts | 54 +++++++++++++------------ apps/uptime/src/worker.test.ts | 35 ++++++++++++++++ apps/uptime/src/worker.ts | 34 +++++----------- bun.lock | 23 +---------- 6 files changed, 80 insertions(+), 76 deletions(-) diff --git a/apps/uptime/package.json b/apps/uptime/package.json index 0f9a8b53e6..9fdcc9b6b1 100644 --- a/apps/uptime/package.json +++ b/apps/uptime/package.json @@ -18,7 +18,8 @@ "effect": "^4.0.0-beta.59", "elysia": "catalog:", "evlog": "catalog:", - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "zod": "catalog:" }, "packageManager": "bun@1.3.14" } diff --git a/apps/uptime/src/sync-schedulers.test.ts b/apps/uptime/src/sync-schedulers.test.ts index c306871efd..a312a4230f 100644 --- a/apps/uptime/src/sync-schedulers.test.ts +++ b/apps/uptime/src/sync-schedulers.test.ts @@ -6,7 +6,6 @@ import * as actualEvlog from "evlog"; const monitors = [{ granularity: "five_minutes", id: "schedule-1" }]; const upsertJobScheduler = mock(async () => undefined); -const getJobScheduler = mock(async () => ({ id: "uptime-schedule-1" })); const dbSelect = mock(() => ({ from: () => ({ where: async () => monitors }), })); @@ -27,7 +26,7 @@ mock.module("@databuddy/db/schema", () => ({ })); mock.module("@databuddy/redis", () => ({ ...actualRedis, - getUptimeQueue: () => ({ getJobScheduler, upsertJobScheduler }), + getUptimeQueue: () => ({ upsertJobScheduler }), UPTIME_CHECK_JOB_NAME: "uptime-check", UPTIME_JOB_OPTIONS: { attempts: 1_000_000 }, uptimeSchedulerId: (scheduleId: string) => `uptime-${scheduleId}`, @@ -41,7 +40,6 @@ const { syncSchedulers } = await import("./sync-schedulers"); beforeEach(() => { dbSelect.mockClear(); - getJobScheduler.mockClear(); logInfo.mockClear(); upsertJobScheduler.mockClear(); }); @@ -54,10 +52,9 @@ afterAll(() => { }); describe("syncSchedulers", () => { - test("updates an existing scheduler with the current durable job options", async () => { + test("upserts an active scheduler with current durable job options", async () => { await syncSchedulers(); - expect(getJobScheduler).not.toHaveBeenCalled(); expect(upsertJobScheduler).toHaveBeenCalledWith( "uptime-schedule-1", { pattern: "*/5 * * * *" }, diff --git a/apps/uptime/src/types.ts b/apps/uptime/src/types.ts index 54aff85f8c..e1f9e6f7fd 100644 --- a/apps/uptime/src/types.ts +++ b/apps/uptime/src/types.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + export const MonitorStatus = { DOWN: 0, UP: 1, @@ -5,31 +7,33 @@ export const MonitorStatus = { MAINTENANCE: 3, } as const; -export interface UptimeData { - attempt: number; - check_type: string; - content_hash: string; - env: string; - error: string; - event_id: string; - failure_streak: number; - http_code: number; - json_data?: string; - probe_ip: string; - probe_region: string; - redirect_count: number; - response_bytes: number; - retries: number; - site_id: string; - ssl_expiry: number; - ssl_valid: number; - status: number; - timestamp: number; - total_ms: number; - ttfb_ms: number; - url: string; - user_agent: string; -} +export const uptimeDataSchema = z.object({ + attempt: z.number(), + check_type: z.string(), + content_hash: z.string(), + env: z.string(), + error: z.string(), + event_id: z.string(), + failure_streak: z.number(), + http_code: z.number(), + json_data: z.string().optional(), + probe_ip: z.string(), + probe_region: z.string(), + redirect_count: z.number(), + response_bytes: z.number(), + retries: z.number(), + site_id: z.string(), + ssl_expiry: z.number(), + ssl_valid: z.number(), + status: z.number(), + timestamp: z.number(), + total_ms: z.number(), + ttfb_ms: z.number(), + url: z.string(), + user_agent: z.string(), +}); + +export type UptimeData = z.infer; export type ScheduleLookupReason = "not_found" | "malformed" | "transient"; diff --git a/apps/uptime/src/worker.test.ts b/apps/uptime/src/worker.test.ts index 439c0d92e8..1229ae9c35 100644 --- a/apps/uptime/src/worker.test.ts +++ b/apps/uptime/src/worker.test.ts @@ -532,6 +532,26 @@ describe("processUptimeCheck", () => { expect(calls.email).toHaveLength(1); }); + it("rejects malformed checkpointed delivery payloads before replaying", async () => { + await expect( + processUptimeJob( + { + name: "uptime-check", + data: { + delivery: { event: { ...uptimeData(), http_code: "200" } }, + scheduleId: "schedule-1", + trigger: "scheduled", + }, + updateData: async () => {}, + }, + deps() + ) + ).rejects.toThrow("Invalid persisted uptime delivery payload"); + + expect(calls.check).toEqual([]); + expect(calls.delivery).toEqual([]); + }); + it("retries a delivery job when Redpanda does not acknowledge it", async () => { const failingDeps = deps(); failingDeps.sendUptimeEvent = async () => ({ @@ -559,4 +579,19 @@ describe("processUptimeCheck", () => { }) ); }); + + it("rejects malformed delivery queue payloads before sending", async () => { + await expect( + processUptimeDeliveryJob( + { + data: { event: { ...uptimeData(), site_id: 1 } }, + id: "uptime-delivery-uptime-event-1", + name: "uptime-event-delivery", + }, + deps() + ) + ).rejects.toThrow("Invalid uptime delivery payload"); + + expect(calls.captureError).toEqual([]); + }); }); diff --git a/apps/uptime/src/worker.ts b/apps/uptime/src/worker.ts index 56c971679f..8abd488dcb 100644 --- a/apps/uptime/src/worker.ts +++ b/apps/uptime/src/worker.ts @@ -29,6 +29,7 @@ import { MonitorStatus, type ActionResult, type ScheduleLookupReason, + uptimeDataSchema, type UptimeData, } from "./types"; import { @@ -434,23 +435,6 @@ export async function processUptimeCheck( } } -function isUptimeData(value: unknown): value is UptimeData { - if (typeof value !== "object" || value === null) { - return false; - } - - const event = value as { - event_id?: unknown; - site_id?: unknown; - timestamp?: unknown; - }; - return ( - typeof event.event_id === "string" && - typeof event.site_id === "string" && - typeof event.timestamp === "number" - ); -} - async function replayPersistedUptimeDelivery( job: UptimeWorkerJob, data: UptimeData, @@ -518,10 +502,11 @@ export async function processUptimeJob( const persistedEvent = job.data.delivery?.event; if (persistedEvent !== undefined) { - if (!isUptimeData(persistedEvent)) { + const parsedEvent = uptimeDataSchema.safeParse(persistedEvent); + if (!parsedEvent.success) { throw new Error("Invalid persisted uptime delivery payload"); } - await replayPersistedUptimeDelivery(job, persistedEvent, deps); + await replayPersistedUptimeDelivery(job, parsedEvent.data, deps); return; } @@ -548,11 +533,12 @@ export async function processUptimeDeliveryJob( if (job.name !== UPTIME_DELIVERY_JOB_NAME) { throw new Error(`Unknown uptime delivery job: ${job.name}`); } - if (!isUptimeData(job.data.event)) { + const parsedEvent = uptimeDataSchema.safeParse(job.data.event); + if (!parsedEvent.success) { throw new Error("Invalid uptime delivery payload"); } - const data = job.data.event; + const data = parsedEvent.data; try { const result = await deps.sendUptimeEvent(data, data.site_id); if (!result.sent) { @@ -629,10 +615,12 @@ export function startUptimeDeliveryWorker() { ); worker.on("failed", (job, error) => { + const parsedEvent = job + ? uptimeDataSchema.safeParse(job.data.event) + : undefined; captureError(error, { error_step: "uptime_delivery_worker_job_failed", - event_id: - job && isUptimeData(job.data.event) ? job.data.event.event_id : "", + event_id: parsedEvent?.success ? parsedEvent.data.event_id : "", job_id: job?.id ?? "", attempts_used: job?.attemptsMade ?? 0, attempts_max: job?.opts?.attempts ?? 1_000_000, diff --git a/bun.lock b/bun.lock index 0dff8b7121..15c5919cc6 100644 --- a/bun.lock +++ b/bun.lock @@ -433,6 +433,7 @@ "elysia": "catalog:", "evlog": "catalog:", "kafkajs": "^2.2.4", + "zod": "catalog:", }, }, "packages/ai": { @@ -4857,8 +4858,6 @@ "@databuddy/test/drizzle-orm": ["drizzle-orm@1.0.0-rc.1", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@effect/sql-pg": ">=4.0.0-beta.58 || >=4.0.0", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@sinclair/typebox": ">=0.34.8", "@sqlitecloud/drivers": ">=1.0.653", "@tidbcloud/serverless": "*", "@tursodatabase/database": ">=0.2.1", "@tursodatabase/database-common": ">=0.2.1", "@tursodatabase/database-wasm": ">=0.2.1", "@types/better-sqlite3": "*", "@types/mssql": "^9.1.4", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "arktype": ">=2.0.0", "better-sqlite3": ">=9.3.0", "bun-types": "*", "effect": ">=4.0.0-beta.58 || >=4.0.0", "expo-sqlite": ">=14.0.0", "mssql": "^11.0.1", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5", "typebox": ">=1.0.0", "valibot": ">=1.0.0-beta.7", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@effect/sql-pg", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@sinclair/typebox", "@sqlitecloud/drivers", "@tidbcloud/serverless", "@tursodatabase/database", "@tursodatabase/database-common", "@tursodatabase/database-wasm", "@types/better-sqlite3", "@types/mssql", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "arktype", "better-sqlite3", "bun-types", "effect", "expo-sqlite", "mssql", "mysql2", "pg", "postgres", "sql.js", "sqlite3", "typebox", "valibot", "zod"] }, "sha512-jGCqAgxpz+OSHP2jQGooUHBxnFMTYl0TTRSfULBl52VNf7CtyNRnazUi+VdbSxvJrDP2lnIsmUh5O+HhKeSJCg=="], - "@databuddy/uptime/effect": ["effect@4.0.0-beta.66", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "fast-check": "^4.6.0", "find-my-way-ts": "^0.1.6", "ini": "^6.0.0", "kubernetes-types": "^1.30.0", "msgpackr": "^1.11.9", "multipasta": "^0.2.7", "toml": "^4.1.1", "uuid": "^13.0.0", "yaml": "^2.8.3" } }, "sha512-4arEr62cziFa8BBVDUwJCJJmaVepXf/kRg7KtC0h8+bufngscrHbwWFhr9c+HonwOF+31U3iD3xUJmw9KzX7Dw=="], - "@dxup/nuxt/tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="], "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], @@ -5641,12 +5640,6 @@ "@databuddy/docs/resend/svix": ["svix@1.92.2", "", { "dependencies": { "standardwebhooks": "1.0.0" } }, "sha512-ZmuA3UVvlnF9EgxlzmPtF7CKjQb64Z6OFlyfdDfU0sdcC7dJa+3aOYX5B9mA+RS6ch1AxBa4UP/l6KmqfGtWBQ=="], - "@databuddy/uptime/effect/ini": ["ini@6.0.0", "", {}, "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ=="], - - "@databuddy/uptime/effect/msgpackr": ["msgpackr@1.11.12", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.2" } }, "sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg=="], - - "@databuddy/uptime/effect/uuid": ["uuid@13.0.2", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw=="], - "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="], "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="], @@ -6461,8 +6454,6 @@ "zip-stream/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], - "@databuddy/uptime/effect/msgpackr/msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="], - "@inquirer/core/wrap-ansi/string-width/is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], "@mapbox/node-pre-gyp/node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], @@ -6747,18 +6738,6 @@ "zip-stream/readable-stream/string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg=="], - - "@databuddy/uptime/effect/msgpackr/msgpackr-extract/@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ=="], - "archiver-utils/glob/minimatch/brace-expansion/balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], "nuxt/oxc-parser/@oxc-parser/binding-wasm32-wasi/@emnapi/core/@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w=="], From 06100421ef9e89a2ecf0e693327250399b4af8ad Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:38:05 +0300 Subject: [PATCH 15/25] fix(basket): retry legacy dedup reservations --- .agents/skills/databuddy-internal/SKILL.md | 1 + apps/basket/src/lib/security.test.ts | 28 +++++++++++- apps/basket/src/lib/security.ts | 52 ++++++++++++++++++---- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/.agents/skills/databuddy-internal/SKILL.md b/.agents/skills/databuddy-internal/SKILL.md index a9de56cf8f..63bdc861e8 100644 --- a/.agents/skills/databuddy-internal/SKILL.md +++ b/.agents/skills/databuddy-internal/SKILL.md @@ -57,6 +57,7 @@ Keep additions **minimal**: one bullet, a new `rg` hint, or a routing note—eno - Slack-reachable shared packages (`@databuddy/ai`, `@databuddy/rpc`) must not import `evlog/elysia`; use host-injected request logger providers from the API and plain evlog fallbacks elsewhere. - AI link tools must assign link folders by existing folder `id` or `slug` only; folder names are display text and must not be used for routing or dedupe. - `apps/basket`: ingest and LLM tracking service, Elysia app on port `4000` +- A legacy Basket dedupe value of `"1"` is an admission reservation, not delivery confirmation; claim or retry it rather than suppressing the event. - `apps/docs`: Next.js + Fumadocs docs app on port `3005` - When a user drops a prototype, remove only prototype-specific wiring and preserve the existing product surfaces it temporarily reused. - `apps/links`: redirect/link service diff --git a/apps/basket/src/lib/security.test.ts b/apps/basket/src/lib/security.test.ts index fcd3706bde..c30c582334 100644 --- a/apps/basket/src/lib/security.test.ts +++ b/apps/basket/src/lib/security.test.ts @@ -186,13 +186,37 @@ describe("duplicate reservations", () => { }); }); - test("suppresses a delivered reservation written by the legacy key format", async () => { + test("claims a legacy admission reservation before delivery", async () => { mockRedisSet.mockResolvedValue(null); mockRedisGet.mockResolvedValue("1"); + mockRedisEval.mockResolvedValue(1); const reservation = await reserveDuplicate("evt_1", "track"); - expect(reservation).toEqual({ duplicate: true }); + expect(reservation).toMatchObject({ + duplicate: false, + key: "dedup:track:evt_1", + token: expect.stringMatching(/^pending:/), + ttl: 86_400, + }); + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + 1, + "dedup:track:evt_1", + "1", + expect.stringMatching(/^pending:/), + 86_400 + ); + }); + + test("requires a retry when a legacy reservation cannot be claimed", async () => { + mockRedisSet.mockResolvedValue(null); + mockRedisGet.mockResolvedValue("1"); + mockRedisEval.mockResolvedValue(0); + + const reservation = await reserveDuplicate("evt_1", "track"); + + expect(reservation).toEqual({ duplicate: false, retryable: true }); }); test("requires a retry when another request owns a pending reservation", async () => { diff --git a/apps/basket/src/lib/security.ts b/apps/basket/src/lib/security.ts index d56247b3dd..143af897ad 100644 --- a/apps/basket/src/lib/security.ts +++ b/apps/basket/src/lib/security.ts @@ -12,11 +12,17 @@ const DEDUP_RETRY_DELAY_MS = 25; const PENDING_DEDUP_PREFIX = "pending:"; const DELIVERED_DEDUP_VALUE = "delivered"; // Kept until all keys written by the pre-reservation implementation expire. -const LEGACY_DELIVERED_DEDUP_VALUE = "1"; +// Unlike "delivered", this value was written before delivery was acknowledged. +const LEGACY_PENDING_DEDUP_VALUE = "1"; const RELEASE_PENDING_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) end return 0`; +const CLAIM_LEGACY_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then + redis.call("SET", KEYS[1], ARGV[2], "EX", ARGV[3]) + return 1 +end +return 0`; const MARK_PENDING_DEDUP_RESERVATION_DELIVERED = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("SET", KEYS[1], ARGV[2], "EX", ARGV[3]) end @@ -156,6 +162,7 @@ export interface DuplicateReservation { type DedupReservationState = | "acquired" | "delivered" + | "legacy" | "pending" | "unreserved"; @@ -164,18 +171,39 @@ async function readDedupReservationState( token: string ): Promise { const value = await redis.get(key); - if ( - value === DELIVERED_DEDUP_VALUE || - value === LEGACY_DELIVERED_DEDUP_VALUE - ) { + if (value === DELIVERED_DEDUP_VALUE) { return "delivered"; } + if (value === LEGACY_PENDING_DEDUP_VALUE) { + return "legacy"; + } if (value === token) { return "acquired"; } return value === null || value === undefined ? "unreserved" : "pending"; } +async function resolveDedupReservationState( + key: string, + ttl: number, + token: string +): Promise { + const state = await readDedupReservationState(key, token); + if (state !== "legacy") { + return state; + } + + const claimed = await redis.eval( + CLAIM_LEGACY_DEDUP_RESERVATION, + 1, + key, + LEGACY_PENDING_DEDUP_VALUE, + token, + ttl + ); + return claimed === 1 ? "acquired" : readDedupReservationState(key, token); +} + async function setDedupKey( key: string, ttl: number, @@ -183,7 +211,9 @@ async function setDedupKey( ): Promise { try { const result = await redis.set(key, token, "EX", ttl, "NX"); - return result === null ? readDedupReservationState(key, token) : "acquired"; + return result === null + ? resolveDedupReservationState(key, ttl, token) + : "acquired"; } catch (firstError) { await wait(DEDUP_RETRY_DELAY_MS); try { @@ -196,10 +226,10 @@ async function setDedupKey( // Seeing our token again proves that this request owns the reservation. // Any other pending token is retryable: only a confirmed delivery may // suppress the event. - return await readDedupReservationState(key, token); + return await resolveDedupReservationState(key, ttl, token); } catch { try { - return await readDedupReservationState(key, token); + return await resolveDedupReservationState(key, ttl, token); } catch { throw firstError; } @@ -228,7 +258,11 @@ export function reserveDuplicate( // This request did not acquire a pending key. Publishing without ownership // can duplicate a concurrent delivery or overwrite its confirmed state. - if (result === "pending" || result === "unreserved") { + if ( + result === "legacy" || + result === "pending" || + result === "unreserved" + ) { return { duplicate: false, retryable: true }; } From b8c7bba2e2835e1b51e3b67676a94182b2c3d1e7 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:38:23 +0300 Subject: [PATCH 16/25] fix(uptime): validate durable job envelopes --- apps/uptime/src/types.ts | 14 ++++++++++++ apps/uptime/src/worker.test.ts | 30 ++++++++++++++++++++++++++ apps/uptime/src/worker.ts | 39 ++++++++++++++++++++++++++-------- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/apps/uptime/src/types.ts b/apps/uptime/src/types.ts index e1f9e6f7fd..a4301f0262 100644 --- a/apps/uptime/src/types.ts +++ b/apps/uptime/src/types.ts @@ -33,6 +33,20 @@ export const uptimeDataSchema = z.object({ user_agent: z.string(), }); +const requiredUnknownSchema = z + .unknown() + .refine((value) => value !== undefined, "Required"); + +export const uptimeCheckJobDataSchema = z.object({ + delivery: z.object({ event: requiredUnknownSchema }).optional(), + scheduleId: z.string(), + trigger: z.enum(["manual", "scheduled"]), +}); + +export const uptimeDeliveryJobDataSchema = z.object({ + event: requiredUnknownSchema, +}); + export type UptimeData = z.infer; export type ScheduleLookupReason = "not_found" | "malformed" | "transient"; diff --git a/apps/uptime/src/worker.test.ts b/apps/uptime/src/worker.test.ts index 1229ae9c35..0309b7bd8c 100644 --- a/apps/uptime/src/worker.test.ts +++ b/apps/uptime/src/worker.test.ts @@ -229,6 +229,21 @@ describe("processUptimeCheck", () => { ); }); + it("rejects malformed BullMQ job envelopes before accessing their fields", async () => { + await expect( + processUptimeJob( + { + name: "uptime-check", + data: null as never, + updateData: async () => {}, + }, + deps() + ) + ).rejects.toThrow("Invalid uptime job payload"); + + expect(calls.check).toEqual([]); + }); + it("runs a scheduled check and emits events, status, and transition email work", async () => { await processUptimeCheckForTest("schedule-1", "scheduled", deps()); @@ -594,4 +609,19 @@ describe("processUptimeCheck", () => { expect(calls.captureError).toEqual([]); }); + + it("rejects malformed delivery job envelopes before accessing their fields", async () => { + await expect( + processUptimeDeliveryJob( + { + data: null as never, + id: "uptime-delivery-uptime-event-1", + name: "uptime-event-delivery", + }, + deps() + ) + ).rejects.toThrow("Invalid uptime delivery job payload"); + + expect(calls.delivery).toEqual([]); + }); }); diff --git a/apps/uptime/src/worker.ts b/apps/uptime/src/worker.ts index 8abd488dcb..0794c86fb3 100644 --- a/apps/uptime/src/worker.ts +++ b/apps/uptime/src/worker.ts @@ -29,7 +29,9 @@ import { MonitorStatus, type ActionResult, type ScheduleLookupReason, + uptimeCheckJobDataSchema, uptimeDataSchema, + uptimeDeliveryJobDataSchema, type UptimeData, } from "./types"; import { @@ -500,7 +502,13 @@ export async function processUptimeJob( throw new Error(`Unknown uptime job: ${job.name}`); } - const persistedEvent = job.data.delivery?.event; + const parsedJobData = uptimeCheckJobDataSchema.safeParse(job.data); + if (!parsedJobData.success) { + throw new Error("Invalid uptime job payload"); + } + + const jobData = parsedJobData.data; + const persistedEvent = jobData.delivery?.event; if (persistedEvent !== undefined) { const parsedEvent = uptimeDataSchema.safeParse(persistedEvent); if (!parsedEvent.success) { @@ -511,8 +519,8 @@ export async function processUptimeJob( } await processUptimeCheck( - job.data.scheduleId, - job.data.trigger, + jobData.scheduleId, + jobData.trigger, deps, { id: job.id, @@ -520,7 +528,7 @@ export async function processUptimeJob( }, async (data) => job.updateData({ - ...job.data, + ...jobData, delivery: { event: data }, }) ); @@ -533,7 +541,12 @@ export async function processUptimeDeliveryJob( if (job.name !== UPTIME_DELIVERY_JOB_NAME) { throw new Error(`Unknown uptime delivery job: ${job.name}`); } - const parsedEvent = uptimeDataSchema.safeParse(job.data.event); + const parsedJobData = uptimeDeliveryJobDataSchema.safeParse(job.data); + if (!parsedJobData.success) { + throw new Error("Invalid uptime delivery job payload"); + } + + const parsedEvent = uptimeDataSchema.safeParse(parsedJobData.data.event); if (!parsedEvent.success) { throw new Error("Invalid uptime delivery payload"); } @@ -571,12 +584,17 @@ export function startUptimeWorker() { const attemptsMade = job?.attemptsMade ?? 0; const maxAttempts = job?.opts?.attempts ?? 1_000_000; const isFinalAttempt = attemptsMade >= maxAttempts; + const parsedJobData = job + ? uptimeCheckJobDataSchema.safeParse(job.data) + : undefined; captureError(error, { error_step: "uptime_worker_job_failed", - schedule_id: job?.data.scheduleId ?? "", + schedule_id: parsedJobData?.success + ? parsedJobData.data.scheduleId + : "", job_id: job?.id ?? "", - trigger: job?.data.trigger ?? "", + trigger: parsedJobData?.success ? parsedJobData.data.trigger : "", attempts_used: attemptsMade, attempts_max: maxAttempts, is_final_attempt: isFinalAttempt, @@ -615,8 +633,11 @@ export function startUptimeDeliveryWorker() { ); worker.on("failed", (job, error) => { - const parsedEvent = job - ? uptimeDataSchema.safeParse(job.data.event) + const parsedJobData = job + ? uptimeDeliveryJobDataSchema.safeParse(job.data) + : undefined; + const parsedEvent = parsedJobData?.success + ? uptimeDataSchema.safeParse(parsedJobData.data.event) : undefined; captureError(error, { error_step: "uptime_delivery_worker_job_failed", From 69ac91fc23bd973187a9771b8f853d4cd9e63083 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:39:12 +0300 Subject: [PATCH 17/25] fix(uptime): preserve durable job metadata --- apps/uptime/src/types.ts | 2 +- apps/uptime/src/worker.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/uptime/src/types.ts b/apps/uptime/src/types.ts index a4301f0262..68b9729e5a 100644 --- a/apps/uptime/src/types.ts +++ b/apps/uptime/src/types.ts @@ -41,7 +41,7 @@ export const uptimeCheckJobDataSchema = z.object({ delivery: z.object({ event: requiredUnknownSchema }).optional(), scheduleId: z.string(), trigger: z.enum(["manual", "scheduled"]), -}); +}).passthrough(); export const uptimeDeliveryJobDataSchema = z.object({ event: requiredUnknownSchema, diff --git a/apps/uptime/src/worker.test.ts b/apps/uptime/src/worker.test.ts index 0309b7bd8c..0a1cac8c3a 100644 --- a/apps/uptime/src/worker.test.ts +++ b/apps/uptime/src/worker.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it } from "bun:test"; +import type { UptimeCheckJobData } from "@databuddy/redis"; import type { ScheduleData } from "./actions"; import type { UptimeData } from "./types"; import { @@ -229,6 +230,33 @@ describe("processUptimeCheck", () => { ); }); + it("preserves unknown job metadata when checkpointing a delivery", async () => { + let checkpoint: unknown; + const data = { + scheduleId: "schedule-1", + trigger: "manual", + futureMetadata: { retryToken: "retry-1" }, + } as UptimeCheckJobData & { + futureMetadata: { retryToken: string }; + }; + + await processUptimeJob( + { + name: "uptime-check", + data, + updateData: async (updated) => { + checkpoint = updated; + }, + }, + deps() + ); + + expect(checkpoint).toMatchObject({ + delivery: { event: uptimeData() }, + futureMetadata: { retryToken: "retry-1" }, + }); + }); + it("rejects malformed BullMQ job envelopes before accessing their fields", async () => { await expect( processUptimeJob( From 5c3b505d5013af15111873b90264af7c79ef9817 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:39:35 +0300 Subject: [PATCH 18/25] refactor(analytics): derive uptime replay identity --- packages/ai/src/query/builders/uptime.test.ts | 19 +++++++++++++ packages/ai/src/query/builders/uptime.ts | 27 ++----------------- packages/rpc/src/routers/status-page.ts | 27 ++----------------- 3 files changed, 23 insertions(+), 50 deletions(-) create mode 100644 packages/ai/src/query/builders/uptime.test.ts diff --git a/packages/ai/src/query/builders/uptime.test.ts b/packages/ai/src/query/builders/uptime.test.ts new file mode 100644 index 0000000000..5312565ea3 --- /dev/null +++ b/packages/ai/src/query/builders/uptime.test.ts @@ -0,0 +1,19 @@ +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; +import { describe, expect, it } from "vitest"; +import { UptimeBuilders } from "./uptime"; + +describe("UptimeBuilders", () => { + it("deduplicates complete physical rows without sorting them first", () => { + const query = UptimeBuilders.uptime_overview?.customSql?.({ + endDate: "2026-08-03", + startDate: "2026-08-01", + websiteId: "website-1", + }); + const sql = query?.sql.replaceAll(/\s+/g, " ").trim() ?? ""; + + expect(sql).toContain( + `LIMIT 1 BY ${TABLE_COLUMNS["uptime.uptime_monitor"].join(", ")}` + ); + expect(sql).not.toContain("ORDER BY timestamp DESC LIMIT 1 BY"); + }); +}); diff --git a/packages/ai/src/query/builders/uptime.ts b/packages/ai/src/query/builders/uptime.ts index ab0e0c5057..eed0c9f3af 100644 --- a/packages/ai/src/query/builders/uptime.ts +++ b/packages/ai/src/query/builders/uptime.ts @@ -1,3 +1,4 @@ +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; import type { SimpleQueryConfig } from "../types"; /** @@ -18,30 +19,7 @@ import type { SimpleQueryConfig } from "../types"; */ const UPTIME_TABLE = "uptime.uptime_monitor"; -const UPTIME_REPLAY_IDENTITY = [ - "site_id", - "url", - "timestamp", - "status", - "http_code", - "ttfb_ms", - "total_ms", - "attempt", - "retries", - "failure_streak", - "response_bytes", - "content_hash", - "redirect_count", - "probe_region", - "probe_ip", - "ssl_expiry", - "ssl_valid", - "env", - "check_type", - "user_agent", - "error", - "json_data", -].join(", "); +const UPTIME_REPLAY_IDENTITY = TABLE_COLUMNS[UPTIME_TABLE].join(", "); /** * A replay keeps the original check payload and timestamp. The deployed @@ -55,7 +33,6 @@ function uptimeEventSource(scope: string): string { FROM ${UPTIME_TABLE} WHERE ${scope} - ORDER BY timestamp DESC LIMIT 1 BY ${UPTIME_REPLAY_IDENTITY} )`; } diff --git a/packages/rpc/src/routers/status-page.ts b/packages/rpc/src/routers/status-page.ts index 59c471beb5..6764f2742e 100644 --- a/packages/rpc/src/routers/status-page.ts +++ b/packages/rpc/src/routers/status-page.ts @@ -1,5 +1,6 @@ import { and, db, desc, eq, inArray, withTransaction } from "@databuddy/db"; import { chQuery } from "@databuddy/db/clickhouse"; +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; import { incidentAffectedMonitors, incidentUpdates, @@ -33,30 +34,7 @@ import { } from "./status-page-health"; const UPTIME_TABLE = "uptime.uptime_monitor"; -const UPTIME_REPLAY_IDENTITY = [ - "site_id", - "url", - "timestamp", - "status", - "http_code", - "ttfb_ms", - "total_ms", - "attempt", - "retries", - "failure_streak", - "response_bytes", - "content_hash", - "redirect_count", - "probe_region", - "probe_ip", - "ssl_expiry", - "ssl_valid", - "env", - "check_type", - "user_agent", - "error", - "json_data", -].join(", "); +const UPTIME_REPLAY_IDENTITY = TABLE_COLUMNS[UPTIME_TABLE].join(", "); /** * Replaying a persisted delivery preserves its original check payload and timestamp. @@ -69,7 +47,6 @@ function uptimeEventSource(scope: string): string { FROM ${UPTIME_TABLE} WHERE ${scope} - ORDER BY timestamp DESC LIMIT 1 BY ${UPTIME_REPLAY_IDENTITY} )`; } From 6e47ee66551745df07b5b271d467959865a8b97e Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:41:04 +0300 Subject: [PATCH 19/25] test(ai): isolate Redis barrel mocks --- packages/ai/src/ai/agents/cache.test.ts | 23 +++++--- .../ai/src/ai/mcp/conversation-store.test.ts | 31 ++++++---- packages/ai/src/test-utils/redis-mock.ts | 56 +++++++++++++++++++ 3 files changed, 92 insertions(+), 18 deletions(-) create mode 100644 packages/ai/src/test-utils/redis-mock.ts diff --git a/packages/ai/src/ai/agents/cache.test.ts b/packages/ai/src/ai/agents/cache.test.ts index 33105755d6..b53d10f00b 100644 --- a/packages/ai/src/ai/agents/cache.test.ts +++ b/packages/ai/src/ai/agents/cache.test.ts @@ -1,5 +1,5 @@ -import * as actualRedis from "@databuddy/redis"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createInertRedisModule } from "../../test-utils/redis-mock"; interface RedisEntry { ttl: number; @@ -45,6 +45,9 @@ const mockEnrichAgentContext = vi.fn( }) => `fresh:${opts.organizationId ?? opts.userId}:${opts.websiteId}` ); +const bunRedisModule = + typeof Bun === "undefined" ? undefined : await import("@databuddy/redis"); + vi.mock("@databuddy/auth", () => ({ auth: {}, websitesApi: { @@ -52,12 +55,18 @@ vi.mock("@databuddy/auth", () => ({ }, })); -const redisModule = { ...actualRedis }; - -vi.mock("@databuddy/redis", () => ({ - ...redisModule, - getRedisCache: () => mockRedisClient, -})); +vi.mock("@databuddy/redis", async (importOriginal) => { + const actualRedis = importOriginal + ? await importOriginal() + : bunRedisModule; + if (!actualRedis) { + throw new Error("Unable to load the Redis test module"); + } + return createInertRedisModule(actualRedis, { + getRedisCache: () => mockRedisClient, + redis: mockRedisClient, + }); +}); vi.mock("../../lib/supermemory", () => ({ formatMemoryForPrompt: vi.fn(() => ""), diff --git a/packages/ai/src/ai/mcp/conversation-store.test.ts b/packages/ai/src/ai/mcp/conversation-store.test.ts index 4edcb70fe9..18ae1a2c41 100644 --- a/packages/ai/src/ai/mcp/conversation-store.test.ts +++ b/packages/ai/src/ai/mcp/conversation-store.test.ts @@ -1,5 +1,5 @@ -import * as actualRedis from "@databuddy/redis"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createInertRedisModule } from "../../test-utils/redis-mock"; const redisStore = new Map(); let failGet = false; @@ -22,17 +22,26 @@ const mockRedisClient = { }), }; -const redisModule = { ...actualRedis }; +const bunRedisModule = + typeof Bun === "undefined" ? undefined : await import("@databuddy/redis"); -vi.mock("@databuddy/redis", () => ({ - ...redisModule, - getRedisCache: () => { - if (redisUnavailable) { - throw new Error("redis unavailable"); - } - return mockRedisClient; - }, -})); +vi.mock("@databuddy/redis", async (importOriginal) => { + const actualRedis = importOriginal + ? await importOriginal() + : bunRedisModule; + if (!actualRedis) { + throw new Error("Unable to load the Redis test module"); + } + return createInertRedisModule(actualRedis, { + getRedisCache: () => { + if (redisUnavailable) { + throw new Error("redis unavailable"); + } + return mockRedisClient; + }, + redis: mockRedisClient, + }); +}); const { appendToConversation, getConversationHistory } = await import( "./conversation-store" diff --git a/packages/ai/src/test-utils/redis-mock.ts b/packages/ai/src/test-utils/redis-mock.ts new file mode 100644 index 0000000000..48fbfdf8d5 --- /dev/null +++ b/packages/ai/src/test-utils/redis-mock.ts @@ -0,0 +1,56 @@ +type RedisModule = typeof import("@databuddy/redis"); + +const noop = () => undefined; +const noopAsync = async () => undefined; + +const pureFunctionExports = [ + "activeStreamKey", + "getAgentContextSnapshotKey", + "getCacheableKey", + "getCacheableTagIndexKey", + "getLinkCacheKey", + "getRateLimitHeaders", + "insightsResumeJobId", + "insightsWebsiteJobId", + "streamBufferKey", + "uptimeDeliveryJobId", + "uptimeImmediateJobId", + "uptimeSchedulerId", +] as const satisfies readonly (keyof RedisModule)[]; + +/** + * Preserves the full Redis barrel shape while making every effectful function + * inert by default. New function exports therefore cannot open Redis or + * BullMQ connections from a test that only needs a small mocked surface. + */ +export function createInertRedisModule( + actualRedis: RedisModule, + options: { getRedisCache: () => unknown; redis: unknown } +): RedisModule { + const module: Record = Object.fromEntries( + Object.entries(actualRedis).map(([name, value]) => [ + name, + typeof value === "function" ? noop : value, + ]) + ); + + for (const name of pureFunctionExports) { + module[name] = actualRedis[name]; + } + + return { + ...module, + cacheable: unknown>(fn: T) => fn, + createDrizzleCache: () => ({ + cleanupEmptySets: noopAsync, + invalidateByKey: noopAsync, + invalidateByTables: noopAsync, + invalidateByTags: noopAsync, + withCache: async ({ queryFn }: { queryFn: () => Promise }) => + queryFn(), + }), + default: options.redis, + getRedisCache: options.getRedisCache, + redis: options.redis, + } as unknown as RedisModule; +} From ae193fdf4a39a84542b33365b9a366ff7b08df94 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:41:10 +0300 Subject: [PATCH 20/25] style(uptime): format job validation --- apps/uptime/src/types.ts | 12 +++++++----- apps/uptime/src/worker.ts | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/uptime/src/types.ts b/apps/uptime/src/types.ts index 68b9729e5a..0b225fa3d3 100644 --- a/apps/uptime/src/types.ts +++ b/apps/uptime/src/types.ts @@ -37,11 +37,13 @@ const requiredUnknownSchema = z .unknown() .refine((value) => value !== undefined, "Required"); -export const uptimeCheckJobDataSchema = z.object({ - delivery: z.object({ event: requiredUnknownSchema }).optional(), - scheduleId: z.string(), - trigger: z.enum(["manual", "scheduled"]), -}).passthrough(); +export const uptimeCheckJobDataSchema = z + .object({ + delivery: z.object({ event: requiredUnknownSchema }).optional(), + scheduleId: z.string(), + trigger: z.enum(["manual", "scheduled"]), + }) + .passthrough(); export const uptimeDeliveryJobDataSchema = z.object({ event: requiredUnknownSchema, diff --git a/apps/uptime/src/worker.ts b/apps/uptime/src/worker.ts index 0794c86fb3..1a4aa10335 100644 --- a/apps/uptime/src/worker.ts +++ b/apps/uptime/src/worker.ts @@ -590,9 +590,7 @@ export function startUptimeWorker() { captureError(error, { error_step: "uptime_worker_job_failed", - schedule_id: parsedJobData?.success - ? parsedJobData.data.scheduleId - : "", + schedule_id: parsedJobData?.success ? parsedJobData.data.scheduleId : "", job_id: job?.id ?? "", trigger: parsedJobData?.success ? parsedJobData.data.trigger : "", attempts_used: attemptsMade, From 6d9bb85af0c8e26b5b898da86ec1c1cbe5c3f902 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:23:58 +0300 Subject: [PATCH 21/25] refactor(basket): simplify acknowledged delivery --- apps/basket/package.json | 1 - apps/basket/src/index.ts | 16 +- .../src/lib/event-service.delivery.test.ts | 33 +- apps/basket/src/lib/event-service.ts | 117 ++- apps/basket/src/lib/producer.delivery.test.ts | 107 ++- apps/basket/src/lib/producer.kafka.test.ts | 39 +- apps/basket/src/lib/producer.test.ts | 39 +- apps/basket/src/lib/producer.ts | 713 +++++++----------- apps/basket/src/lib/security.test.ts | 105 +-- apps/basket/src/lib/security.ts | 115 +-- apps/basket/src/routes/integration.test.ts | 8 +- bun.lock | 1 - 12 files changed, 502 insertions(+), 792 deletions(-) diff --git a/apps/basket/package.json b/apps/basket/package.json index 419e2ab675..6df1e552a4 100644 --- a/apps/basket/package.json +++ b/apps/basket/package.json @@ -24,7 +24,6 @@ "@maxmind/geoip2-node": "^6.3.4", "@types/ua-parser-js": "^0.7.39", "async-mutex": "^0.5.0", - "effect": "^4.0.0-beta.90", "elysia": "catalog:", "evlog": "catalog:", "kafkajs": "^2.2.4", diff --git a/apps/basket/src/index.ts b/apps/basket/src/index.ts index 0d87a76a4c..b1f5f28550 100644 --- a/apps/basket/src/index.ts +++ b/apps/basket/src/index.ts @@ -8,12 +8,7 @@ import { import { shutdownPostgres } from "@databuddy/db"; import { clickHouse } from "@databuddy/db/clickhouse"; import { getRedisCache } from "@databuddy/redis/redis"; -import { - disconnect, - disposeRuntime, - runPromise, - ShutdownDrainError, -} from "@lib/producer"; +import { disconnect, ShutdownDrainError } from "@lib/producer"; import { Kafka } from "kafkajs"; import { databuddyEvlogRedaction } from "@databuddy/shared/evlog-redaction"; import { @@ -86,7 +81,7 @@ async function gracefulShutdown(signal: string, exitCode = 0) { const { shutdownRedis } = await import("@databuddy/redis"); // Wait for acknowledged delivery before tearing down its dependencies. try { - await runPromise(disconnect); + await disconnect(); } catch (error) { finalExitCode = 1; if (error instanceof ShutdownDrainError) { @@ -100,13 +95,6 @@ async function gracefulShutdown(signal: string, exitCode = 0) { } else { logErr("producerDrain")(error); } - } finally { - try { - await disposeRuntime(); - } catch (error) { - finalExitCode = 1; - logErr("runtimeDispose")(error); - } } await Promise.all([ shutdownRedis().catch(logErr("redisShutdown")), diff --git a/apps/basket/src/lib/event-service.delivery.test.ts b/apps/basket/src/lib/event-service.delivery.test.ts index dda91d9562..a8631817b3 100644 --- a/apps/basket/src/lib/event-service.delivery.test.ts +++ b/apps/basket/src/lib/event-service.delivery.test.ts @@ -9,7 +9,6 @@ const { mockParseUserAgent, mockReleaseDuplicateReservation, mockReserveDuplicate, - mockRunPromise, mockSend, mockSendBatch, mockShouldAnonymizeVisitorIds, @@ -35,14 +34,12 @@ const { token: "reservation-token", }) ), - mockRunPromise: vi.fn(() => Promise.resolve()), - mockSend: vi.fn(() => ({ _tag: "MockEffect" })), - mockSendBatch: vi.fn(() => ({ _tag: "MockEffect" })), + mockSend: vi.fn(() => Promise.resolve()), + mockSendBatch: vi.fn(() => Promise.resolve()), mockShouldAnonymizeVisitorIds: vi.fn(() => false), })); vi.mock("@lib/producer", () => ({ - runPromise: mockRunPromise, send: mockSend, sendBatch: mockSendBatch, })); @@ -117,9 +114,8 @@ describe("core event delivery", () => { }); mockParseUserAgent.mockResolvedValue({ browserName: "Chrome" }); mockReserveDuplicate.mockResolvedValue(trackReservation); - mockRunPromise.mockResolvedValue(undefined); - mockSend.mockReturnValue({ _tag: "MockEffect" }); - mockSendBatch.mockReturnValue({ _tag: "MockEffect" }); + mockSend.mockResolvedValue(undefined); + mockSendBatch.mockResolvedValue(undefined); mockShouldAnonymizeVisitorIds.mockReturnValue(false); }); @@ -136,7 +132,6 @@ describe("core event delivery", () => { "analytics-events", expect.objectContaining({ client_id: "ws_test", event_name: "pageview" }) ); - expect(mockRunPromise).toHaveBeenCalledWith(mockSend.mock.results[0]?.value); expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( trackReservation ); @@ -145,7 +140,7 @@ describe("core event delivery", () => { test("releases a track reservation and returns retryable 503 when delivery fails", async () => { const deliveryFailure = new Error("ClickHouse unavailable"); - mockRunPromise.mockRejectedValueOnce(deliveryFailure); + mockSend.mockRejectedValueOnce(deliveryFailure); await expect( insertTrackEvent( @@ -167,13 +162,14 @@ describe("core event delivery", () => { test("reuses the same ClickHouse UUID when a client retries a failed track event", async () => { const deliveredEvents: Array<{ id: string }> = []; + let attempts = 0; mockSend.mockImplementation((_topic, event) => { deliveredEvents.push(event as { id: string }); - return { _tag: "MockEffect" }; + attempts += 1; + return attempts === 1 + ? Promise.reject(new Error("ambiguous delivery")) + : Promise.resolve(); }); - mockRunPromise - .mockRejectedValueOnce(new Error("ambiguous delivery")) - .mockResolvedValueOnce(undefined); await expect( insertTrackEvent( @@ -211,7 +207,7 @@ describe("core event delivery", () => { ttl: 86_400, }; mockReserveDuplicate.mockResolvedValueOnce(reservation); - mockRunPromise.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + mockSend.mockRejectedValueOnce(new Error("ClickHouse unavailable")); await expect( insertOutgoingLink( @@ -229,7 +225,6 @@ describe("core event delivery", () => { "analytics-outgoing-links", expect.objectContaining({ client_id: "ws_test" }) ); - expect(mockRunPromise).toHaveBeenCalledWith(mockSend.mock.results[0]?.value); expect(mockReleaseDuplicateReservation).toHaveBeenCalledWith(reservation); }); @@ -245,7 +240,6 @@ describe("core event delivery", () => { ); expect(mockSend).not.toHaveBeenCalled(); - expect(mockRunPromise).not.toHaveBeenCalled(); }); test("returns retryable 503 without publishing a track event owned by another request", async () => { @@ -265,7 +259,6 @@ describe("core event delivery", () => { ).rejects.toMatchObject({ status: 503 }); expect(mockSend).not.toHaveBeenCalled(); - expect(mockRunPromise).not.toHaveBeenCalled(); expect(mockMarkDuplicateReservationDelivered).not.toHaveBeenCalled(); expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); }); @@ -289,7 +282,6 @@ describe("core event delivery", () => { ).rejects.toMatchObject({ status: 503 }); expect(mockSend).not.toHaveBeenCalled(); - expect(mockRunPromise).not.toHaveBeenCalled(); expect(mockMarkDuplicateReservationDelivered).not.toHaveBeenCalled(); expect(mockReleaseDuplicateReservation).not.toHaveBeenCalled(); }); @@ -307,7 +299,6 @@ describe("core event delivery", () => { ); expect(mockSend).toHaveBeenCalledOnce(); - expect(mockRunPromise).toHaveBeenCalledOnce(); expect(mockMarkDuplicateReservationDelivered).toHaveBeenCalledWith( unavailableReservation ); @@ -363,7 +354,7 @@ describe("core event delivery", () => { }); test("releases batch reservations when durable delivery fails", async () => { - mockRunPromise.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + mockSendBatch.mockRejectedValueOnce(new Error("ClickHouse unavailable")); await expect(insertTrackEventsBatch([trackBatchItem("evt_1")])).rejects.toMatchObject({ status: 503, diff --git a/apps/basket/src/lib/event-service.ts b/apps/basket/src/lib/event-service.ts index 39e10dd00b..fd5e668702 100644 --- a/apps/basket/src/lib/event-service.ts +++ b/apps/basket/src/lib/event-service.ts @@ -5,7 +5,7 @@ import type { WebVitalsSpansInsert, } from "@databuddy/db/clickhouse/tables"; import type { ErrorSpan, IndividualVital } from "@databuddy/validation"; -import { runPromise, send, sendBatch } from "@lib/producer"; +import { send, sendBatch } from "@lib/producer"; import { getDailySalt, applyVisitorIdPrivacy, @@ -152,6 +152,36 @@ export function buildTrackEvent( }; } +async function deliverReserved( + deliveryId: string, + eventType: "outgoing_link" | "track", + sourceEventId: string, + deliver: () => Promise +): Promise { + const reservation = await reserveDuplicate( + deliveryId, + eventType, + sourceEventId + ); + if (reservation.duplicate) { + return; + } + if (reservation.retryable) { + throw deliveryUnavailable( + new Error("A concurrent attempt owns this analytics event") + ); + } + + try { + await deliver(); + } catch (error) { + await releaseDuplicateReservation(reservation); + throw deliveryUnavailable(error); + } + + await markDuplicateReservationDelivered(reservation); +} + export function insertTrackEvent( trackData: any, clientId: string, @@ -161,26 +191,14 @@ export function insertTrackEvent( ): Promise { return record("insertTrackEvent", async () => { const log = useLogger(); - let eventId = sanitizeString( - trackData.eventId, - VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH - ); - if (!eventId) { - eventId = randomUUIDv7(); - } + const eventId = + sanitizeString( + trackData.eventId, + VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH + ) || randomUUIDv7(); const deliveryId = stableAnalyticsEventId(clientId, "track", eventId); - const reservation = await reserveDuplicate(deliveryId, "track", eventId); - if (reservation.duplicate) { - return; - } - if (reservation.retryable) { - throw deliveryUnavailable( - new Error("A concurrent attempt owns this analytics event") - ); - } - - try { + await deliverReserved(deliveryId, "track", eventId, async () => { const geoData = await getGeo(ip, request); const trustedCountry = extractTrustedClientIp(request) ? geoData.country @@ -220,13 +238,8 @@ export function insertTrackEvent( now, }); - await runPromise(send("analytics-events", trackEvent)); - } catch (error) { - await releaseDuplicateReservation(reservation); - throw deliveryUnavailable(error); - } - - await markDuplicateReservationDelivered(reservation); + await send("analytics-events", trackEvent); + }); }); } @@ -237,35 +250,18 @@ export function insertOutgoingLink( ): Promise { return record("insertOutgoingLink", async () => { const log = useLogger(); - let eventId = sanitizeString( - linkData.eventId, - VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH - ); - - if (!eventId) { - eventId = randomUUIDv7(); - } + const eventId = + sanitizeString( + linkData.eventId, + VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH + ) || randomUUIDv7(); const deliveryId = stableAnalyticsEventId( clientId, "outgoing_link", eventId ); - const reservation = await reserveDuplicate( - deliveryId, - "outgoing_link", - eventId - ); - if (reservation.duplicate) { - return; - } - if (reservation.retryable) { - throw deliveryUnavailable( - new Error("A concurrent attempt owns this analytics event") - ); - } - - try { + await deliverReserved(deliveryId, "outgoing_link", eventId, async () => { log.set({ event: { id: eventId, type: "outgoing_link", href: linkData.href }, }); @@ -301,13 +297,8 @@ export function insertOutgoingLink( typeof linkData.timestamp === "number" ? linkData.timestamp : now, }; - await runPromise(send("analytics-outgoing-links", outgoingLinkEvent)); - } catch (error) { - await releaseDuplicateReservation(reservation); - throw deliveryUnavailable(error); - } - - await markDuplicateReservationDelivered(reservation); + await send("analytics-outgoing-links", outgoingLinkEvent); + }); }); } @@ -362,11 +353,9 @@ async function deliverBatch( } try { - await runPromise( - sendBatch( - topic, - accepted.map(({ item }) => item.event) - ) + await sendBatch( + topic, + accepted.map(({ item }) => item.event) ); } catch (error) { await Promise.allSettled( @@ -437,7 +426,7 @@ export function insertErrorSpans( ) || "Error", })); - await runPromise(sendBatch("analytics-error-spans", spans)); + await sendBatch("analytics-error-spans", spans); }); } @@ -472,7 +461,7 @@ export function insertIndividualVitals( metric_value: vital.metricValue, })); - await runPromise(sendBatch("analytics-vitals-spans", spans)); + await sendBatch("analytics-vitals-spans", spans); }); } @@ -552,6 +541,6 @@ export function insertCustomEvents( : undefined, })); - await runPromise(sendBatch("analytics-custom-events", spans)); + await sendBatch("analytics-custom-events", spans); }); } diff --git a/apps/basket/src/lib/producer.delivery.test.ts b/apps/basket/src/lib/producer.delivery.test.ts index 3c67a47bf1..9c75b877c6 100644 --- a/apps/basket/src/lib/producer.delivery.test.ts +++ b/apps/basket/src/lib/producer.delivery.test.ts @@ -1,5 +1,5 @@ import type { ClickHouseClient } from "@clickhouse/client"; -import { Effect } from "effect"; +import type { Producer } from "kafkajs"; import { describe, expect, test, vi } from "vitest"; import type { ProducerConfig } from "./producer"; @@ -26,7 +26,7 @@ vi.mock("@lib/tracing", () => ({ record: (_name: string, fn: () => Promise) => fn(), })); -const { createProducerEffects } = await import("./producer"); +const { createProducer } = await import("./producer"); const topicMap = { "analytics-events": "analytics.events" }; @@ -49,26 +49,24 @@ const event = (id: string) => ({ timestamp: 1, }); -async function makeEffects( +function makeProducer( insert: (input: unknown) => Promise, config: Partial = {} ) { - return Effect.runPromise( - createProducerEffects( - { ...baseConfig, ...config }, - null, - { insert } as unknown as ClickHouseClient, - topicMap - ) + return createProducer( + { ...baseConfig, ...config }, + null, + { insert } as unknown as ClickHouseClient, + topicMap ); } describe("producer delivery guarantees", () => { test("resolves a core send only after direct ClickHouse fallback succeeds", async () => { const insert = vi.fn(() => Promise.resolve()); - const effects = await makeEffects(insert); + const producer = makeProducer(insert); - await Effect.runPromise(effects.sendOne("analytics-events", event("event_1"))); + await producer.sendOne("analytics-events", event("event_1")); expect(insert).toHaveBeenCalledWith( expect.objectContaining({ @@ -77,29 +75,24 @@ describe("producer delivery guarantees", () => { values: [event("event_1")], }) ); - const stats = await Effect.runPromise(effects.stats); - expect(stats).toMatchObject({ inFlight: 0, sent: 1 }); }); test("returns a retryable error instead of acknowledging a failed direct fallback", async () => { - const effects = await makeEffects(() => Promise.reject(new Error("offline"))); + const producer = makeProducer(() => Promise.reject(new Error("offline"))); await expect( - Effect.runPromise(effects.sendOne("analytics-events", event("event_1"))) + producer.sendOne("analytics-events", event("event_1")) ).rejects.toMatchObject({ _tag: "ClickHouseFallbackError", retryable: true, table: "analytics.events", topic: "analytics-events", }); - - const stats = await Effect.runPromise(effects.stats); - expect(stats).toMatchObject({ errors: 1, inFlight: 0, sent: 0 }); }); - test("keeps an active direct delivery counted when shutdown rejects another send", async () => { + test("does not admit a send after shutdown starts while delivery is active", async () => { let releaseInsert: (() => void) | undefined; - const effects = await makeEffects( + const producer = makeProducer( () => new Promise((resolve) => { releaseInsert = resolve; @@ -107,22 +100,17 @@ describe("producer delivery guarantees", () => { { shutdownDrainTimeout: 500 } ); - const activeDelivery = Effect.runPromise( - effects.sendOne("analytics-events", event("event_1")) - ); + const activeDelivery = producer.sendOne("analytics-events", event("event_1")); await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); - const shutdown = Effect.runPromise(effects.shutDown); + const shutdown = producer.shutDown(); await expect( - Effect.runPromise(effects.sendOne("analytics-events", event("event_2"))) + producer.sendOne("analytics-events", event("event_2")) ).rejects.toMatchObject({ _tag: "ProducerShuttingDownError", retryable: true, }); - const stats = await Effect.runPromise(effects.stats); - expect(stats.inFlight).toBe(1); - releaseInsert?.(); await activeDelivery; await shutdown; @@ -130,7 +118,7 @@ describe("producer delivery guarantees", () => { test("reports an in-flight direct delivery when shutdown reaches its deadline", async () => { let releaseInsert: (() => void) | undefined; - const effects = await makeEffects( + const producer = makeProducer( () => new Promise((resolve) => { releaseInsert = resolve; @@ -138,11 +126,9 @@ describe("producer delivery guarantees", () => { { shutdownDrainTimeout: 20 } ); - const activeDelivery = Effect.runPromise( - effects.sendOne("analytics-events", event("event_1")) - ); + const activeDelivery = producer.sendOne("analytics-events", event("event_1")); await vi.waitFor(() => expect(releaseInsert).toBeTypeOf("function")); - await expect(Effect.runPromise(effects.shutDown)).rejects.toMatchObject({ + await expect(producer.shutDown()).rejects.toMatchObject({ _tag: "ShutdownDrainError", inFlight: 1, retryable: true, @@ -151,4 +137,57 @@ describe("producer delivery guarantees", () => { releaseInsert?.(); await activeDelivery; }); + + test("shares a cold Kafka connection across concurrent sends", async () => { + let resolveConnect: (() => void) | undefined; + const connect = vi.fn( + () => + new Promise((resolve) => { + resolveConnect = resolve; + }) + ); + const send = vi.fn(() => Promise.resolve()); + const kafka = { + connect, + disconnect: vi.fn(() => Promise.resolve()), + send, + } as unknown as Producer; + const producer = createProducer( + { ...baseConfig, broker: "redpanda.test:9092", selfHost: false }, + kafka, + { insert: vi.fn(() => Promise.resolve()) } as unknown as ClickHouseClient, + topicMap + ); + + const first = producer.sendOne("analytics-events", event("event_1")); + const second = producer.sendOne("analytics-events", event("event_2")); + await vi.waitFor(() => expect(connect).toHaveBeenCalledOnce()); + resolveConnect?.(); + + await Promise.all([first, second]); + expect(send).toHaveBeenCalledTimes(2); + }); + + test("falls back when Kafka connect throws synchronously", async () => { + const insert = vi.fn(() => Promise.resolve()); + const kafka = { + connect: vi.fn(() => { + throw new Error("Kafka client is not ready"); + }), + disconnect: vi.fn(() => Promise.resolve()), + send: vi.fn(() => Promise.resolve()), + } as unknown as Producer; + const producer = createProducer( + { ...baseConfig, broker: "redpanda.test:9092", selfHost: false }, + kafka, + { insert } as unknown as ClickHouseClient, + topicMap + ); + + await producer.sendOne("analytics-events", event("event_1")); + + expect(insert).toHaveBeenCalledWith( + expect.objectContaining({ values: [event("event_1")] }) + ); + }); }); diff --git a/apps/basket/src/lib/producer.kafka.test.ts b/apps/basket/src/lib/producer.kafka.test.ts index 9e8b38ef22..f424db02ca 100644 --- a/apps/basket/src/lib/producer.kafka.test.ts +++ b/apps/basket/src/lib/producer.kafka.test.ts @@ -72,9 +72,7 @@ vi.mock("@lib/tracing", () => ({ record: (_name: string, fn: Function) => Promise.resolve().then(() => fn()), })); -const { disconnect, disposeRuntime, getStats, runPromise, send } = await import( - "./producer" -); +const { disconnect, send } = await import("./producer"); beforeEach(() => { mockCaptureError.mockClear(); @@ -82,7 +80,6 @@ beforeEach(() => { }); afterAll(async () => { - await disposeRuntime().catch(() => {}); for (const [key, value] of Object.entries(originalEnv)) { if (value === undefined) { delete process.env[key]; @@ -94,35 +91,25 @@ afterAll(async () => { describe("producer Kafka send failure handling", () => { test("falls back to direct ClickHouse writes and still disconnects on shutdown", async () => { - await runPromise( - send("analytics-events", { - client_id: "ws_1", - event_id: "event_1", - timestamp: Date.now(), - }) - ); - - await runPromise( - send("analytics-events", { - client_id: "ws_1", - event_id: "event_2", - timestamp: Date.now(), - }) - ); - - const stats = await runPromise(getStats); + await send("analytics-events", { + client_id: "ws_1", + event_id: "event_1", + timestamp: Date.now(), + }); + + await send("analytics-events", { + client_id: "ws_1", + event_id: "event_2", + timestamp: Date.now(), + }); expect(mockKafka).toHaveBeenCalledTimes(1); expect(mockProducer).toHaveBeenCalledTimes(1); expect(mockConnect).toHaveBeenCalledTimes(1); expect(mockSend).toHaveBeenCalledTimes(1); expect(mockClickHouseInsert).toHaveBeenCalledTimes(2); - expect(stats?.sent).toBe(2); - expect(stats?.connected).toBe(false); - expect(stats?.failed).toBe(true); - expect(stats?.failedCount).toBe(1); - await runPromise(disconnect); + await disconnect(); expect(mockDisconnect).toHaveBeenCalledTimes(1); }); diff --git a/apps/basket/src/lib/producer.test.ts b/apps/basket/src/lib/producer.test.ts index 09532266fa..cce3ffd1a5 100644 --- a/apps/basket/src/lib/producer.test.ts +++ b/apps/basket/src/lib/producer.test.ts @@ -29,12 +29,7 @@ vi.mock("@lib/tracing", () => ({ record: (_name: string, fn: Function) => Promise.resolve().then(() => fn()), })); -const { - disposeRuntime, - getStats, - runPromise, - send, -} = await import("./producer"); +const { send } = await import("./producer"); beforeEach(async () => { mockCaptureError.mockClear(); @@ -42,7 +37,6 @@ beforeEach(async () => { }); afterAll(async () => { - await disposeRuntime().catch(() => {}); if (originalSelfHost === undefined) { delete process.env.SELFHOST; } else { @@ -50,19 +44,14 @@ afterAll(async () => { } }); -describe("producer fallback topics", () => { + describe("producer fallback topics", () => { test("blocked traffic is delivered through the direct ClickHouse fallback", async () => { - await runPromise( - send("analytics-blocked-traffic", { - id: "blocked_1", - client_id: "ws_1", - timestamp: Date.now(), - }) - ); + await send("analytics-blocked-traffic", { + id: "blocked_1", + client_id: "ws_1", + timestamp: Date.now(), + }); - const stats = await runPromise(getStats); - expect(stats?.errors).toBe(0); - expect(stats?.sent).toBe(1); expect(mockClickHouseInsert).toHaveBeenCalledWith( expect.objectContaining({ table: "analytics.blocked_traffic", @@ -75,21 +64,17 @@ describe("producer fallback topics", () => { test("unknown topics include the missing topic in error context", async () => { await expect( - runPromise( - send("analytics-unmapped-topic", { - id: "evt_1", - client_id: "ws_1", - timestamp: Date.now(), - }) - ) + send("analytics-unmapped-topic", { + id: "evt_1", + client_id: "ws_1", + timestamp: Date.now(), + }) ).rejects.toMatchObject({ _tag: "UnknownKafkaTopicError", retryable: false, topic: "analytics-unmapped-topic", }); - const stats = await runPromise(getStats); - expect(stats?.errors).toBe(1); expect(mockCaptureError).toHaveBeenCalledWith( expect.objectContaining({ message: "Unknown Kafka topic" }), { topic: "analytics-unmapped-topic" } diff --git a/apps/basket/src/lib/producer.ts b/apps/basket/src/lib/producer.ts index 522bb8a3e4..2f7ae64066 100644 --- a/apps/basket/src/lib/producer.ts +++ b/apps/basket/src/lib/producer.ts @@ -2,7 +2,6 @@ import type { ClickHouseClient } from "@clickhouse/client"; import { clickHouse, TABLE_NAMES } from "@databuddy/db/clickhouse"; import { readBooleanEnv } from "@databuddy/env/boolean"; import { captureError, record } from "@lib/tracing"; -import { Data, Effect, Layer, ManagedRuntime, Ref } from "effect"; import { createError } from "evlog"; import { CompressionTypes, Kafka, type Producer } from "kafkajs"; @@ -12,61 +11,76 @@ function stringifyEvent(event: unknown): string { ); } -export class KafkaConnectionError extends Data.TaggedError( - "KafkaConnectionError" -)<{ readonly cause?: Error }> {} -export class KafkaSendError extends Data.TaggedError("KafkaSendError")<{ - readonly topic: string; - readonly cause?: Error; -}> {} -export class ProducerShuttingDownError extends Data.TaggedError( - "ProducerShuttingDownError" -)<{ +function toError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +export class ProducerShuttingDownError extends Error { + readonly _tag = "ProducerShuttingDownError"; readonly eventCount: number; - readonly retryable: true; -}> {} -export class ProducerUnavailableError extends Data.TaggedError( - "ProducerUnavailableError" -)<{ readonly retryable: true }> {} -export class UnknownKafkaTopicError extends Data.TaggedError( - "UnknownKafkaTopicError" -)<{ - readonly retryable: false; + readonly retryable = true; + + constructor({ eventCount }: { eventCount: number }) { + super("Basket producer is shutting down"); + this.name = this._tag; + this.eventCount = eventCount; + } +} + +export class UnknownKafkaTopicError extends Error { + readonly _tag = "UnknownKafkaTopicError"; + readonly retryable = false; readonly topic: string; -}> {} -export class ClickHouseFallbackError extends Data.TaggedError( - "ClickHouseFallbackError" -)<{ - readonly cause?: Error; - readonly retryable: true; + + constructor({ topic }: { topic: string }) { + super(`Unknown Kafka topic: ${topic}`); + this.name = this._tag; + this.topic = topic; + } +} + +export class ClickHouseFallbackError extends Error { + readonly _tag = "ClickHouseFallbackError"; + override readonly cause?: Error; + readonly retryable = true; readonly table: string; readonly topic: string; -}> {} -export class ShutdownDrainError extends Data.TaggedError("ShutdownDrainError")<{ + + constructor({ + cause, + table, + topic, + }: { + cause?: Error; + table: string; + topic: string; + }) { + super(`Direct ClickHouse fallback insert failed for ${topic}`); + this.name = this._tag; + this.cause = cause; + this.table = table; + this.topic = topic; + } +} + +export class ShutdownDrainError extends Error { + readonly _tag = "ShutdownDrainError"; readonly deadlineMs: number; readonly inFlight: number; - readonly retryable: true; -}> {} - -export type ProducerError = - | KafkaConnectionError - | KafkaSendError - | ProducerShuttingDownError - | ProducerUnavailableError - | UnknownKafkaTopicError - | ClickHouseFallbackError; - -interface ProducerState { - connected: boolean; - connectionFailed: boolean; - errors: number; - failedCount: number; - inFlight: number; - lastErrorTime: number | null; - lastRetry: number; - producerInitialized: boolean; - sent: number; - shuttingDown: boolean; + readonly retryable = true; + + constructor({ + deadlineMs, + inFlight, + }: { + deadlineMs: number; + inFlight: number; + }) { + super("Basket producer did not drain before shutdown"); + this.name = this._tag; + this.deadlineMs = deadlineMs; + this.inFlight = inFlight; + } } export interface ProducerConfig { @@ -82,35 +96,10 @@ export interface ProducerConfig { username?: string; } -export interface ProducerEffects { - sendMany: ( - topic: string, - events: unknown[] - ) => Effect.Effect; - sendOne: ( - topic: string, - event: unknown, - key?: string - ) => Effect.Effect; - shutDown: Effect.Effect; - stats: Effect.Effect; -} - -const INITIAL_STATE: ProducerState = { - sent: 0, - failedCount: 0, - errors: 0, - lastErrorTime: null, - connected: false, - connectionFailed: false, - lastRetry: 0, - producerInitialized: false, - shuttingDown: false, - inFlight: 0, -}; - -function toError(err: unknown): Error { - return err instanceof Error ? err : new Error(String(err)); +export interface BasketProducer { + sendMany: (topic: string, events: unknown[]) => Promise; + sendOne: (topic: string, event: unknown, key?: string) => Promise; + shutDown: () => Promise; } async function insertClickHouseChunks( @@ -118,353 +107,231 @@ async function insertClickHouseChunks( table: string, events: unknown[], chunkSize: number -) { - for (let i = 0; i < events.length; i += chunkSize) { +): Promise { + for (let index = 0; index < events.length; index += chunkSize) { await ch.insert({ table, - values: events.slice(i, i + chunkSize), + values: events.slice(index, index + chunkSize), format: "JSONEachRow", }); } } -function makeProducerEffects( +export function createProducer( config: ProducerConfig, kafka: Producer | null, ch: ClickHouseClient, - topicMap: Record, - ref: Ref.Ref -): ProducerEffects { - const enabled = !config.selfHost && Boolean(config.broker); - - const inc = (field: keyof ProducerState, n = 1) => - Ref.update(ref, (s) => ({ ...s, [field]: (s[field] as number) + n })); - - const connect: Effect.Effect = Effect.gen(function* () { - if (!(enabled && kafka)) { - return (yield* Ref.get(ref)).connected; + topicMap: Record +): BasketProducer { + const kafkaEnabled = + !config.selfHost && Boolean(config.broker) && kafka !== null; + let connected = false; + let connectionFailed = false; + let connecting: Promise | null = null; + let inFlight = 0; + let lastRetry = 0; + let producerInitialized = false; + let shuttingDown = false; + + function connect(): Promise { + if (!(kafkaEnabled && kafka)) { + return Promise.resolve(false); + } + if (connected) { + return Promise.resolve(true); } - const s = yield* Ref.get(ref); - if (s.connected) { - return true; + if (connectionFailed && Date.now() - lastRetry < config.reconnectCooldown) { + return Promise.resolve(false); } - if ( - s.connectionFailed && - Date.now() - s.lastRetry < config.reconnectCooldown - ) { - return false; + if (connecting) { + return connecting; } - return yield* Effect.tryPromise({ - try: () => kafka.connect(), - catch: (e) => new KafkaConnectionError({ cause: toError(e) }), - }).pipe( - Effect.tap(() => - Ref.update(ref, (st) => ({ - ...st, - connected: true, - connectionFailed: false, - lastRetry: 0, - producerInitialized: true, - })) - ), - Effect.as(true), - Effect.catchTag("KafkaConnectionError", (err) => - Ref.update(ref, (st) => ({ - ...st, - connectionFailed: true, - lastRetry: Date.now(), - errors: st.errors + 1, - lastErrorTime: Date.now(), - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(err.cause, { - message: - "Redpanda connection failed, using ClickHouse fallback", - }) - ) - ), - Effect.as(false) - ) - ) - ); - }); + connecting = Promise.resolve() + .then(() => kafka.connect()) + .then(() => { + connected = true; + connectionFailed = false; + lastRetry = 0; + producerInitialized = true; + return true; + }) + .catch((error) => { + connectionFailed = true; + lastRetry = Date.now(); + captureError(toError(error), { + message: "Redpanda connection failed, using ClickHouse fallback", + }); + return false; + }) + .finally(() => { + connecting = null; + }); - const resolveTopic = ( - topic: string - ): Effect.Effect => - Effect.gen(function* () { - const table = topicMap[topic]; - if (table) { - return table; - } + return connecting; + } - yield* inc("errors", 1); - yield* Effect.sync(() => - captureError( - createError({ - code: "basket.UNKNOWN_KAFKA_TOPIC", - message: "Unknown Kafka topic", - status: 500, - why: `Topic "${topic}" is not mapped to a ClickHouse table.`, - fix: "Check topicMap configuration.", - }), - { topic } - ) - ); - return yield* Effect.fail( - new UnknownKafkaTopicError({ retryable: false, topic }) - ); - }); + function resolveTopic(topic: string): string { + const table = topicMap[topic]; + if (table) { + return table; + } - const persistDirectly = ( + captureError( + createError({ + code: "basket.UNKNOWN_KAFKA_TOPIC", + message: "Unknown Kafka topic", + status: 500, + why: `Topic "${topic}" is not mapped to a ClickHouse table.`, + fix: "Check topicMap configuration.", + }), + { topic } + ); + throw new UnknownKafkaTopicError({ topic }); + } + + async function persistDirectly( topic: string, events: unknown[] - ): Effect.Effect => - Effect.gen(function* () { - const table = yield* resolveTopic(topic); - yield* Effect.tryPromise({ - try: () => - record("clickhouseDirectFallbackInsert", () => - insertClickHouseChunks(ch, table, events, config.chunkSize) - ), - catch: (e) => - new ClickHouseFallbackError({ - cause: toError(e), - retryable: true, - table, - topic, - }), - }).pipe( - Effect.tap(() => inc("sent", events.length)), - Effect.catchTag("ClickHouseFallbackError", (error) => - Ref.update(ref, (s) => ({ - ...s, - errors: s.errors + 1, - lastErrorTime: Date.now(), - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(error.cause, { - message: - "Direct ClickHouse fallback insert failed; rejecting delivery", - table, - topic, - }) - ) - ), - Effect.flatMap(() => Effect.fail(error)) - ) - ) + ): Promise { + const table = resolveTopic(topic); + try { + await record("clickhouseDirectFallbackInsert", () => + insertClickHouseChunks(ch, table, events, config.chunkSize) ); - }); + } catch (error) { + const fallbackError = new ClickHouseFallbackError({ + cause: toError(error), + table, + topic, + }); + captureError(fallbackError.cause, { + message: "Direct ClickHouse fallback insert failed; rejecting delivery", + table, + topic, + }); + throw fallbackError; + } + } - const acquireSendSlot = (eventCount: number) => - Ref.modify(ref, (s) => { - if (s.shuttingDown) { - return [false, s] as const; - } - return [true, { ...s, inFlight: s.inFlight + eventCount }] as const; - }).pipe( - Effect.flatMap((accepted) => - accepted - ? Effect.void - : Effect.fail( - new ProducerShuttingDownError({ - eventCount, - retryable: true, - }) - ) - ) - ); + function acquireSendSlot(eventCount: number): void { + if (shuttingDown) { + throw new ProducerShuttingDownError({ eventCount }); + } + inFlight += eventCount; + } - const releaseSendSlot = (eventCount: number) => - Ref.update(ref, (s) => ({ - ...s, - inFlight: Math.max(0, s.inFlight - eventCount), - })); + function releaseSendSlot(eventCount: number): void { + inFlight = Math.max(0, inFlight - eventCount); + } - const sendViaKafka = ( + async function sendViaKafka( topic: string, messages: Array<{ value: string; key?: string }>, fallbackEvents: unknown[] - ): Effect.Effect => - acquireSendSlot(fallbackEvents.length).pipe( - Effect.flatMap(() => - Effect.gen(function* () { - if (enabled && kafka) { - const isConnected = yield* connect; - if (isConnected) { - const sent = yield* Effect.tryPromise({ - try: () => - kafka.send({ - topic, - messages, - timeout: config.kafkaTimeout, - compression: CompressionTypes.GZIP, - }), - catch: (e) => new KafkaSendError({ topic, cause: toError(e) }), - }).pipe( - Effect.tap(() => inc("sent", messages.length)), - Effect.as(true), - Effect.catchTag("KafkaSendError", (err) => - Ref.update(ref, (st) => ({ - ...st, - connectionFailed: true, - connected: false, - lastRetry: Date.now(), - failedCount: st.failedCount + messages.length, - })).pipe( - Effect.tap(() => - Effect.sync(() => - captureError(err.cause, { - message: - "Redpanda send failed, falling back to ClickHouse", - message_count: messages.length, - topic, - }) - ) - ), - Effect.as(false) - ) - ) - ); - if (sent) { - return; - } - } - } - - yield* persistDirectly(topic, fallbackEvents); - }).pipe(Effect.ensuring(releaseSendSlot(fallbackEvents.length))) - ) - ); + ): Promise { + acquireSendSlot(fallbackEvents.length); - const sendOne = ( - topic: string, - event: unknown, - key?: string - ): Effect.Effect => - sendViaKafka( - topic, - [ - { - value: stringifyEvent(event), - key: key || (event as { client_id?: string }).client_id, - }, - ], - [event] - ); + try { + if (kafkaEnabled && kafka && (await connect())) { + try { + await kafka.send({ + topic, + messages, + timeout: config.kafkaTimeout, + compression: CompressionTypes.GZIP, + }); + return; + } catch (error) { + connectionFailed = true; + connected = false; + lastRetry = Date.now(); + captureError(toError(error), { + message: "Redpanda send failed, falling back to ClickHouse", + message_count: messages.length, + topic, + }); + } + } - const sendMany = ( - topic: string, - events: unknown[] - ): Effect.Effect => { - if (events.length === 0) { - return Effect.void; + await persistDirectly(topic, fallbackEvents); + } finally { + releaseSendSlot(fallbackEvents.length); } - return sendViaKafka( - topic, - events.map((e) => ({ - value: stringifyEvent(e), - key: - (e as { client_id?: string }).client_id || - (e as { event_id?: string }).event_id, - })), - events - ); - }; + } - const disconnectKafka = Effect.gen(function* () { - const state = yield* Ref.get(ref); - if (!(state.producerInitialized && kafka)) { + async function disconnectKafka(): Promise { + if (!(producerInitialized && kafka)) { return; } - yield* Effect.tryPromise({ - try: () => kafka.disconnect(), - catch: (e) => new KafkaConnectionError({ cause: toError(e) }), - }).pipe( - Effect.ensuring( - Ref.update(ref, (s) => ({ - ...s, - connected: false, - producerInitialized: false, - })) - ), - Effect.catch((err) => - Effect.sync(() => - captureError(err.cause, { - message: "Error disconnecting Redpanda producer", - }) - ) - ) - ); - }); - - const drainInFlight = Effect.gen(function* () { - while ((yield* Ref.get(ref)).inFlight > 0) { - yield* Effect.sleep("10 millis"); + try { + await kafka.disconnect(); + } catch (error) { + captureError(toError(error), { + message: "Error disconnecting Redpanda producer", + }); + } finally { + connected = false; + producerInitialized = false; } - }); + } - const shutDown: Effect.Effect = Effect.gen( - function* () { - yield* Ref.update(ref, (s) => ({ ...s, shuttingDown: true })); - yield* drainInFlight.pipe( - Effect.timeout(`${config.shutdownDrainTimeout} millis`), - Effect.catch(() => - Ref.get(ref).pipe( - Effect.flatMap((state) => - Effect.fail( - new ShutdownDrainError({ - deadlineMs: config.shutdownDrainTimeout, - inFlight: state.inFlight, - retryable: true, - }) - ) - ) - ) - ), - Effect.ensuring(disconnectKafka) + async function waitForInFlight(): Promise { + const deadline = Date.now() + config.shutdownDrainTimeout; + while (inFlight > 0) { + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new ShutdownDrainError({ + deadlineMs: config.shutdownDrainTimeout, + inFlight, + }); + } + await new Promise((resolve) => + setTimeout(resolve, Math.min(10, remaining)) ); } - ); - - const stats: Effect.Effect = Ref.get(ref).pipe( - Effect.map( - ({ - shuttingDown: _s, - connectionFailed, - producerInitialized: _p, - ...rest - }) => ({ - ...rest, - failed: connectionFailed, - kafkaEnabled: enabled, - }) - ) - ); + } return { - sendOne, - sendMany, - shutDown, - stats, + async sendOne(topic, event, key): Promise { + await sendViaKafka( + topic, + [ + { + value: stringifyEvent(event), + key: key || (event as { client_id?: string }).client_id, + }, + ], + [event] + ); + }, + async sendMany(topic, events): Promise { + if (events.length === 0) { + return; + } + await sendViaKafka( + topic, + events.map((event) => ({ + value: stringifyEvent(event), + key: + (event as { client_id?: string }).client_id || + (event as { event_id?: string }).event_id, + })), + events + ); + }, + async shutDown(): Promise { + shuttingDown = true; + try { + await waitForInFlight(); + } finally { + await disconnectKafka(); + } + }, }; } -export const createProducerEffects = ( - config: ProducerConfig, - kafka: Producer | null, - ch: ClickHouseClient, - topicMap: Record -): Effect.Effect => - Ref.make(INITIAL_STATE).pipe( - Effect.map((ref) => makeProducerEffects(config, kafka, ch, topicMap, ref)) - ); - function initializeKafka(config: ProducerConfig): Producer | null { if (config.selfHost || !config.broker) { return null; @@ -505,18 +372,6 @@ function initializeKafka(config: ProducerConfig): Producer | null { }); } -export interface ProducerStatsSnapshot { - connected: boolean; - errors: number; - failed: boolean; - failedCount: number; - inFlight: number; - kafkaEnabled: boolean; - lastErrorTime: number | null; - lastRetry: number; - sent: number; -} - const CONFIG: ProducerConfig = { broker: process.env.REDPANDA_BROKER, username: process.env.REDPANDA_USER, @@ -541,61 +396,25 @@ const TOPIC_MAP: Record = { "analytics-link-visits": TABLE_NAMES.link_visits, }; -let fx: ReturnType | null = null; - -const ProducerLive = Layer.effectDiscard( - Effect.gen(function* () { - const effects = yield* createProducerEffects( - CONFIG, - initializeKafka(CONFIG), - clickHouse, - TOPIC_MAP - ); - fx = effects; - }) +const producer = createProducer( + CONFIG, + initializeKafka(CONFIG), + clickHouse, + TOPIC_MAP ); -const runtime = ManagedRuntime.make(ProducerLive); - -const withFx = ( - fn: (f: NonNullable) => Effect.Effect -): Effect.Effect => - Effect.suspend(() => (fx ? fn(fx) : Effect.succeed(undefined))); - -const withActiveFx = ( - fn: (f: NonNullable) => Effect.Effect -): Effect.Effect => - Effect.suspend( - (): Effect.Effect => - fx - ? fn(fx) - : Effect.fail(new ProducerUnavailableError({ retryable: true })) - ); - export const send = (topic: string, event: unknown, key?: string) => - withActiveFx((f) => f.sendOne(topic, event, key)); + producer.sendOne(topic, event, key); export const sendBatch = (topic: string, events: unknown[]) => - withActiveFx((f) => f.sendMany(topic, events)); - -export const disconnect = withFx((f) => f.shutDown); - -export const getStats = withFx((f) => f.stats); - -export const runFork = (effect: Effect.Effect) => - runtime.runFork( - effect.pipe( - Effect.tapError((error) => - Effect.sync(() => - captureError(error, { - message: "Asynchronous producer delivery rejected", - }) - ) - ) - ) - ); + producer.sendMany(topic, events); -export const runPromise = (effect: Effect.Effect) => - runtime.runPromise(effect); +export const disconnect = () => producer.shutDown(); -export const disposeRuntime = () => runtime.dispose(); +export function runFork(delivery: Promise): void { + delivery.catch((error) => { + captureError(error, { + message: "Asynchronous producer delivery rejected", + }); + }); +} diff --git a/apps/basket/src/lib/security.test.ts b/apps/basket/src/lib/security.test.ts index c30c582334..edfcd16c5a 100644 --- a/apps/basket/src/lib/security.test.ts +++ b/apps/basket/src/lib/security.test.ts @@ -118,18 +118,15 @@ describe("visitor ID anonymization helpers", () => { // ── duplicate reservations (needs Redis mock) ── -const { mockRedisSet, mockRedisGet, mockRedisEval, mockLoggerSet, mockCaptureError } = - vi.hoisted(() => ({ - mockRedisSet: vi.fn(() => Promise.resolve("OK")), - mockRedisGet: vi.fn(() => Promise.resolve(null)), - mockRedisEval: vi.fn(() => Promise.resolve(1)), - mockLoggerSet: vi.fn(() => {}), - mockCaptureError: vi.fn(), - })); +const { mockRedisEval, mockLoggerSet, mockCaptureError } = vi.hoisted(() => ({ + mockRedisEval: vi.fn(), + mockLoggerSet: vi.fn(() => {}), + mockCaptureError: vi.fn(), +})); vi.mock("@databuddy/redis/redis", () => ({ - redis: { set: mockRedisSet, get: mockRedisGet, eval: mockRedisEval }, - getRedisCache: () => ({ set: mockRedisSet }), + redis: { eval: mockRedisEval }, + getRedisCache: () => ({}), })); vi.mock("@databuddy/redis/cacheable", () => ({ cacheable: (fn: () => Promise) => fn, @@ -147,15 +144,13 @@ vi.mock("@lib/tracing", () => ({ describe("duplicate reservations", () => { beforeEach(() => { - mockRedisSet.mockReset(); - mockRedisGet.mockReset(); mockRedisEval.mockReset(); mockLoggerSet.mockReset(); mockCaptureError.mockReset(); }); - test("writes a pending reservation for a new event", async () => { - mockRedisSet.mockResolvedValue("OK"); + test("atomically writes a pending reservation for a new event", async () => { + mockRedisEval.mockResolvedValue("acquired"); const reservation = await reserveDuplicate("evt_1", "track"); @@ -165,18 +160,17 @@ describe("duplicate reservations", () => { token: expect.stringMatching(/^pending:/), ttl: 86_400, }); - expect(mockRedisSet).toHaveBeenCalledWith( + expect(mockRedisEval).toHaveBeenCalledWith( + expect.stringContaining('redis.call("GET", KEYS[1])'), + 1, "dedup:track:evt_1", expect.stringMatching(/^pending:/), - "EX", - 86_400, - "NX" + 86_400 ); }); test("suppresses only a confirmed delivered reservation", async () => { - mockRedisSet.mockResolvedValue(null); - mockRedisGet.mockResolvedValue("delivered"); + mockRedisEval.mockResolvedValue("delivered"); const reservation = await reserveDuplicate("evt_1", "track"); @@ -187,9 +181,7 @@ describe("duplicate reservations", () => { }); test("claims a legacy admission reservation before delivery", async () => { - mockRedisSet.mockResolvedValue(null); - mockRedisGet.mockResolvedValue("1"); - mockRedisEval.mockResolvedValue(1); + mockRedisEval.mockResolvedValue("acquired"); const reservation = await reserveDuplicate("evt_1", "track"); @@ -200,86 +192,59 @@ describe("duplicate reservations", () => { ttl: 86_400, }); expect(mockRedisEval).toHaveBeenCalledWith( - expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), + expect.stringContaining('if value == "1" then'), 1, "dedup:track:evt_1", - "1", expect.stringMatching(/^pending:/), 86_400 ); }); - test("requires a retry when a legacy reservation cannot be claimed", async () => { - mockRedisSet.mockResolvedValue(null); - mockRedisGet.mockResolvedValue("1"); - mockRedisEval.mockResolvedValue(0); - - const reservation = await reserveDuplicate("evt_1", "track"); - - expect(reservation).toEqual({ duplicate: false, retryable: true }); - }); - test("requires a retry when another request owns a pending reservation", async () => { - mockRedisSet.mockResolvedValue(null); - mockRedisGet.mockResolvedValue("pending:other-attempt"); - - const reservation = await reserveDuplicate("evt_1", "track"); - - expect(reservation).toEqual({ duplicate: false, retryable: true }); - }); - - test("requires a retry when a lost reservation vanishes before it can be read", async () => { - mockRedisSet.mockResolvedValue(null); - mockRedisGet.mockResolvedValue(null); + mockRedisEval.mockResolvedValue("pending"); const reservation = await reserveDuplicate("evt_1", "track"); expect(reservation).toEqual({ duplicate: false, retryable: true }); }); - test("recovers ownership when an ambiguous SET wrote this request's token", async () => { - mockRedisSet.mockResolvedValue(null); - mockRedisGet.mockImplementation(async () => mockRedisSet.mock.calls[0]?.[1]); + test("recovers ownership when an ambiguous EVAL wrote this request's token", async () => { + mockRedisEval + .mockRejectedValueOnce(new Error("stale connection")) + .mockResolvedValueOnce("acquired"); const reservation = await reserveDuplicate("evt_1", "track"); - expect(reservation.token).toBe(mockRedisSet.mock.calls[0]?.[1]); + expect(reservation).toMatchObject({ + duplicate: false, + token: expect.stringMatching(/^pending:/), + }); + expect(mockRedisEval).toHaveBeenCalledTimes(2); + expect(mockRedisEval.mock.calls[1]).toEqual(mockRedisEval.mock.calls[0]); + expect(mockCaptureError).not.toHaveBeenCalled(); }); test("preserves the exit-event retention when its delivery id is hashed", async () => { - mockRedisSet.mockResolvedValue("OK"); + mockRedisEval.mockResolvedValue("acquired"); await reserveDuplicate("stable-delivery-id", "track", "exit_abc"); - expect(mockRedisSet).toHaveBeenCalledWith( + expect(mockRedisEval).toHaveBeenCalledWith( + expect.any(String), + 1, "dedup:track:stable-delivery-id", expect.stringMatching(/^pending:/), - "EX", - 172_800, - "NX" + 172_800 ); }); - test("retries a Redis error before returning a retryable reservation", async () => { - mockRedisSet - .mockRejectedValueOnce(new Error("stale connection")) - .mockResolvedValueOnce("OK"); - - const reservation = await reserveDuplicate("evt_1", "track"); - - expect(reservation.duplicate).toBe(false); - expect(mockRedisSet).toHaveBeenCalledTimes(2); - expect(mockCaptureError).not.toHaveBeenCalled(); - }); - test("fails open for a Redis outage without turning an event into a duplicate", async () => { - mockRedisSet.mockRejectedValue(new Error("Redis down")); - mockRedisGet.mockRejectedValue(new Error("Redis down")); + mockRedisEval.mockRejectedValue(new Error("Redis down")); const reservation = await reserveDuplicate("evt_1", "track"); expect(reservation).toEqual({ duplicate: false }); - expect(mockRedisSet).toHaveBeenCalledTimes(2); + expect(mockRedisEval).toHaveBeenCalledTimes(2); expect(mockCaptureError).toHaveBeenCalledOnce(); }); diff --git a/apps/basket/src/lib/security.ts b/apps/basket/src/lib/security.ts index 143af897ad..ec07c0a095 100644 --- a/apps/basket/src/lib/security.ts +++ b/apps/basket/src/lib/security.ts @@ -14,15 +14,26 @@ const DELIVERED_DEDUP_VALUE = "delivered"; // Kept until all keys written by the pre-reservation implementation expire. // Unlike "delivered", this value was written before delivery was acknowledged. const LEGACY_PENDING_DEDUP_VALUE = "1"; +const RESERVE_DEDUP_RESERVATION = `local value = redis.call("GET", KEYS[1]) +if value == false then + redis.call("SET", KEYS[1], ARGV[1], "EX", ARGV[2]) + return "acquired" +end +if value == ARGV[1] then + return "acquired" +end +if value == "${DELIVERED_DEDUP_VALUE}" then + return "delivered" +end +if value == "${LEGACY_PENDING_DEDUP_VALUE}" then + redis.call("SET", KEYS[1], ARGV[1], "EX", ARGV[2]) + return "acquired" +end +return "pending"`; const RELEASE_PENDING_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) end return 0`; -const CLAIM_LEGACY_DEDUP_RESERVATION = `if redis.call("GET", KEYS[1]) == ARGV[1] then - redis.call("SET", KEYS[1], ARGV[2], "EX", ARGV[3]) - return 1 -end -return 0`; const MARK_PENDING_DEDUP_RESERVATION_DELIVERED = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("SET", KEYS[1], ARGV[2], "EX", ARGV[3]) end @@ -137,10 +148,6 @@ export function applyVisitorIdPrivacy( return saltAnonymousId(sanitized, salt); } -function wait(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export interface DuplicateReservation { readonly duplicate: boolean; readonly key?: string; @@ -159,80 +166,30 @@ export interface DuplicateReservation { readonly ttl?: number; } -type DedupReservationState = - | "acquired" - | "delivered" - | "legacy" - | "pending" - | "unreserved"; - -async function readDedupReservationState( - key: string, - token: string -): Promise { - const value = await redis.get(key); - if (value === DELIVERED_DEDUP_VALUE) { - return "delivered"; - } - if (value === LEGACY_PENDING_DEDUP_VALUE) { - return "legacy"; - } - if (value === token) { - return "acquired"; - } - return value === null || value === undefined ? "unreserved" : "pending"; -} - -async function resolveDedupReservationState( +async function reserveDedupKey( key: string, ttl: number, token: string -): Promise { - const state = await readDedupReservationState(key, token); - if (state !== "legacy") { - return state; - } - - const claimed = await redis.eval( - CLAIM_LEGACY_DEDUP_RESERVATION, - 1, - key, - LEGACY_PENDING_DEDUP_VALUE, - token, - ttl - ); - return claimed === 1 ? "acquired" : readDedupReservationState(key, token); -} +): Promise<"acquired" | "delivered" | "pending"> { + const reserve = async () => { + const result = await redis.eval( + RESERVE_DEDUP_RESERVATION, + 1, + key, + token, + ttl + ); + return result === "acquired" || result === "delivered" ? result : "pending"; + }; -async function setDedupKey( - key: string, - ttl: number, - token: string -): Promise { try { - const result = await redis.set(key, token, "EX", ttl, "NX"); - return result === null - ? resolveDedupReservationState(key, ttl, token) - : "acquired"; + return await reserve(); } catch (firstError) { - await wait(DEDUP_RETRY_DELAY_MS); + await new Promise((resolve) => setTimeout(resolve, DEDUP_RETRY_DELAY_MS)); try { - const retryResult = await redis.set(key, token, "EX", ttl, "NX"); - if (retryResult !== null) { - return "acquired"; - } - - // The first SET may have succeeded even though the client saw an error. - // Seeing our token again proves that this request owns the reservation. - // Any other pending token is retryable: only a confirmed delivery may - // suppress the event. - return await resolveDedupReservationState(key, ttl, token); + return await reserve(); } catch { - try { - return await resolveDedupReservationState(key, ttl, token); - } catch { - throw firstError; - } + throw firstError; } } } @@ -250,7 +207,7 @@ export function reserveDuplicate( const token = `${PENDING_DEDUP_PREFIX}${crypto.randomUUID()}`; try { - const result = await setDedupKey(key, ttl, token); + const result = await reserveDedupKey(key, ttl, token); if (result === "delivered") { useLogger().set({ dedup: { duplicate: true, eventType } }); return { duplicate: true }; @@ -258,11 +215,7 @@ export function reserveDuplicate( // This request did not acquire a pending key. Publishing without ownership // can duplicate a concurrent delivery or overwrite its confirmed state. - if ( - result === "legacy" || - result === "pending" || - result === "unreserved" - ) { + if (result === "pending") { return { duplicate: false, retryable: true }; } diff --git a/apps/basket/src/routes/integration.test.ts b/apps/basket/src/routes/integration.test.ts index 6c8bf5f7a7..65aad76c49 100644 --- a/apps/basket/src/routes/integration.test.ts +++ b/apps/basket/src/routes/integration.test.ts @@ -3,7 +3,6 @@ import { vi, beforeEach, describe, expect, test } from "vitest"; const { noop, - noopAsync, mockLogger, mockValidateRequest, mockCheckForBot, @@ -23,7 +22,6 @@ const { mockResolveApiKeyOwnerId, } = vi.hoisted(() => { const noop = vi.fn(() => {}); - const noopAsync = vi.fn(() => Promise.resolve()); const defaultApiKey = { id: "key_1", organizationId: "org_1", @@ -40,7 +38,6 @@ const { }; return { noop, - noopAsync, mockLogger: { set: vi.fn(() => {}), warn: vi.fn(() => {}), @@ -173,9 +170,8 @@ vi.mock("@hooks/auth", () => ({ vi.mock("@lib/producer", () => ({ runFork: noop, - send: vi.fn(() => ({})), - sendBatch: vi.fn(() => ({})), - runPromise: noopAsync, + send: vi.fn(() => Promise.resolve()), + sendBatch: vi.fn(() => Promise.resolve()), })); // ── Import routes after mocks ── diff --git a/bun.lock b/bun.lock index 15c5919cc6..d0cb641314 100644 --- a/bun.lock +++ b/bun.lock @@ -106,7 +106,6 @@ "@maxmind/geoip2-node": "^6.3.4", "@types/ua-parser-js": "^0.7.39", "async-mutex": "^0.5.0", - "effect": "^4.0.0-beta.90", "elysia": "catalog:", "evlog": "catalog:", "kafkajs": "^2.2.4", From 4df2431369b2c348f605c5f4aa9ca46d43378335 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:24:04 +0300 Subject: [PATCH 22/25] refactor(uptime): simplify durable delivery --- apps/uptime/src/index.ts | 8 +--- apps/uptime/src/lib/producer.test.ts | 41 ++++++++++++------- apps/uptime/src/lib/producer.ts | 28 ++++--------- apps/uptime/src/worker.test.ts | 13 +++--- apps/uptime/src/worker.ts | 60 ++++++++++------------------ 5 files changed, 64 insertions(+), 86 deletions(-) diff --git a/apps/uptime/src/index.ts b/apps/uptime/src/index.ts index fcac07a5e3..67b75a1e9f 100644 --- a/apps/uptime/src/index.ts +++ b/apps/uptime/src/index.ts @@ -209,20 +209,17 @@ const probe = (_name: string, fn: () => Promise) => const healthCheck = Effect.gen(function* () { const { db, sql } = yield* Effect.promise(() => import("@databuddy/db")); - const { getUptimeDeliveryQueue, getUptimeQueue } = yield* Effect.promise( + const { getUptimeQueue } = yield* Effect.promise( () => import("@databuddy/redis") ); const { Kafka } = yield* Effect.promise(() => import("kafkajs")); - const [postgres, bullmqRedis, deliveryRedis, redpanda] = yield* Effect.all( + const [postgres, bullmqRedis, redpanda] = yield* Effect.all( [ probe("postgres", () => db.execute(sql`SELECT 1`).then(() => {})), probe("bullmqRedis", async () => { await getUptimeQueue().count(); }), - probe("uptimeDeliveryRedis", async () => { - await getUptimeDeliveryQueue().count(); - }), probe("redpanda", async () => { const broker = process.env.REDPANDA_BROKER; if (!broker) { @@ -257,7 +254,6 @@ const healthCheck = Effect.gen(function* () { const services = { postgres, bullmqRedis, - uptimeDeliveryRedis: deliveryRedis, redpanda, }; const status = Object.values(services).every((s) => s.status === "ok") diff --git a/apps/uptime/src/lib/producer.test.ts b/apps/uptime/src/lib/producer.test.ts index 3ad177a6fe..4d36e36357 100644 --- a/apps/uptime/src/lib/producer.test.ts +++ b/apps/uptime/src/lib/producer.test.ts @@ -91,9 +91,7 @@ describe("sendUptimeEvent", () => { expect(resolveConnection).toBeDefined(); resolveConnection?.(); - expect(await Promise.all(sends)).toEqual( - Array.from({ length: 20 }, () => ({ sent: true })) - ); + await Promise.all(sends); expect(producer.send).toHaveBeenCalledTimes(20); expect(producer.send).toHaveBeenCalledWith( expect.objectContaining({ acks: -1 }) @@ -108,25 +106,40 @@ describe("sendUptimeEvent", () => { const recoveredProducer = createProducer(); producers.push(failedProducer, recoveredProducer); - expect(await sendUptimeEvent({ attempt: 1 })).toEqual({ - sent: false, - reason: "send_failed", - }); + await expect(sendUptimeEvent({ attempt: 1 })).rejects.toThrow( + "broker unavailable" + ); expect(failedProducer.disconnect).toHaveBeenCalledTimes(1); - expect(await sendUptimeEvent({ attempt: 2 })).toEqual({ sent: true }); + await expect(sendUptimeEvent({ attempt: 2 })).resolves.toBeUndefined(); expect(recoveredProducer.connect).toHaveBeenCalledTimes(1); expect(recoveredProducer.send).toHaveBeenCalledTimes(1); expect(kafkaConfigs).toHaveLength(2); }); - test("reports an unavailable producer when Kafka is not configured", async () => { + test("reconnects after a failed cold-start connection", async () => { + const failedProducer = createProducer({ + connect: () => Promise.reject(new Error("broker unavailable")), + }); + const recoveredProducer = createProducer(); + producers.push(failedProducer, recoveredProducer); + + await expect(sendUptimeEvent({ attempt: 1 })).rejects.toThrow( + "broker unavailable" + ); + + await expect(sendUptimeEvent({ attempt: 2 })).resolves.toBeUndefined(); + expect(recoveredProducer.connect).toHaveBeenCalledTimes(1); + expect(recoveredProducer.send).toHaveBeenCalledTimes(1); + expect(kafkaConfigs).toHaveLength(2); + }); + + test("rejects when Kafka is not configured", async () => { delete process.env.REDPANDA_BROKER; - expect(await sendUptimeEvent({ ok: true })).toEqual({ - sent: false, - reason: "producer_unavailable", - }); + await expect(sendUptimeEvent({ ok: true })).rejects.toThrow( + "REDPANDA_BROKER not set" + ); expect(kafkaConfigs).toEqual([]); }); @@ -135,7 +148,7 @@ describe("sendUptimeEvent", () => { const producer = createProducer(); producers.push(producer); - expect(await sendUptimeEvent({ ok: true })).toEqual({ sent: true }); + await expect(sendUptimeEvent({ ok: true })).resolves.toBeUndefined(); expect(kafkaConfigs[0]).toEqual( expect.objectContaining({ ssl: true }) ); diff --git a/apps/uptime/src/lib/producer.ts b/apps/uptime/src/lib/producer.ts index 939a8097ba..46974e362e 100644 --- a/apps/uptime/src/lib/producer.ts +++ b/apps/uptime/src/lib/producer.ts @@ -30,24 +30,13 @@ const connectProducer = (): Promise => { }; let singletonProducer: Producer | null = null; -let singletonConnection: Promise | null = null; +let singletonConnection: Promise | null = null; -export type UptimeEventSendResult = - | { sent: true } - | { - sent: false; - reason: "producer_unavailable" | "send_failed"; - }; - -function ensureProducer(): Promise { +function ensureProducer(): Promise { if (singletonProducer) { return Promise.resolve(singletonProducer); } - if (!process.env.REDPANDA_BROKER) { - return Promise.resolve(null); - } - if (singletonConnection) { return singletonConnection; } @@ -60,7 +49,7 @@ function ensureProducer(): Promise { .catch((error) => { captureError(error, { error_step: "kafka_producer_connect" }); singletonProducer = null; - return null; + throw error; }) .finally(() => { singletonConnection = null; @@ -86,11 +75,8 @@ async function resetProducer(producer: Producer): Promise { export async function sendUptimeEvent( event: unknown, key?: string -): Promise { +): Promise { const p = await ensureProducer(); - if (!p) { - return { sent: false, reason: "producer_unavailable" }; - } try { await p.send({ @@ -104,16 +90,16 @@ export async function sendUptimeEvent( ], compression: CompressionTypes.GZIP, }); - return { sent: true }; } catch (error) { captureError(error, { error_step: "kafka_producer_send" }); await resetProducer(p); - return { sent: false, reason: "send_failed" }; + throw error; } } export async function disconnectProducer(): Promise { - const producer = singletonProducer ?? (await singletonConnection); + const producer = + singletonProducer ?? (await singletonConnection?.catch(() => null)); if (!producer) { return; } diff --git a/apps/uptime/src/worker.test.ts b/apps/uptime/src/worker.test.ts index 0a1cac8c3a..787d3a5c8a 100644 --- a/apps/uptime/src/worker.test.ts +++ b/apps/uptime/src/worker.test.ts @@ -120,7 +120,7 @@ function deps(): UptimeWorkerDeps { "enabled" in config && config.enabled === true, lookupSchedule: async () => lookupResult, - sendUptimeEvent: async () => ({ sent: true }), + sendUptimeEvent: async () => {}, reapOrphanScheduler: async (scheduleId: string) => { calls.reaped.push(scheduleId); if (reapBehaviour === "throw") { @@ -595,12 +595,11 @@ describe("processUptimeCheck", () => { expect(calls.delivery).toEqual([]); }); - it("retries a delivery job when Redpanda does not acknowledge it", async () => { + it("retries a delivery job when Redpanda rejects it", async () => { const failingDeps = deps(); - failingDeps.sendUptimeEvent = async () => ({ - sent: false, - reason: "send_failed", - }); + failingDeps.sendUptimeEvent = async () => { + throw new Error("Redpanda send failed"); + }; await expect( processUptimeDeliveryJob( @@ -611,7 +610,7 @@ describe("processUptimeCheck", () => { }, failingDeps ) - ).rejects.toThrow("Redpanda delivery failed: send_failed"); + ).rejects.toThrow("Redpanda send failed"); expect(calls.captureError).toContainEqual( expect.objectContaining({ diff --git a/apps/uptime/src/worker.ts b/apps/uptime/src/worker.ts index 1a4aa10335..f6ba28c81e 100644 --- a/apps/uptime/src/worker.ts +++ b/apps/uptime/src/worker.ts @@ -12,7 +12,7 @@ import { uptimeDeliveryJobId, uptimeSchedulerId, } from "@databuddy/redis"; -import { Worker } from "bullmq"; +import { type Job, Worker } from "bullmq"; import type { RequestLogger } from "evlog"; import { createLogger, log } from "evlog"; import { Cause, Data, Effect, Exit } from "effect"; @@ -23,7 +23,7 @@ import { lookupSchedule, } from "./actions"; import { isHealthExtractionEnabled } from "./json-parser"; -import { sendUptimeEvent, type UptimeEventSendResult } from "./lib/producer"; +import { sendUptimeEvent } from "./lib/producer"; import { captureError } from "./lib/tracing"; import { MonitorStatus, @@ -49,11 +49,6 @@ const REAPABLE_REASONS: ReadonlySet = new Set([ "malformed", ]); -async function defaultReapOrphanScheduler(scheduleId: string): Promise { - const queue = getUptimeQueue(); - await queue.removeJobScheduler(uptimeSchedulerId(scheduleId)); -} - class SchedulePaused extends Data.TaggedError("SchedulePaused")< Record > {} @@ -93,29 +88,26 @@ export interface UptimeWorkerDeps { isHealthExtractionEnabled: (config: unknown) => boolean; lookupSchedule: (scheduleId: string) => Promise>; reapOrphanScheduler: (scheduleId: string) => Promise; - sendUptimeEvent: ( - event: unknown, - key?: string - ) => Promise; -} - -async function defaultEnqueueUptimeDelivery(data: UptimeData): Promise { - await getUptimeDeliveryQueue().add( - UPTIME_DELIVERY_JOB_NAME, - { event: data }, - { jobId: uptimeDeliveryJobId(data.event_id) } - ); + sendUptimeEvent: (event: unknown, key?: string) => Promise; } const uptimeWorkerDeps: UptimeWorkerDeps = { captureError, checkUptime, createLogger: (fields) => createLogger(fields), - enqueueUptimeDelivery: defaultEnqueueUptimeDelivery, + enqueueUptimeDelivery: async (data) => { + await getUptimeDeliveryQueue().add( + UPTIME_DELIVERY_JOB_NAME, + { event: data }, + { jobId: uptimeDeliveryJobId(data.event_id) } + ); + }, getPreviousMonitorStatus, isHealthExtractionEnabled, lookupSchedule, - reapOrphanScheduler: defaultReapOrphanScheduler, + reapOrphanScheduler: async (scheduleId) => { + await getUptimeQueue().removeJobScheduler(uptimeSchedulerId(scheduleId)); + }, sendUptimeEvent, fireTransitionAlerts, }; @@ -138,20 +130,15 @@ export function getUptimeWorkerConcurrency( return parsed; } -export interface UptimeWorkerJob { - attemptsMade?: number; - data: UptimeCheckJobData; - id?: string; - name: string; - updateData: (data: UptimeCheckJobData) => Promise; -} +export type UptimeWorkerJob = Pick< + Job, + "attemptsMade" | "data" | "id" | "name" | "updateData" +>; -export interface UptimeDeliveryWorkerJob { - attemptsMade?: number; - data: UptimeDeliveryJobData; - id?: string; - name: string; -} +export type UptimeDeliveryWorkerJob = Pick< + Job, + "attemptsMade" | "data" | "id" | "name" +>; const timed = ( label: string, @@ -553,10 +540,7 @@ export async function processUptimeDeliveryJob( const data = parsedEvent.data; try { - const result = await deps.sendUptimeEvent(data, data.site_id); - if (!result.sent) { - throw new Error(`Redpanda delivery failed: ${result.reason}`); - } + await deps.sendUptimeEvent(data, data.site_id); } catch (error) { deps.captureError(error, { error_step: "uptime_delivery_send", From 16588fa2fd7d23833e2e0ea89c9c2018240807fd Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:24:08 +0300 Subject: [PATCH 23/25] refactor(ingest): remove redundant Vector defaults --- infra/ingest/vector.yaml | 63 ++-------------------------------------- 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/infra/ingest/vector.yaml b/infra/ingest/vector.yaml index 6a524f266d..9f6540e3de 100644 --- a/infra/ingest/vector.yaml +++ b/infra/ingest/vector.yaml @@ -2,6 +2,9 @@ # each acknowledged ClickHouse sink below holds its Kafka offset until the # warehouse confirms delivery, so an interrupted process replays from Kafka. +acknowledgements: + enabled: true + sources: redpanda: type: kafka @@ -60,12 +63,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -86,12 +83,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -112,12 +103,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -138,12 +123,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -164,12 +143,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -190,12 +163,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -216,12 +183,6 @@ sinks: max_events: 50000 max_bytes: 5000000 timeout_secs: 60 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -242,12 +203,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -268,12 +223,6 @@ sinks: max_events: 5000 max_bytes: 5000000 timeout_secs: 5 - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block date_time_best_effort: true skip_unknown_fields: true encoding: @@ -292,11 +241,5 @@ sinks: password: "${VECTOR_KAFKA_PASSWORD}" tls: enabled: true - acknowledgements: - enabled: true - buffer: - type: memory - max_events: 500 - when_full: block encoding: codec: json From 44ac677d2daac0db5dbf9c1302741ca87da730c4 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:24:11 +0300 Subject: [PATCH 24/25] refactor(tracker): remove redundant delivery paths --- packages/tracker/src/core/tracker.ts | 3 +-- packages/tracker/src/plugins/pixel.ts | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/tracker/src/core/tracker.ts b/packages/tracker/src/core/tracker.ts index d4518fe071..445376cd99 100644 --- a/packages/tracker/src/core/tracker.ts +++ b/packages/tracker/src/core/tracker.ts @@ -649,8 +649,7 @@ export class BaseTracker { deliveryGeneration === this.deliveryGeneration && !result.ok && result.retryable && - !this.shouldBlockQueuedDelivery() && - deliveryGeneration === this.deliveryGeneration + !this.shouldBlockQueuedDelivery() ) { queue.unshift(...items); meta.retryAttempts += 1; diff --git a/packages/tracker/src/plugins/pixel.ts b/packages/tracker/src/plugins/pixel.ts index 2fc9b40b2d..d66df097a3 100644 --- a/packages/tracker/src/plugins/pixel.ts +++ b/packages/tracker/src/plugins/pixel.ts @@ -228,6 +228,4 @@ export function initPixelTracking(tracker: BaseTracker) { // keeps BaseTracker's queue for its fetch-style pixel fallback instead of // dropping it immediately on an unverified load. tracker.sendBeacon = () => false; - - tracker.sendBatchBeacon = () => false; } From 844f2ec32c02c080cfdcb01c1df400d5d25c3554 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:24:15 +0300 Subject: [PATCH 25/25] refactor(analytics): consolidate uptime replay queries --- packages/ai/src/query/builders/uptime.test.ts | 19 ------------------- packages/ai/src/query/simple-builder.test.ts | 5 +++-- packages/rpc/src/routers/status-page.ts | 10 +++------- 3 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 packages/ai/src/query/builders/uptime.test.ts diff --git a/packages/ai/src/query/builders/uptime.test.ts b/packages/ai/src/query/builders/uptime.test.ts deleted file mode 100644 index 5312565ea3..0000000000 --- a/packages/ai/src/query/builders/uptime.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; -import { describe, expect, it } from "vitest"; -import { UptimeBuilders } from "./uptime"; - -describe("UptimeBuilders", () => { - it("deduplicates complete physical rows without sorting them first", () => { - const query = UptimeBuilders.uptime_overview?.customSql?.({ - endDate: "2026-08-03", - startDate: "2026-08-01", - websiteId: "website-1", - }); - const sql = query?.sql.replaceAll(/\s+/g, " ").trim() ?? ""; - - expect(sql).toContain( - `LIMIT 1 BY ${TABLE_COLUMNS["uptime.uptime_monitor"].join(", ")}` - ); - expect(sql).not.toContain("ORDER BY timestamp DESC LIMIT 1 BY"); - }); -}); diff --git a/packages/ai/src/query/simple-builder.test.ts b/packages/ai/src/query/simple-builder.test.ts index 6b8f7b1bd7..b07f6f4023 100644 --- a/packages/ai/src/query/simple-builder.test.ts +++ b/packages/ai/src/query/simple-builder.test.ts @@ -1,3 +1,4 @@ +import { TABLE_COLUMNS } from "@databuddy/db/clickhouse/tables"; import { describe, expect, it } from "vitest"; import { QueryBuilders } from "./builders"; import { @@ -1048,9 +1049,9 @@ describe("SimpleQueryBuilder.compile", () => { const { sql } = compileBuilder(type, config); expect(sql).toContain( - "LIMIT 1 BY site_id, url, timestamp, status, http_code, ttfb_ms, total_ms" + `LIMIT 1 BY ${TABLE_COLUMNS["uptime.uptime_monitor"].join(", ")}` ); - expect(sql).toContain("user_agent, error, json_data"); + expect(sql).not.toContain("ORDER BY timestamp DESC LIMIT 1 BY"); expect(sql).not.toContain("event_id"); expect(sql.match(/FROM uptime\.uptime_monitor/g)).toHaveLength(1); } diff --git a/packages/rpc/src/routers/status-page.ts b/packages/rpc/src/routers/status-page.ts index 6764f2742e..79963be645 100644 --- a/packages/rpc/src/routers/status-page.ts +++ b/packages/rpc/src/routers/status-page.ts @@ -248,12 +248,6 @@ function groupDailyRows(rows: DailyRow[]): Map { return grouped; } -function indexLatestChecks( - rows: LatestCheckRow[] -): Map { - return new Map(rows.map((row) => [row.site_id, row])); -} - function applyIncidentImpacts( monitors: z.infer[], activeIncidents: z.infer[], @@ -422,7 +416,9 @@ async function _fetchStatusPageData( const websiteMap = new Map(websiteRows.map((w) => [w.id, w] as const)); const dailyBySite = groupDailyRows(allDailyData); - const latestBySite = indexLatestChecks(allRecentChecks); + const latestBySite = new Map( + allRecentChecks.map((row) => [row.site_id, row]) + ); const monitors = schedules.map((schedule) => { const siteId = schedule.websiteId ?? schedule.id;