From dabad8b01615792578f344c8cc518a19023b97c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 17:05:03 -0700 Subject: [PATCH 01/13] feat: retry only transient failures, with exponential backoff and jitter Adopt one retry policy across the workspace: retry what a retry could plausibly clear, escalate the wait, bound the sequence, and let exactly one layer own the budget. Two shared primitives, one per language. `moq_net::retry` provides `Backoff` (capped exponential, equal jitter, optional give-up deadline) plus `io_retryable`/`status_retryable`; `@moq/net`'s `Retry` provides the same `Backoff` and a `Terminal` error class. Retryability is classified by `is_retryable()` on `moq_net`, `moq_native` (and each backend error enum), `moq_mux`, and `moq_hls` errors, always as an exhaustive `match` so a new variant has to be classified rather than defaulting to retryable. TypeScript inverts the default on purpose: the browser throws untyped platform errors, and treating those as terminal would strand connections a retry would have recovered, so `Terminal` marks what is settled and everything else is retried. Every retry site the audit flagged now either becomes terminal on a deterministic failure or uses the shared schedule: the native reconnect loop, cluster peer dials, HLS import and export, the RTMP accept loop, the audio playback driver, the JS reconnect loop, and browser capture reopen. The systemd units escalate their restart delay instead of restarting every five seconds forever. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 13 ++ doc/bin/gstreamer.md | 6 +- doc/bin/relay/cluster.md | 8 + js/CLAUDE.md | 1 + js/net/src/connection/connect.ts | 11 +- js/net/src/connection/handshake.ts | 3 +- js/net/src/connection/reload.test.ts | 31 +++ js/net/src/connection/reload.ts | 61 +++--- js/net/src/index.ts | 2 + js/net/src/retry.test.ts | 61 ++++++ js/net/src/retry.ts | 146 ++++++++++++++ js/publish/src/source/camera.ts | 7 +- js/publish/src/source/microphone.ts | 7 +- js/publish/src/source/retry.test.ts | 190 ++++++++++++------ js/publish/src/source/retry.ts | 50 ++++- nix/modules/moq-relay.nix | 7 + packaging/moq-relay/moq-relay.service | 7 + rs/CLAUDE.md | 1 + rs/moq-audio/src/playback/driver.rs | 37 ++-- rs/moq-gst/src/sink/session.rs | 10 +- rs/moq-hls/src/error.rs | 65 ++++++ rs/moq-hls/src/export/mod.rs | 27 ++- rs/moq-hls/src/import.rs | 36 +++- rs/moq-mux/src/error.rs | 14 ++ rs/moq-native/src/error.rs | 71 +++++++ rs/moq-native/src/iroh.rs | 34 ++++ rs/moq-native/src/noq.rs | 52 +++++ rs/moq-native/src/quiche.rs | 53 +++++ rs/moq-native/src/quinn.rs | 57 ++++++ rs/moq-native/src/reconnect.rs | 64 ++++-- rs/moq-native/src/tcp.rs | 22 ++ rs/moq-native/src/unix.rs | 17 ++ rs/moq-native/src/websocket.rs | 22 ++ rs/moq-native/tests/reconnect.rs | 77 +++++++ rs/moq-net/src/error.rs | 80 ++++++++ rs/moq-net/src/lib.rs | 1 + rs/moq-net/src/retry.rs | 278 ++++++++++++++++++++++++++ rs/moq-relay/src/cluster.rs | 62 ++++-- rs/moq-rtmp/src/server.rs | 26 ++- 39 files changed, 1537 insertions(+), 180 deletions(-) create mode 100644 js/net/src/retry.test.ts create mode 100644 js/net/src/retry.ts create mode 100644 rs/moq-native/tests/reconnect.rs create mode 100644 rs/moq-net/src/retry.rs diff --git a/CLAUDE.md b/CLAUDE.md index 2678695774..086540aab6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,6 +97,19 @@ Don't document deprecated flags, options, or APIs. User-facing docs (`/doc`), `- The rename/removal rationale lives in the commit message and PR description, not in docs that users read. Warning someone who *uses* the deprecated path is not just fine but encouraged -- at compile time (Rust's `#[deprecated(note = "...")]`) or at runtime (a log line). Those fire on use, so they reach the one person who needs them and nobody else; they aren't documentation. A standing note in the docs that advertises the dead name is what's banned. +## Retries + +Retrying is the reflex that hides bugs, so a new retry loop has to answer four questions in the code, not in the reviewer's head. + +- **What is worth retrying?** Only failures a retry could plausibly clear: a dropped connection, a timed-out request, an OS error that isn't a missing file or an occupied port, and the narrow "ask again later" HTTP set (408, 429, 502, 503, 504). Auth rejections, parse and transmux failures, unsupported features, invalid URLs, and configuration errors fail identically forever; surface them instead. Retryable is the explicit case, never the fallback, so classification lives in an exhaustive `match` on the error type and a new variant forces a decision. Rust: `moq_net::Error::is_retryable` and the same method on `moq_native`, `moq_mux`, and `moq_hls` errors. TypeScript is the deliberate exception (the browser throws untyped platform errors, and defaulting those to terminal would strand recoverable connections), so it inverts: `@moq/net`'s `Retry.Terminal` marks what is settled and everything else is retried. +- **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Use the shared primitive rather than a hand-rolled `sleep`: `moq_net::retry::Backoff` in Rust, `Retry.Backoff` from `@moq/net` in TypeScript. +- **When does it stop?** A deadline or an attempt budget. Unlimited retries belong only to a supervisor whose job is to outlive an outage (a reconnecting publisher, a cluster peer, a listener), and only once classification is doing the stopping. +- **Who owns the budget?** Exactly one layer. An outer supervisor that rebuilds an inner retry loop resets its backoff to the initial delay, so the escalation never happens and a fixed-interval hammer wears an exponential costume. Watch the inner loop's terminal signal instead of restarting it. + +Resetting a backoff is its own claim: only after an outcome that says the earlier failures no longer describe reality (a session that stayed healthy, a request that succeeded, a changed destination). Resetting on an attempt that failed immediately turns escalation into a tight loop. + +Not every wait is a retry. Periodic refreshes, readiness probes, stream reads, alternate-address races, and test synchronization don't repeat a failed operation, so none of this applies to them. + ## Root Cause First - Before fixing a bug, reproduce it and explain the mechanism. A fix that adds a retry, sleep, widened timeout, defensive check, or call-site special case without a stated mechanism is a symptom patch, not a fix. diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index fe13843891..6b9b127d38 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -42,8 +42,10 @@ signal when it changes, so you can poll it via `g_object_get` or connect to `not | `estimated-recv-bitrate` | uint64 | Estimated receive bitrate in bits per second; 0 when unavailable | `status` distinguishes a transient drop (`disconnected`, the reconnect loop is still retrying) from a -permanent give-up (`failed`, a non-retryable error such as an auth rejection), which a bare -`connected` bool cannot. +permanent give-up (`failed`), which a bare `connected` bool cannot. The sink retries transport +failures for as long as the pipeline runs, so a relay outage of any length is ridden out; it goes +`failed` only on something a retry cannot clear, such as a rejected token, unusable TLS material, or +a URL no compiled-in backend can dial. ## Prerequisites diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index f1c545b058..f5a2760fc7 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -117,6 +117,14 @@ Cluster peers must authenticate to each other: See [Authentication](/bin/relay/auth) for the full setup. +A peer that is merely unreachable is redialed indefinitely, with exponential backoff and jitter so a +restarting cluster doesn't reconnect in lockstep. A peer that *rejects* us is not: a bad token, an +ALPN neither side speaks, or a URL this build can't dial produces the same failure on every dial, so +the relay logs `cluster peer rejected us` and gives up on that peer rather than hiding the cause +behind a warning every few seconds. After fixing the cause, the peer is dialed again once it leaves +and re-enters the dial set (it stops and resumes gossiping, or drops out of and back into the +`connect_api` list); a static `connect` entry needs a relay restart. + ## Migration from older configs `cluster.root` was removed. To dial cluster peers use `cluster.connect`; to advertise this relay's own address set `cluster.node` and enable `cluster.mesh`. `cluster.mesh` is now a boolean gossip toggle (it used to take this relay's URL); the URL moved to `cluster.node`. The old `mesh = ""` form still works for backwards compatibility: it enables gossip and is treated as `cluster.node`, with a deprecation warning (or an error if it conflicts with an explicit `cluster.node`). diff --git a/js/CLAUDE.md b/js/CLAUDE.md index 4fcf7aae71..edf56f32e1 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -80,6 +80,7 @@ Plain custom elements built directly on `@moq/signals`, no framework (except moq ## Conventions +- **Retries go through `@moq/net`'s `Retry`** (root Retries explains the why). `Retry.Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget); `next()` returns the delay to hand to `effect.timer`, or `undefined` once the budget is spent, and `reset()` starts a fresh sequence. Never hand-roll a fixed delay in a failure path. Classification inverts the Rust rule: the platform throws untyped errors here, so `Retry.isRetryable` treats everything as retryable except `Retry.Terminal`. Throw a `Terminal` (rather than a plain `Error`) wherever the next attempt is provably identical to the one that just failed: an ALPN this build can't speak, a certificate that won't parse, an option set that leaves no usable transport. - **Avoid callback parameters.** A function taking a `fn`/`create`/`onXxx` to invoke later reads poorly and hides control flow. Prefer returning a value the caller acts on, exposing a method or getter, or splitting into a couple of small calls the caller sequences itself (e.g. a cache `get()` then `insert(value)`, not `getOrCreate(key, () => value)`). Reserve callbacks for genuine event/subscription sinks where there is no alternative (`effect.subscribe`, DOM listeners, `Signal` subscriptions). - ESM only (`"type": "module"`). Relative imports include the `.ts`/`.tsx` extension in the lower-level packages (`net`, `signals`, `hang`); `rewriteRelativeImportExtensions` in `tsconfig.json` rewrites them to `.js` on build. Some higher-level packages (watch/publish) still omit extensions, so match the file you are editing. - Document every exported symbol and add a top-of-file `@module` doc block to each entrypoint (root convention; the published JSR/`.d.ts` docs render these). Use `@public` on the load-bearing classes. diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index ca7efdcb3b..cda901316d 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -1,6 +1,7 @@ import Session, { type Version as QmuxVersion } from "@moq/qmux"; import * as Ietf from "../ietf/index.ts"; import * as Lite from "../lite/index.ts"; +import { Terminal } from "../retry.ts"; import { Stream } from "../stream.ts"; import * as Hex from "../util/hex.ts"; import { isWebTransportSupported } from "./browser.ts"; @@ -155,7 +156,7 @@ async function connectInner(url: URL, props: ConnectProps | undefined, abort: Pr : undefined; if (!websocket && !webtransport) { - throw new Error("no transport available; WebTransport not supported and WebSocket is disabled"); + throw new Terminal("no transport available; WebTransport not supported and WebSocket is disabled"); } // Race the available transports, using `.any` to ignore if one participant has an error. @@ -220,7 +221,7 @@ async function connectTransport(url: URL, session: WebTransport, discovery: bool } else if (protocol === Lite.ALPN || protocol === "" || protocol === undefined) { setupVersion = Ietf.Version.DRAFT_14; } else { - throw new Error(`unsupported WebTransport protocol: ${protocol}`); + throw new Terminal(`unsupported WebTransport protocol: ${protocol}`); } const stream = await Stream.open(session); @@ -245,7 +246,7 @@ async function connectTransport(url: URL, session: WebTransport, discovery: bool const serverCompat = await stream.reader.u53(); if (serverCompat !== Lite.StreamId.ServerCompat) { - throw new Error(`unsupported server message type: ${serverCompat.toString()}`); + throw new Terminal(`unsupported server message type: ${serverCompat.toString()}`); } const server = await Ietf.ServerSetup.decode(stream.reader, setupVersion); @@ -270,7 +271,7 @@ async function connectTransport(url: URL, session: WebTransport, discovery: bool version: server.version as Ietf.IetfVersion, }); } else { - throw new Error(`unsupported server version: ${server.version.toString()}`); + throw new Terminal(`unsupported server version: ${server.version.toString()}`); } } @@ -305,7 +306,7 @@ type WebTransportHash = NonNullable { const match = pem.match(/-----BEGIN CERTIFICATE-----([\s\S]+?)-----END CERTIFICATE-----/); if (!match) { - throw new Error("invalid PEM certificate: missing -----BEGIN/END CERTIFICATE----- armor"); + throw new Terminal("invalid PEM certificate: missing -----BEGIN/END CERTIFICATE----- armor"); } const binary = atob(match[1].replace(/\s+/g, "")); diff --git a/js/net/src/connection/handshake.ts b/js/net/src/connection/handshake.ts index 59d822e923..3e00a0afa6 100644 --- a/js/net/src/connection/handshake.ts +++ b/js/net/src/connection/handshake.ts @@ -1,4 +1,5 @@ import * as Ietf from "../ietf/index.ts"; +import { Terminal } from "../retry.ts"; import { Reader, Stream, Writer } from "../stream.ts"; /** @@ -58,7 +59,7 @@ async function receiveSetup( const streamType = await reader.u53(); if (streamType !== Ietf.Setup.id) { - throw new Error(`unexpected stream type on setup uni: 0x${streamType.toString(16)}`); + throw new Terminal(`unexpected stream type on setup uni: 0x${streamType.toString(16)}`); } await Ietf.Setup.decode(reader, version); diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 18d7ae7d18..3855848cb8 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -118,6 +118,37 @@ test("a peer that severs immediately keeps escalating the backoff", async () => } }); +test("a failure no retry can clear stops after one attempt", async () => { + const original = globalThis.WebTransport; + const url = new URL("https://example.com/"); + let connects = 0; + + // The relay answers with an ALPN this build doesn't speak, which `connect` reports as + // `Terminal`. Redialing produces the same answer forever, so the loop has to stop. + const stub = function StubWebTransport() { + connects++; + return createMockTransportPair("moq-from-the-future").client; + }; + globalThis.WebTransport = stub as unknown as typeof WebTransport; + + // A delay far longer than the test's patience: reaching the rejection at all proves nothing + // was scheduled, and the count proves it wasn't retried. + const reload = new Reload({ + enabled: true, + url, + websocket: { enabled: false }, + delay: { initial: 60000, multiplier: 2, max: 60000 }, + }); + + try { + await expect(reload.closed).rejects.toThrow(/unsupported WebTransport protocol/); + expect(connects).toBe(1); + } finally { + reload.close(); + globalThis.WebTransport = original; + } +}); + // Polls until `pred` holds, so a regression fails the test instead of hanging it. async function waitUntil(pred: () => boolean): Promise { for (let i = 0; i < 500; i++) { diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index bf2360b29f..07bd0ce9b7 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -1,12 +1,19 @@ import { Effect, type Getter, Signal } from "@moq/signals"; import * as Announce from "../announced.ts"; +import { error } from "../error.ts"; import type * as Path from "../path.ts"; import { empty as emptyPath } from "../path.ts"; +import { Backoff, isRetryable } from "../retry.ts"; import { type ConnectProps, connect, type WebSocketOptions, type WebTransportProps } from "./connect.ts"; import type { Established } from "./established.ts"; import type { Probe, Stats } from "./stats.ts"; -/** Exponential backoff settings for {@link Reload}'s reconnect loop. */ +/** + * Exponential backoff settings for {@link Reload}'s reconnect loop. + * + * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in + * lockstep. Only failures a retry could clear are retried at all; see {@link isRetryable}. + */ export type ReloadDelay = { /** The delay in milliseconds before reconnecting (default: 1000). */ initial: DOMHighResTimeStamp; @@ -85,10 +92,9 @@ export class Reload { #closedResolve!: () => void; #closedReject!: (err: Error) => void; - #delay: DOMHighResTimeStamp; - - // Timestamp when the current retry sequence started (for timeout). - #retryStart: DOMHighResTimeStamp | undefined; + // The current retry sequence's schedule, built from `delay` when the sequence starts. Undefined + // between sequences, so a later edit to `delay` applies to the next one. + #backoff: Backoff | undefined; // Increased by 1 each time to trigger a reload. #tick = new Signal(0); @@ -107,8 +113,6 @@ export class Reload { this.websocket = props?.websocket; this.discovery = props?.discovery; - this.#delay = this.delay.initial; - this.closed = new Promise((resolve, reject) => { this.#closedResolve = resolve; this.#closedReject = reject; @@ -190,9 +194,10 @@ export class Reload { } /** - * Schedule the next connect attempt after the current backoff, or give up when the - * retry window has expired. `connected` is when the dead session was established, if - * it ever was, and `cause` the error that killed it, if it died with one. + * Schedule the next connect attempt after the current backoff, or stop when the failure isn't + * one a retry can clear and when the retry window has expired. `connected` is when the dead + * session was established, if it ever was, and `cause` the error that killed it, if it died + * with one. */ #retry(effect: Effect, connected: DOMHighResTimeStamp | undefined, cause?: unknown): void { // Any session is dead now: report disconnected during the backoff rather than @@ -200,34 +205,34 @@ export class Reload { this.established.set(undefined); this.status.set("disconnected"); + // A relay speaking a protocol this build doesn't, a certificate that won't parse, no usable + // transport at all: every attempt produces the same failure, so surface it instead of + // hiding it behind a console warning every few seconds. + if (cause !== undefined && !isRetryable(cause)) { + this.#closedReject(error(cause)); + return; + } + // A session that outlived the initial delay was healthy, so clear the backoff and // start a fresh retry window: a one-off drop should reconnect promptly. Anything // shorter is a peer that accepts and immediately severs, which has to keep // escalating or we hammer it forever at the initial delay. if (connected !== undefined && performance.now() - connected >= this.delay.initial) { - this.#delay = this.delay.initial; - this.#retryStart = undefined; + this.#backoff = undefined; } - // Track retry start for timeout. - this.#retryStart ??= performance.now(); - - const timeout = this.delay.timeout ?? 300000; - if (timeout > 0) { - const elapsed = performance.now() - this.#retryStart; - if (elapsed >= timeout) { - console.warn("reconnect timed out"); - // A graceful close has no error, so report the timeout itself. - if (cause === undefined) this.#closedReject(new Error("reconnect timed out")); - else this.#closedReject(cause instanceof Error ? cause : new Error(String(cause))); - return; - } + this.#backoff ??= new Backoff(this.delay); + + const wait = this.#backoff.delay(); + if (wait === undefined) { + console.warn("reconnect timed out"); + // A graceful close has no error, so report the timeout itself. + this.#closedReject(cause === undefined ? new Error("reconnect timed out") : error(cause)); + return; } const tick = this.#tick.peek() + 1; - effect.timer(() => this.#tick.update((prev) => Math.max(prev, tick)), this.#delay); - - this.#delay = Math.min(this.#delay * this.delay.multiplier, this.delay.max); + effect.timer(() => this.#tick.update((prev) => Math.max(prev, tick)), wait); } /** diff --git a/js/net/src/index.ts b/js/net/src/index.ts index b53d74f168..845154dd09 100644 --- a/js/net/src/index.ts +++ b/js/net/src/index.ts @@ -19,6 +19,8 @@ export { RemoteError } from "./error.ts"; export * as Group from "./group.ts"; /** Broadcast path utilities with delimiter-aware prefix matching. */ export * as Path from "./path.ts"; +/** Retry policy: which failures are worth repeating, and how long to wait before repeating them. */ +export * as Retry from "./retry.ts"; /** Branded time types (nanoseconds, microseconds, milliseconds, seconds) with conversions. */ export * as Time from "./time.ts"; /** Track role handles. */ diff --git a/js/net/src/retry.test.ts b/js/net/src/retry.test.ts new file mode 100644 index 0000000000..c4c2f79085 --- /dev/null +++ b/js/net/src/retry.test.ts @@ -0,0 +1,61 @@ +import { expect, test } from "bun:test"; +import { Backoff, isRetryable, Terminal } from "./retry.ts"; + +test("the window escalates to the cap, each delay inside its jitter band", () => { + const backoff = new Backoff({ initial: 1000, multiplier: 2, max: 8000, timeout: 0 }); + + for (const window of [1000, 2000, 4000, 8000, 8000, 8000]) { + const delay = backoff.delay(); + expect(delay).toBeGreaterThanOrEqual(window / 2); + expect(delay).toBeLessThanOrEqual(window); + } +}); + +test("jitter separates identical schedules", () => { + const props = { initial: 1000, multiplier: 2, max: 8000, timeout: 0 }; + const a = new Backoff(props); + const b = new Backoff(props); + + // One shared draw could collide by chance; a run of them colliding means no jitter at all. + const differs = Array.from({ length: 8 }).some(() => a.delay() !== b.delay()); + expect(differs).toBe(true); +}); + +test("reset returns to the initial window", () => { + const backoff = new Backoff({ initial: 1000, multiplier: 2, max: 8000, timeout: 0 }); + for (let i = 0; i < 4; i++) backoff.delay(); + + backoff.reset(); + expect(backoff.delay()).toBeLessThanOrEqual(1000); +}); + +test("a zero timeout never gives up", () => { + const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 0 }); + for (let i = 0; i < 64; i++) expect(backoff.delay()).toBeDefined(); +}); + +test("the budget is a deadline over the whole sequence", () => { + // Already expired by the time the second call reads the clock. + const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 0.0001 }); + + expect(backoff.delay()).toBeDefined(); + expect(backoff.delay()).toBeUndefined(); + + // A reset says the earlier failures no longer describe reality, so the budget refills. + backoff.reset(); + expect(backoff.delay()).toBeDefined(); +}); + +test("only a Terminal failure stops the retry", () => { + expect(isRetryable(new Terminal("unsupported WebTransport protocol: moq-99"))).toBe(false); + + // The browser hands back untyped failures, and those are overwhelmingly the network. + expect(isRetryable(new Error("connection lost"))).toBe(true); + expect(isRetryable(new DOMException("closed", "AbortError"))).toBe(true); + + // A lost transport race: worth repeating if any half of it was. + expect(isRetryable(new AggregateError([new Terminal("no WebSocket"), new Error("timed out")]))).toBe(true); + expect(isRetryable(new AggregateError([new Terminal("no WebSocket"), new Terminal("no WebTransport")]))).toBe( + false, + ); +}); diff --git a/js/net/src/retry.ts b/js/net/src/retry.ts new file mode 100644 index 0000000000..ab8b79b1d1 --- /dev/null +++ b/js/net/src/retry.ts @@ -0,0 +1,146 @@ +/** + * The retry schedule shared by every loop that re-attempts a failed operation. + * + * Two halves, kept apart on purpose. {@link isRetryable} answers *whether* an attempt is worth + * repeating; {@link Backoff} answers *when*. A loop with only the second half retries deterministic + * failures forever, which is the bug this module exists to prevent, so classify first and back off + * second. + * + * @module + */ + +/** Delay before the first retry, in milliseconds. */ +const DEFAULT_INITIAL = 1000; +/** Multiplier applied to the delay after each failure. */ +const DEFAULT_MULTIPLIER = 2; +/** Ceiling on the delay, in milliseconds. */ +const DEFAULT_MAX = 30000; +/** How long to keep retrying before giving up, in milliseconds. */ +const DEFAULT_TIMEOUT = 300000; + +/** + * How long to wait between attempts, and how long to keep making them. + * + * The defaults suit a long-lived connection: a second before the first retry, doubling to a + * half-minute ceiling, giving up after five minutes. + */ +export type BackoffProps = { + /** Delay in milliseconds before the first retry (default: 1000). */ + initial?: DOMHighResTimeStamp; + + /** Multiplier applied to the delay after each failure (default: 2). */ + multiplier?: number; + + /** Ceiling on the delay in milliseconds, however many failures have piled up (default: 30000). */ + max?: DOMHighResTimeStamp; + + /** + * How long to keep retrying before giving up, in milliseconds (default: 300000, five minutes). + * Measured from the first delay after a {@link Backoff.reset}. Zero retries forever, which only + * belongs in a supervisor whose job is to outlive an outage. + */ + timeout?: DOMHighResTimeStamp; +}; + +/** + * A failure that a retry cannot clear. + * + * Throw this instead of a plain `Error` when the next attempt is byte-for-byte the same as the one + * that just failed: a relay speaking a protocol this build doesn't, a certificate that won't parse, + * an option combination that leaves no usable transport. {@link isRetryable} reports `false` for it, + * so a reconnect loop surfaces it rather than repeating it every few seconds. + * + * @public + */ +export class Terminal extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "Terminal"; + } +} + +/** + * Whether repeating the failed operation could plausibly succeed with nothing else changing. + * + * Only {@link Terminal} is treated as settled. Unlike the Rust side, which classifies an error enum + * variant by variant, the browser hands back whatever the platform threw: a `WebTransportError`, a + * `DOMException`, an `AggregateError` wrapping a lost race, a bare `Error` from a relay. Defaulting + * those to terminal would strand a connection that a retry would have recovered, so the burden sits + * on whoever *knows* a failure is settled to say so. + */ +export function isRetryable(err: unknown): boolean { + if (err instanceof Terminal) return false; + + // `Promise.any` rejects with every transport's failure at once; the attempt is worth repeating + // if any of them was. + if (err instanceof AggregateError) return err.errors.some(isRetryable); + + return true; +} + +/** + * A capped exponential backoff with jitter and a give-up budget. + * + * Each delay is drawn from the top half of the current window (equal jitter), so a fleet that fails + * together doesn't retry together, while still waiting at least half the escalating delay. The + * window grows by {@link BackoffProps.multiplier} per failure up to {@link BackoffProps.max}, and + * {@link BackoffProps.timeout} bounds the whole sequence. + * + * Call {@link delay} after each failure and {@link reset} after a success worth trusting. Nothing + * else may own a competing schedule for the same operation: an outer supervisor that rebuilds an + * inner loop restarts its backoff at the initial delay and the escalation never happens. + * + * @public + */ +export class Backoff { + readonly #initial: DOMHighResTimeStamp; + readonly #multiplier: number; + readonly #max: DOMHighResTimeStamp; + readonly #timeout: DOMHighResTimeStamp; + + /** The current window's upper bound, grown per failure. */ + #window: DOMHighResTimeStamp; + + /** When the budget runs out, or undefined while the sequence hasn't started. */ + #deadline: DOMHighResTimeStamp | undefined; + + constructor(props?: BackoffProps) { + this.#initial = props?.initial ?? DEFAULT_INITIAL; + this.#multiplier = props?.multiplier ?? DEFAULT_MULTIPLIER; + this.#max = props?.max ?? DEFAULT_MAX; + this.#timeout = props?.timeout ?? DEFAULT_TIMEOUT; + this.#window = this.#initial; + } + + /** How long to wait before the next attempt, or undefined once the budget is spent. */ + delay(): DOMHighResTimeStamp | undefined { + if (this.#timeout > 0) { + const now = performance.now(); + if (this.#deadline === undefined) { + // The first delay of a sequence starts the clock, so a loop that ran healthy for + // hours still gets its full budget when it finally does fail. + this.#deadline = now + this.#timeout; + } else if (now >= this.#deadline) { + return undefined; + } + } + + // Equal jitter: at least half the window, never more than all of it. + const delay = this.#window / 2 + Math.random() * (this.#window / 2); + this.#window = Math.min(this.#window * this.#multiplier, this.#max); + + return delay; + } + + /** + * Start over: the next delay is {@link BackoffProps.initial} again and the budget is full. + * + * Only call this after an outcome that says the earlier failures no longer describe reality: a + * session that stayed up, a request that succeeded, a changed destination. Resetting on an + * attempt that failed immediately turns the escalation into a tight loop. + */ + reset(): void { + this.#window = this.#initial; + this.#deadline = undefined; + } +} diff --git a/js/publish/src/source/camera.ts b/js/publish/src/source/camera.ts index b6c4028de6..2c33a6b6b8 100644 --- a/js/publish/src/source/camera.ts +++ b/js/publish/src/source/camera.ts @@ -76,9 +76,10 @@ export class Camera { const constraints = effect.get(this.constraints); if (!this.#retry.begin(effect, [device, constraints])) { - // Out of budget with the same settings. Only a change to what is plugged in is new - // information worth another attempt, so watch the device list here and not while - // healthy, where a rerun would restart a working capture for unrelated device churn. + // Waiting out a backoff, or out of budget entirely, with the same settings. Either way + // a change to what is plugged in is new information worth acting on now, so watch the + // device list here and not while healthy, where a rerun would restart a working capture + // for unrelated device churn. const spent = this.device.out.available.peek(); effect.subscribe(this.device.out.available, (available) => { if (available !== spent) this.#retry.refund(); diff --git a/js/publish/src/source/microphone.ts b/js/publish/src/source/microphone.ts index 250084ff28..54506beabd 100644 --- a/js/publish/src/source/microphone.ts +++ b/js/publish/src/source/microphone.ts @@ -64,9 +64,10 @@ export class Microphone { const constraints = effect.get(this.constraints); if (!this.#retry.begin(effect, [device, constraints])) { - // Out of budget with the same settings. Only a change to what is plugged in is new - // information worth another attempt, so watch the device list here and not while - // healthy, where a rerun would restart a working capture for unrelated device churn. + // Waiting out a backoff, or out of budget entirely, with the same settings. Either way + // a change to what is plugged in is new information worth acting on now, so watch the + // device list here and not while healthy, where a rerun would restart a working capture + // for unrelated device churn. const spent = this.device.out.available.peek(); effect.subscribe(this.device.out.available, (available) => { if (available !== spent) this.#retry.refund(); diff --git a/js/publish/src/source/retry.test.ts b/js/publish/src/source/retry.test.ts index 2f7c6ae0ac..e47602dad7 100644 --- a/js/publish/src/source/retry.test.ts +++ b/js/publish/src/source/retry.test.ts @@ -43,7 +43,12 @@ class FakeMediaDevices extends EventTarget { return this.devices; } + // Every capture attempt, including the ones that never produce a track. `tracks` can't stand in: + // a missing device rejects without pushing one. + attempts = 0; + async getUserMedia(): Promise { + this.attempts += 1; if (this.missing) throw new Error("NotFoundError"); const track = new FakeTrack(); @@ -109,6 +114,42 @@ async function settle(times = 20): Promise { for (let i = 0; i < times; i++) await flush(); } +/** + * Poll until `pred` holds, so a regression fails the test instead of hanging it. + * + * A reopen waits out {@link Retry.DELAY} on a real timer, so microtask flushing alone never + * reaches it. Polling rather than sleeping for the exact delay keeps a loaded runner from + * deciding the outcome. + */ +async function waitUntil(pred: () => boolean): Promise { + const deadline = Date.now() + 10000; + while (!pred()) { + if (Date.now() > deadline) throw new Error("timed out waiting for condition"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +/** + * Poll until the capture stops reopening, meaning the budget is spent. + * + * Watches for quiet rather than an exact attempt count: a device-list change can refund the budget + * mid-cascade and buy another round. The window outlasts the largest backoff, so quiet here means + * stopped rather than mid-wait. + */ +async function waitSpent(media: FakeMediaDevices): Promise { + const quiet = (Retry.DELAY.max ?? 0) + 100; + + let seen = -1; + while (seen !== media.attempts) { + seen = media.attempts; + await new Promise((resolve) => setTimeout(resolve, quiet)); + } + await settle(); +} + +// Burning the whole budget waits out every backoff, which outlasts the default per-test timeout. +const SPENT_TIMEOUT = 30000; + /** The track a source published, or undefined. */ function published(source: { track: MediaStreamTrack } | MediaStreamTrack | undefined): unknown { if (!source) return undefined; @@ -123,10 +164,10 @@ test("a microphone re-opens when its track dies", async () => { expect(media.tracks).toHaveLength(1); media.latest().end(); + await waitUntil(() => media.tracks.length === 2); await settle(); // A second capture happened, and the live replacement is what got published. - expect(media.tracks).toHaveLength(2); expect(published(mic.out.source.peek())).toBe(media.latest()); mic.close(); @@ -140,9 +181,9 @@ test("a camera re-opens when its track dies", async () => { expect(media.tracks).toHaveLength(1); media.latest().end(); + await waitUntil(() => media.tracks.length === 2); await settle(); - expect(media.tracks).toHaveLength(2); expect(published(camera.out.source.peek())).toBe(media.latest()); camera.close(); @@ -154,12 +195,18 @@ test("running out of retries clears the source instead of publishing a corpse", const mic = new Microphone({ enabled: true }); await settle(); - // Kill every replacement the moment it arrives. - for (let i = 0; i < Retry.LIMIT + 5; i++) { + // Kill every replacement the moment it arrives; the budget allows one reopen per failure. + for (let i = 0; i < Retry.LIMIT; i++) { + const before = media.tracks.length; media.latest().end(); - await settle(); + await waitUntil(() => media.tracks.length > before); } + // The budget is spent, so this death schedules nothing at all: a settle is enough to catch a + // reopen that shouldn't happen. + media.latest().end(); + await settle(); + // The first capture plus one retry per allowed failure, and nothing after. expect(media.tracks).toHaveLength(Retry.LIMIT + 1); @@ -182,85 +229,100 @@ test("a track that arrives dead is not published", async () => { mic.close(); }); -test("a replug recovers a capture whose retries all failed", async () => { - const media = install(new FakeMediaDevices()); +test( + "a replug recovers a capture whose retries all failed", + async () => { + const media = install(new FakeMediaDevices()); - const camera = new Camera({ enabled: true }); - await settle(); - expect(media.tracks).toHaveLength(1); + const camera = new Camera({ enabled: true }); + await settle(); + expect(media.tracks).toHaveLength(1); - // Unplug the only camera: the track dies and every reopen finds nothing. - media.missing = true; - media.replug([]); - media.latest().end(); - await settle(); + // Unplug the only camera: the track dies and every reopen finds nothing. + media.missing = true; + media.replug([]); + media.latest().end(); + await waitSpent(media); - const spent = media.tracks.length; - expect(camera.out.source.peek()).toBeUndefined(); + const spent = media.tracks.length; + expect(camera.out.source.peek()).toBeUndefined(); - // Plug it back in. Without the device-list refund this stays dead forever, because an unpinned - // `requested` is undefined both before and after, so nothing else reruns the capture. - media.missing = false; - media.replug([device("cam")]); - await settle(40); + // Plug it back in. Without the device-list refund this stays dead forever, because an unpinned + // `requested` is undefined both before and after, so nothing else reruns the capture. + media.missing = false; + media.replug([device("cam")]); + await waitUntil(() => media.tracks.length > spent); + await settle(); - expect(media.tracks.length).toBeGreaterThan(spent); - expect(published(camera.out.source.peek())).toBe(media.latest()); + expect(media.tracks.length).toBeGreaterThan(spent); + expect(published(camera.out.source.peek())).toBe(media.latest()); - camera.close(); -}); + camera.close(); + }, + SPENT_TIMEOUT, +); -test("picking a different device revives a capture whose retries all failed", async () => { - const media = install(new FakeMediaDevices()); - media.devices = [device("cam"), device("cam2")]; +test( + "picking a different device revives a capture whose retries all failed", + async () => { + const media = install(new FakeMediaDevices()); + media.devices = [device("cam"), device("cam2")]; - const camera = new Camera({ enabled: true }); - await settle(); + const camera = new Camera({ enabled: true }); + await settle(); - // Burn the budget: every attempt hands back a dead track. - media.bornDead = true; - camera.device.preferred.set("cam"); - await settle(40); + // Burn the budget: every attempt hands back a dead track. + media.bornDead = true; + camera.device.preferred.set("cam"); + await waitSpent(media); - const spent = media.tracks.length; - expect(camera.out.source.peek()).toBeUndefined(); + const spent = media.tracks.length; + expect(camera.out.source.peek()).toBeUndefined(); - // Selecting another device is the user's obvious recovery, and the device list has not changed, - // so nothing else would rerun the capture. - media.bornDead = false; - camera.device.preferred.set("cam2"); - await settle(40); + // Selecting another device is the user's obvious recovery, and the device list has not changed, + // so nothing else would rerun the capture. + media.bornDead = false; + camera.device.preferred.set("cam2"); + await waitUntil(() => media.tracks.length > spent); + await settle(); - expect(media.tracks.length).toBeGreaterThan(spent); - expect(published(camera.out.source.peek())).toBe(media.latest()); + expect(media.tracks.length).toBeGreaterThan(spent); + expect(published(camera.out.source.peek())).toBe(media.latest()); - camera.close(); -}); + camera.close(); + }, + SPENT_TIMEOUT, +); -test("fixing a constraint revives a capture whose retries all failed", async () => { - const media = install(new FakeMediaDevices()); +test( + "fixing a constraint revives a capture whose retries all failed", + async () => { + const media = install(new FakeMediaDevices()); - const mic = new Microphone({ enabled: true, constraints: { channelCount: 99 } }); - await settle(); + const mic = new Microphone({ enabled: true, constraints: { channelCount: 99 } }); + await settle(); - // An impossible constraint fails instantly on every attempt, which is the quickest way to spend - // the whole budget. - media.missing = true; - mic.constraints.set({ channelCount: 98 }); - await settle(40); + // An impossible constraint fails instantly on every attempt, which is the quickest way to spend + // the whole budget. + media.missing = true; + mic.constraints.set({ channelCount: 98 }); + await waitSpent(media); - const spent = media.tracks.length; - expect(mic.out.source.peek()).toBeUndefined(); + const spent = media.tracks.length; + expect(mic.out.source.peek()).toBeUndefined(); - media.missing = false; - mic.constraints.set({ channelCount: 1 }); - await settle(40); + media.missing = false; + mic.constraints.set({ channelCount: 1 }); + await waitUntil(() => media.tracks.length > spent); + await settle(); - expect(media.tracks.length).toBeGreaterThan(spent); - expect(published(mic.out.source.peek())).toBe(media.latest()); + expect(media.tracks.length).toBeGreaterThan(spent); + expect(published(mic.out.source.peek())).toBe(media.latest()); - mic.close(); -}); + mic.close(); + }, + SPENT_TIMEOUT, +); test("unrelated device churn does not disturb a healthy capture", async () => { const media = install(new FakeMediaDevices()); diff --git a/js/publish/src/source/retry.ts b/js/publish/src/source/retry.ts index fba3a5df67..234c83cd80 100644 --- a/js/publish/src/source/retry.ts +++ b/js/publish/src/source/retry.ts @@ -1,3 +1,4 @@ +import { Retry as NetRetry } from "@moq/net"; import { type Effect, Signal } from "@moq/signals"; /** @@ -18,17 +19,32 @@ export class Retry { /** How long a track has to survive before earlier failures stop counting against it. */ static readonly SETTLED = 5000; + /** + * Escalating pause between attempts, so a device mid-re-enumerate gets a moment rather than + * being asked three more times in the same millisecond. + * + * No give-up deadline: {@link LIMIT} is this budget, counted in attempts. Counting time instead + * would make the outcome depend on how long the OS takes to say no. + */ + static readonly DELAY: NetRetry.BackoffProps = { initial: 250, multiplier: 2, max: 1000, timeout: 0 }; + readonly #rerun = new Signal(0); // Deliberately plain fields: effect reruns must not unwind them, or the budget never runs out. #failures = 0; #settings: unknown[] | undefined; + #backoff = new NetRetry.Backoff(Retry.DELAY); + + // How long the next attempt still owes the backoff, set by `failed` and paid by `begin`. + #wait: DOMHighResTimeStamp | undefined; /** - * Subscribe the capture effect and report whether an attempt is still worth making. + * Subscribe the capture effect and report whether an attempt is worth making right now. * - * False means the budget is spent: return from the run without capturing, and the previous run's - * cleanup clears whatever it published. + * False means don't capture on this run: either the budget is spent, or the backoff from the + * last failure hasn't elapsed and a rerun is already scheduled for when it has. Return from the + * run either way, and the previous run's cleanup clears whatever it published. Splitting it like + * this is what lets a dead track be dropped immediately while the *reopen* still waits. * * The budget belongs to `settings`, the caller's live capture settings. Changing any of them is * new intent rather than another go at the same thing, so it starts a fresh budget: picking a @@ -39,22 +55,33 @@ export class Retry { if (settings.some((setting, i) => setting !== this.#settings?.[i])) { this.#settings = settings; - this.#failures = 0; + this.#clear(); + } + + if (this.#failures > Retry.LIMIT) return false; + + const wait = this.#wait; + if (wait !== undefined) { + this.#wait = undefined; + effect.timer(() => this.#rerun.update((rerun) => rerun + 1), wait); + return false; } - return this.#failures <= Retry.LIMIT; + return true; } /** The attempt produced no usable track. Spends budget and reruns the effect. */ failed(): void { this.#failures += 1; + // Unlimited budget, so there is always a next delay. + this.#wait = this.#backoff.delay(); this.#rerun.update((rerun) => rerun + 1); } /** The attempt produced a live track. Reruns the effect if it dies. */ succeeded(effect: Effect, track: MediaStreamTrack): void { effect.timer(() => { - this.#failures = 0; + this.#clear(); }, Retry.SETTLED); effect.event(track, "ended", () => this.failed()); @@ -62,9 +89,16 @@ export class Retry { /** Refund the budget, because something changed that makes another attempt worth trying. */ refund(): void { - if (this.#failures === 0) return; + if (this.#failures === 0 && this.#wait === undefined) return; - this.#failures = 0; + this.#clear(); this.#rerun.update((rerun) => rerun + 1); } + + /** Forget the failures so far, along with the pause they earned. */ + #clear(): void { + this.#failures = 0; + this.#wait = undefined; + this.#backoff.reset(); + } } diff --git a/nix/modules/moq-relay.nix b/nix/modules/moq-relay.nix index 4abe8351a2..341a9eb9dd 100644 --- a/nix/modules/moq-relay.nix +++ b/nix/modules/moq-relay.nix @@ -201,8 +201,15 @@ in ExecStart = "${cfg.package}/bin/moq-relay"; + # Escalating restart backoff: 5s after the first failure, stepping up to a minute. A relay + # that dies on a bad config or an unbindable port would otherwise restart every 5s forever, + # which buries the real error under a restart loop and hammers whatever it dials. + # RestartSteps/RestartMaxDelaySec need systemd 254+; older versions log an unknown-directive + # warning and keep the fixed RestartSec. Restart = "on-failure"; RestartSec = "5s"; + RestartSteps = 5; + RestartMaxDelaySec = "1min"; # Security hardening NoNewPrivileges = true; diff --git a/packaging/moq-relay/moq-relay.service b/packaging/moq-relay/moq-relay.service index cb383948cd..9ea9d34621 100644 --- a/packaging/moq-relay/moq-relay.service +++ b/packaging/moq-relay/moq-relay.service @@ -7,8 +7,15 @@ Wants=network-online.target [Service] Type=notify ExecStart=/usr/bin/moq-relay --file /etc/moq-relay/relay.toml +# Escalating restart backoff: 5s after the first failure, stepping up to a minute. A relay that +# dies on a bad config or an unbindable port would otherwise restart every 5s forever, which buries +# the real error under a restart loop and hammers whatever it dials. RestartSteps and +# RestartMaxDelaySec need systemd 254+; older versions log an unknown-directive warning and keep the +# fixed RestartSec. Restart=on-failure RestartSec=5s +RestartSteps=5 +RestartMaxDelaySec=1min LimitNOFILE=1048576 # Sandboxing. DynamicUser allocates a transient UID, StateDirectory creates diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 865c5a045c..be4400769e 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,6 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions +- **Retries go through `moq_net::retry`** (root Retries explains the why). `retry::Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `next()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `retry::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm. Pair it with the error's `is_retryable()`, and add that method to any error type a retry loop branches on, as an exhaustive `match` with no wildcard so a new variant has to be classified. `retry::io_retryable` and `retry::status_retryable` cover the two shapes (`std::io::Error`, an HTTP status) that recur across crates. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/moq-audio/src/playback/driver.rs b/rs/moq-audio/src/playback/driver.rs index 410597e1b7..5a29fe41db 100644 --- a/rs/moq-audio/src/playback/driver.rs +++ b/rs/moq-audio/src/playback/driver.rs @@ -18,12 +18,20 @@ use super::mixer::{self, Mixer}; use super::sink::{Registration, Sink}; use crate::Error; -/// Backoff bounds for reopening a device that failed. The first retry is quick -/// because the common case is a device that came right back (a USB re-enumerate, -/// a sample-rate change); the ceiling keeps a permanently gone device from -/// spinning. -const RETRY_MIN: Duration = Duration::from_millis(500); -const RETRY_MAX: Duration = Duration::from_secs(4); +/// Backoff for reopening a device that failed. The first retry is quick because the common case is +/// a device that came right back (a USB re-enumerate, a sample-rate change); the ceiling keeps a +/// permanently gone device from spinning. +/// +/// No give-up budget: the engine outlives any one device, and the user plugging a headset back in is +/// exactly the external change a retry is waiting for. Unlimited retries are also what keeps this +/// clock-free, so the driver thread can stay on [`std::time::Instant`]. +fn retry_backoff() -> moq_net::retry::Backoff { + let mut config = moq_net::retry::Config::default(); + config.initial = Duration::from_millis(500); + config.max = Duration::from_secs(4); + config.timeout = Duration::ZERO; + moq_net::retry::Backoff::new(config) +} /// Problems tolerated in [`ERROR_WINDOW`] before the stream is rebuilt. /// @@ -355,7 +363,7 @@ pub(super) fn run( stream: None, retired: None, generation: 0, - retry: RETRY_MIN, + retry: retry_backoff(), retry_at: None, underruns: 0, unclassified: 0, @@ -417,7 +425,8 @@ struct Driver { /// Bumped on every stream, so an error from a retired one can be told apart /// from one the live stream raised. generation: u64, - retry: Duration, + /// Escalating delay before reopening a device that would not start. + retry: moq_net::retry::Backoff, /// When a failed start may be retried, and what the command wait times out /// against. `None` while the stream is healthy. retry_at: Option, @@ -475,7 +484,7 @@ impl Driver { // Replaces the previous receiver, dropping anything the old stream // retired and never got drained. self.retired = Some(retired_rx); - self.retry = RETRY_MIN; + self.retry.reset(); tracing::info!(rate, channels, ?format, "opened audio output"); Ok(()) @@ -592,11 +601,11 @@ impl Driver { tracing::warn!("audio output is not keeping up with sink changes"); } - /// When the next restart may be attempted, doubling the backoff. + /// When the next restart may be attempted, escalating the backoff. + /// + /// The backoff never gives up, so this always yields a deadline. fn schedule(&mut self) -> Instant { - let at = Instant::now() + self.retry; - self.retry = (self.retry * 2).min(RETRY_MAX); - at + Instant::now() + self.retry.delay().expect("unlimited retry budget") } /// Whether a failure reported by stream `generation` should rebuild the @@ -798,7 +807,7 @@ mod tests { stream: None, retired: None, generation: 7, - retry: RETRY_MIN, + retry: retry_backoff(), retry_at: None, underruns: 0, unclassified: 0, diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index fc2170bb20..73e75f82f0 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -136,10 +136,12 @@ impl Session { let errored = Arc::new(AtomicBool::new(false)); // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. - // `timeout = 0` retries transport/connection failures indefinitely so an unattended publisher - // outlives relay/QUIC outages; non-retryable errors (e.g. auth) stay terminal. During an outage - // the pad threads keep writing (bounded by moq-net's per-group eviction) and the relay catches up - // from a group boundary on reconnect. A bounded policy is available via `ClientConfig::backoff`. + // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC + // outages of any length. Safe to leave unbounded because the loop only retries what a retry + // can fix (`moq_native::Error::is_retryable`): a rejected token, unusable TLS material, or a + // URL no backend can dial still ends it, posting the bus error below. During an outage the pad + // threads keep writing (bounded by moq-net's per-group eviction) and the relay catches up from + // a group boundary on reconnect. A bounded policy is available via `ClientConfig::backoff`. let mut config = moq_native::ClientConfig::default(); config.tls.disable_verify = Some(settings.tls_disable_verify); config.backoff.timeout = std::time::Duration::ZERO; diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index efa4b09532..9d0db20f83 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -124,6 +124,44 @@ pub enum Error { Other(std::sync::Arc), } +impl Error { + /// Whether repeating the failed operation could plausibly succeed with nothing else changing. + /// See [`moq_net::Error::is_retryable`]. + /// + /// The gateway sits between an HTTP origin and a MoQ relay, so both halves can be transient. A + /// playlist that didn't parse, a segment whose byte range didn't add up, or a URL that isn't one + /// will fail identically on the next pass: those end the import instead of looping on it. + pub fn is_retryable(&self) -> bool { + match self { + Self::Moq(err) => err.is_retryable(), + Self::Mux(err) => err.is_retryable(), + Self::Io(err) => moq_net::retry::io_retryable(err), + // No response at all is the network; a response that arrived is the origin's answer. + Self::Reqwest(err) => err + .status() + .is_none_or(|status| moq_net::retry::status_retryable(status.as_u16())), + + // The playlist, its URLs, or the segments it points at are malformed. + Self::InvalidPlaylistUrl + | Self::InvalidFilePath + | Self::InvalidFileUrl + | Self::UrlParse(_) + | Self::ParsePlaylist(_) + | Self::NoVariants + | Self::MissingMap + | Self::EmptySegmentUri + | Self::MissingByteRangeOffset { .. } + | Self::InvalidByteRange { .. } + | Self::ByteRangeLengthMismatch { .. } + | Self::ByteRangeResponseMismatch { .. } + | Self::SequenceOverflow { .. } => false, + + // Untyped, so there is nothing to classify on. + Self::Other(_) => false, + } + } +} + impl From for Error { fn from(err: reqwest::Error) -> Self { Error::Reqwest(std::sync::Arc::new(err)) @@ -144,3 +182,30 @@ impl From for Error { /// Convenience alias for results from the HLS gateway. pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + /// The import loop retries on this classification, so a malformed playlist ending up on the + /// retryable side is an infinite loop that publishes nothing. + #[test] + fn only_transient_failures_are_retryable() { + assert!(Error::Moq(moq_net::Error::Transport("connection lost".to_string())).is_retryable()); + assert!(Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionReset)).is_retryable()); + + for err in [ + Error::ParsePlaylist("not a playlist".to_string()), + Error::NoVariants, + Error::MissingMap, + Error::InvalidPlaylistUrl, + Error::SequenceOverflow { + kind: SequenceKind::Media, + value: u64::MAX, + }, + Error::from(std::io::Error::from(std::io::ErrorKind::NotFound)), + ] { + assert!(!err.is_retryable(), "{err} should be terminal"); + } + } +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index e6178e8b4a..2bcd326564 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -31,8 +31,19 @@ use moq_mux::catalog::{self, CatalogFormat, Stream}; pub(crate) use playlist::render_media; pub use rendition::{Kind, Rendition}; -/// How long to wait before retrying the initial catalog subscription. -const CATALOG_RETRY: Duration = Duration::from_millis(250); +/// Backoff for the initial catalog subscription. +/// +/// The usual failure is a publisher that has announced its broadcast but not yet written its +/// catalog track, so this waits for external state rather than repeating a failed request. Unbounded +/// for that reason, but escalating: a source that stays silent for an hour must not be polled four +/// times a second for an hour. The broadcast closing is what ends the wait. +fn catalog_backoff() -> moq_net::retry::Backoff { + let mut config = moq_net::retry::Config::default(); + config.initial = Duration::from_millis(250); + config.max = Duration::from_secs(5); + config.timeout = Duration::ZERO; + moq_net::retry::Backoff::new(config) +} /// Export tuning shared across renditions. /// @@ -173,13 +184,23 @@ async fn watch_catalog( config: Config, renditions: renditions::Producer, ) { + let mut backoff = catalog_backoff(); + let mut consumer = loop { match catalog::Consumer::<()>::new(&broadcast, CatalogFormat::Hang).await { Ok(consumer) => break consumer, + // The catalog exists but this build can't read it, or the session is gone. Waiting + // changes neither, and a broadcast with no servable renditions is what an empty + // rendition set already means. + Err(err) if !err.is_retryable() => { + tracing::warn!(%err, "cannot subscribe to broadcast catalog"); + renditions.close(); + return; + } Err(err) => { tracing::warn!(%err, "failed to subscribe to broadcast catalog, retrying"); tokio::select! { - _ = tokio::time::sleep(CATALOG_RETRY) => {} + _ = backoff.sleep() => {} _ = kio::wait(|waiter| broadcast.poll_closed(waiter)) => { renditions.close(); return; diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 8aef546c66..de64a7e732 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -29,9 +29,19 @@ use crate::{Error, Result, SequenceKind}; /// Per-request timeout for the default HTTP client (playlist + segment fetches). const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); -/// Backoff before retrying after a failed import step, so a transient upstream -/// error (a 5xx, a truncated segment) doesn't tear down the whole import. -const ERROR_BACKOFF: Duration = Duration::from_secs(1); +/// Backoff for retrying a failed import step, so a transient upstream error (a 503, a dropped +/// connection) doesn't tear down the whole import. +/// +/// Bounded by the default give-up budget: an origin that has been unreachable for minutes is an +/// outage the caller should hear about, not one the import should paper over indefinitely while +/// publishing nothing. The ceiling is lower than the default, since a live playlist window is +/// measured in seconds and a longer wait would blow past it anyway. +fn error_backoff() -> moq_net::retry::Backoff { + let mut config = moq_net::retry::Config::default(); + config.initial = Duration::from_secs(1); + config.max = Duration::from_secs(10); + moq_net::retry::Backoff::new(config) +} /// How far back from the live edge to start when (re-)anchoring to a playlist window. /// @@ -589,15 +599,27 @@ impl Import { /// Run the import loop until cancelled. /// - /// A failed step (e.g. a transient playlist fetch error) is logged and - /// retried after a short backoff rather than ending the import. + /// A transient step failure (an origin 503, a dropped connection) is logged and retried with + /// escalating backoff. A failure that says the source is broken rather than briefly unavailable + /// (a playlist that doesn't parse, a segment whose byte range doesn't add up) ends the import: + /// the next pass reads the same bytes and fails the same way, so looping on it only hides the + /// cause. The import also ends once the backoff budget is spent, so a permanently unreachable + /// origin surfaces instead of being retried forever. pub async fn run(&mut self) -> Result<()> { + let mut backoff = error_backoff(); + loop { let outcome = match self.step(OnError::Warn).await { - Ok(outcome) => outcome, + Ok(outcome) => { + backoff.reset(); + outcome + } + Err(err) if !err.is_retryable() => return Err(err), Err(err) => { warn!(%err, "HLS import step failed, retrying"); - tokio::time::sleep(ERROR_BACKOFF).await; + if !backoff.sleep().await { + return Err(err); + } continue; } }; diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 9ca1979840..539cf2df32 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -139,6 +139,20 @@ impl Error { pub(crate) fn unsupported_container(container: &hang::catalog::UnknownContainer) -> Self { Self::UnsupportedContainer(container.kind().unwrap_or("").to_string()) } + + /// Whether repeating the failed operation could plausibly succeed with nothing else changing. + /// See [`moq_net::Error::is_retryable`]. + /// + /// Only the transport and local I/O can be transient here. Everything else is a property of the + /// bytes themselves: a container, codec, or catalog that failed to parse once parses to the same + /// failure every time, so a loop that retries it never converges. + pub fn is_retryable(&self) -> bool { + match self { + Self::Moq(err) => err.is_retryable(), + Self::Io(err) => moq_net::retry::io_retryable(err), + _ => false, + } + } } impl From for Error { diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index 434a85ae7b..6dc1359beb 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -145,6 +145,77 @@ impl Error { pub fn is_auth(&self) -> bool { self.connect_error().is_some_and(|err| err.is_auth()) } + + /// Whether reconnecting could plausibly succeed with nothing else changing. + /// + /// A reconnect loop should call this before every retry. Half of what can go wrong here is + /// configuration (an unbuildable TLS config, a URL no compiled-in backend can dial, a flag the + /// backend doesn't support) or credentials, and those fail identically forever: the loop has to + /// surface them instead of hiding them behind a warning every few seconds. + /// + /// Retryable is the explicit case, never the fallback. The match is exhaustive so a new variant + /// is a decision rather than an accident. + pub fn is_retryable(&self) -> bool { + match self { + // The OS refused a socket or a file. `kind` separates a refused port from a missing + // certificate, which is the difference between a retry and a typo. + Self::Io(err) => io_retryable(err), + + // The MoQ session's own classification, once the transport was up. + Self::MoqNet(err) => err.is_retryable(), + + // Every backend gave up, or the dial plus handshake outlived its deadline. Both are the + // network failing to answer. + Self::ConnectFailed | Self::ConnectTimeout(_) => true, + + // The race is retryable if either half is: one transport being permanently unusable + // (say, no WebSocket route) shouldn't retire the other. + #[cfg(feature = "websocket")] + Self::TransportRace { quic, websocket } => quic.is_retryable() || websocket.is_retryable(), + + #[cfg(feature = "quinn")] + Self::Quinn(err) => err.is_retryable(), + #[cfg(feature = "noq")] + Self::Noq(err) => err.is_retryable(), + #[cfg(feature = "quiche")] + Self::Quiche(err) => err.is_retryable(), + #[cfg(feature = "iroh")] + Self::Iroh(err) => err.is_retryable(), + #[cfg(feature = "websocket")] + Self::WebSocket(err) => err.is_retryable(), + #[cfg(feature = "tcp")] + Self::Tcp(err) => err.is_retryable(), + #[cfg(all(feature = "uds", unix))] + Self::Unix(err) => err.is_retryable(), + + // The server rejected our credentials. Retrying needs a new token, not a new attempt. + Self::Connect(_) => false, + + // Build and configuration failures: nothing about the next attempt differs. + Self::NoBackend(_) | Self::QlogUnsupported | Self::MtlsUnsupported | Self::InvalidStatusCode => false, + #[cfg(feature = "iroh")] + Self::IrohDisabled => false, + Self::Tls(_) => false, + + // Process setup, reached long before any connect. + Self::Directive(_) | Self::SetSubscriber(_) | Self::Logcat(_) => false, + + // A reconnect loop already gave up here. Retrying it is the nested-retry bug. + Self::Reconnect(_) => false, + } + } +} + +pub(crate) use moq_net::retry::io_retryable; + +/// Whether an HTTP failure is worth another attempt. +/// +/// No response at all is the network failing; a response that did arrive is the server's answer, so +/// only [`moq_net::retry::status_retryable`] statuses invite another try. +#[cfg(any(feature = "quinn", feature = "noq", feature = "quiche"))] +pub(crate) fn http_retryable(err: &reqwest::Error) -> bool { + err.status() + .is_none_or(|status| moq_net::retry::status_retryable(status.as_u16())) } // The wrapped sources aren't `Clone`, so `#[from]` can't store them behind `Arc` diff --git a/rs/moq-native/src/iroh.rs b/rs/moq-native/src/iroh.rs index 872eda047d..88dff389d3 100644 --- a/rs/moq-native/src/iroh.rs +++ b/rs/moq-native/src/iroh.rs @@ -115,6 +115,40 @@ pub enum Error { type Result = std::result::Result; +impl Error { + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + pub(crate) fn is_retryable(&self) -> bool { + match self { + // Reading or writing the secret key file. `kind` tells a transient failure from a path + // that isn't there. + Self::Io(err) => crate::error::io_retryable(err), + + // The endpoint's UDP socket, which relay discovery and hole punching sit on top of. + Self::Bind(_) => true, + + // The exchange with the peer: dial, handshake, established connection, WebTransport. + Self::Connect(_) + | Self::Connecting(_) + | Self::Alpn(_) + | Self::Connection(_) + | Self::Client(_) + | Self::Server(_) + | Self::RecvRequest(_) => true, + + // Configuration: the key, the bind address, or a URL that isn't an endpoint id. + Self::Secret(_) + | Self::BindAddr(_) + | Self::MissingHost + | Self::InvalidEndpointId(_) + | Self::InvalidUrl + | Self::Url(_) => false, + + // Negotiation produced something we can't speak, and GSO can't be turned off here. + Self::DecodeAlpn(_) | Self::UnsupportedAlpn(_) | Self::GsoUnsupported => false, + } + } +} + /// Settings for the shared iroh endpoint, used by both the client and server. #[derive(clap::Args, Clone, Debug, Default, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields, default)] diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index 8ca8e26786..cfa942fcf9 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -397,6 +397,58 @@ impl Error { _ => None, } } + + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + pub(crate) fn is_retryable(&self) -> bool { + match self { + // Local socket and endpoint setup. `kind` tells a port already in use (permanent until + // something else moves) from a transient failure to allocate one. + Self::BindSocket(err) | Self::CreateEndpoint(err) | Self::LocalAddr(err) | Self::ResolveBind(err) => { + crate::error::io_retryable(err) + } + + // DNS is a service like any other: a lookup failure, or an answer that hasn't + // propagated yet, resolves on its own. + Self::DnsLookup(_) | Self::NoDnsEntries => true, + + // The `http://` fingerprint bootstrap, which is a plain HTTP request to the relay. + Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { + crate::error::http_retryable(err) + } + + // The QUIC exchange itself: handshake, established connection, WebTransport CONNECT. + Self::Connection(_) | Self::Establish(_) | Self::Client(_) | Self::Server(_) | Self::RecvRequest(_) => true, + + // Retryable if any raced address is: one unroutable address must not retire the rest. + Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), + + // noq refused before a packet left the machine, so the next attempt is identical. + Self::Connect(_) => false, + + // The server's settled answer on our credentials. + Self::ConnectRejected(_) => false, + + // Configuration: the URL, the QUIC-LB sizing, the TLS material, or a missing runtime. + Self::NoRuntime + | Self::InvalidDnsName + | Self::InvalidFingerprint(_) + | Self::InvalidScheme + | Self::UnsupportedScheme(_) + | Self::QuicLbNonceTooSmall + | Self::QuicLbCidTooLong(_) + | Self::ClientVerifier(_) + | Self::NoInitialCipherSuite(_) + | Self::Tls(_) => false, + + // Negotiation produced something we can't speak. Both ends have to change first. + Self::MissingHandshake + | Self::MissingAlpn + | Self::DecodeAlpn(_) + | Self::UnsupportedAlpn(_) + | Self::MissingServerName + | Self::BuildUrl(_) => false, + } + } } fn map_client_error(err: web_transport_noq::ClientError) -> Error { diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index f93e65f45f..a2fccf97d3 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -473,6 +473,59 @@ impl Error { _ => None, } } + + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + // The deprecated variants are never constructed, but an exhaustive match is what makes a new + // variant a deliberate classification rather than a silent "not retryable". + #[allow(deprecated)] + pub(crate) fn is_retryable(&self) -> bool { + match self { + // Sockets and local addresses. `kind` tells a port already in use (permanent until + // something else moves) from a transient failure to allocate one. + Self::Io(err) | Self::ResolveBind(err) | Self::Connect(err) | Self::ServerBuild(err) => { + crate::error::io_retryable(err) + } + + // DNS is a service like any other: a lookup failure, or an answer that hasn't + // propagated yet, resolves on its own. + Self::DnsLookup(_) | Self::NoDnsEntries => true, + + // The `http://` fingerprint bootstrap, which is a plain HTTP request to the relay. + Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { + crate::error::http_retryable(err) + } + + // The QUIC exchange itself: handshake, established connection, WebTransport CONNECT. + Self::Connection(_) + | Self::Establish(_) + | Self::ClientConnect(_) + | Self::AcceptRequest(_) + | Self::Accept(_) + | Self::Reject(_) => true, + + // Retryable if any raced address is: one unroutable address must not retire the rest. + Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), + + // The server's settled answer on our credentials. + Self::ConnectRejected(_) => false, + + // Configuration: the URL, the certificates, the fingerprint, or an unbound server. + Self::InvalidDnsName + | Self::InvalidFingerprint(_) + | Self::FingerprintLength(_) + | Self::InvalidScheme + | Self::NoLocalAddr + | Self::CertRequired + | Self::CertPairMismatch + | Self::Tls(_) => false, + + // Negotiation produced something we can't speak. Both ends have to change first. + Self::MissingAlpn | Self::DecodeAlpn(_) | Self::UnsupportedAlpn(_) => false, + + // Unsupported build/flag combinations, kept only so old code still compiles. + Self::FingerprintUnsupported | Self::HostNameUnsupported | Self::GsoUnsupported => false, + } + } } fn map_client_error(err: web_transport_quiche::ClientError) -> Error { diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 1ba24ba6e3..63edfac776 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -410,6 +410,63 @@ impl Error { _ => None, } } + + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + pub(crate) fn is_retryable(&self) -> bool { + match self { + // Local socket and endpoint setup. `kind` tells a port already in use (permanent until + // something else moves) from a transient failure to allocate one. + Self::BindSocket(err) | Self::CreateEndpoint(err) | Self::LocalAddr(err) | Self::ResolveBind(err) => { + crate::error::io_retryable(err) + } + + // DNS is a service like any other: a lookup failure, or an answer that hasn't + // propagated yet, resolves on its own. + Self::DnsLookup(_) | Self::NoDnsEntries => true, + + // The `http://` fingerprint bootstrap, which is a plain HTTP request to the relay. + Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { + crate::error::http_retryable(err) + } + + // The QUIC exchange itself: handshake, established connection, WebTransport CONNECT. + // Deliberately not decomposed. A rejected certificate arrives as a closed connection + // here and is retried until the give-up budget expires, which is the right call while + // certificates rotate underneath a long-lived publisher. + Self::Connection(_) | Self::Establish(_) | Self::Client(_) | Self::Server(_) | Self::RecvRequest(_) => true, + + // Retryable if any raced address is: one unroutable address must not retire the rest. + Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), + + // Quinn refused before a packet left the machine, so the next attempt is identical. + Self::Connect(_) => false, + + // The server's settled answer on our credentials. + Self::ConnectRejected(_) => false, + + // Configuration: the URL, the qlog directory, the QUIC-LB sizing, the TLS material, + // or a runtime that isn't there. + Self::CreateQlog(_) + | Self::NoRuntime + | Self::InvalidDnsName + | Self::InvalidFingerprint(_) + | Self::InvalidScheme + | Self::UnsupportedScheme(_) + | Self::QuicLbNonceTooSmall + | Self::QuicLbCidTooLong(_) + | Self::ClientVerifier(_) + | Self::NoInitialCipherSuite(_) + | Self::Tls(_) => false, + + // Negotiation produced something we can't speak. Both ends have to change first. + Self::MissingHandshake + | Self::MissingAlpn + | Self::DecodeAlpn(_) + | Self::UnsupportedAlpn(_) + | Self::MissingServerName + | Self::BuildUrl(_) => false, + } + } } fn map_client_error(err: web_transport_quinn::ClientError) -> Error { diff --git a/rs/moq-native/src/reconnect.rs b/rs/moq-native/src/reconnect.rs index a409cac753..3b6cd69dd7 100644 --- a/rs/moq-native/src/reconnect.rs +++ b/rs/moq-native/src/reconnect.rs @@ -9,6 +9,10 @@ use url::Url; use crate::{Client, Error}; /// Exponential backoff configuration for reconnection attempts. +/// +/// Only failures that could plausibly clear on their own are retried at all +/// ([`Error::is_retryable`]); this decides how long to wait between those retries and when to stop. +/// The delays carry jitter, so a fleet knocked offline together doesn't reconnect in lockstep. #[derive(Clone, Debug, clap::Args, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] @@ -65,6 +69,17 @@ impl Default for Backoff { } } +impl From<&Backoff> for moq_net::retry::Config { + fn from(backoff: &Backoff) -> Self { + let mut config = Self::default(); + config.initial = backoff.initial; + config.multiplier = backoff.multiplier; + config.max = backoff.max; + config.timeout = backoff.timeout; + config + } +} + impl Backoff { /// How long broadcasts fed by a reconnecting session should outlive a session /// drop (see [`moq_net::origin::Info::linger`]): slightly past the give-up @@ -99,7 +114,8 @@ struct State { status: Option, /// The negotiated MoQ version of the live session, or `None` when disconnected. version: Option, - /// Set when the reconnect loop permanently gives up (reconnect timeout exceeded). + /// Set when the reconnect loop permanently gives up: a failure no retry can clear, or the + /// backoff timeout expiring. error: Option, /// The currently-connected session, or `None` while reconnecting. Read by /// [`ConnectionStatsReader`] to snapshot live connection stats. @@ -125,7 +141,11 @@ impl ConnectionStatsReader { /// Handle to a background reconnect loop. /// /// Spawns a tokio task that connects, waits for session close, then reconnects with exponential -/// backoff. The read surface mirrors [`moq_net::Session`] so a caller can treat it like a session +/// backoff. This loop is the only retry owner for the connection: a caller that rebuilds it on +/// failure restarts the backoff from its initial delay, which turns the escalation back into a tight +/// loop. Watch [`closed`](Self::closed) instead. +/// +/// The read surface mirrors [`moq_net::Session`] so a caller can treat it like a session /// that transparently reconnects: [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), /// and [`recv_bandwidth`](Self::recv_bandwidth) track the live session and reset while disconnected. /// The extra toggle a plain session doesn't have is the connection lifecycle: [`connected`](Self::connected) @@ -180,20 +200,10 @@ impl Reconnect { url: Url, backoff: Backoff, ) -> crate::Result<()> { - let mut delay = backoff.initial; - let mut retry_start = tokio::time::Instant::now(); + let mut retry = moq_net::retry::Backoff::new((&backoff).into()); let mut last_error: Option = None; loop { - if !backoff.timeout.is_zero() && retry_start.elapsed() > backoff.timeout { - let timeout = backoff.timeout; - let msg = match last_error { - Some(err) => format!("reconnect timed out after {timeout:?}: {err}"), - None => format!("reconnect timed out after {timeout:?}"), - }; - return Err(Error::Reconnect(msg)); - } - tracing::info!(%url, "connecting"); match client.connect(url.clone()).await { @@ -222,8 +232,7 @@ impl Reconnect { // Stayed up past the initial backoff: a healthy session. Reset the backoff // window so a one-off drop reconnects promptly. tracing::warn!(%url, "session closed, reconnecting"); - delay = backoff.initial; - retry_start = tokio::time::Instant::now(); + retry.reset(); last_error = None; } else { // Connected then dropped almost immediately (e.g. the server accepts then @@ -232,6 +241,9 @@ impl Reconnect { // sleep below so repeated flaps escalate instead of spinning the CPU. if let Err(err) = closed { let err = Error::from(err); + if !err.is_retryable() { + return Err(err); + } tracing::warn!(%url, %err, "session severed immediately, retrying"); last_error = Some(err); } else { @@ -240,16 +252,25 @@ impl Reconnect { } } Err(err) => { - if err.is_auth() { + // Auth, TLS material, an unsupported flag, a URL no backend can dial: the next + // dial is byte-for-byte the same, so surface it instead of hiding it behind a + // warning every few seconds. + if !err.is_retryable() { return Err(err); } last_error = Some(err); } } - tracing::warn!(%url, ?delay, "reconnecting after backoff"); - tokio::time::sleep(delay).await; - delay = std::cmp::min(delay * backoff.multiplier, backoff.max); + tracing::warn!(%url, "reconnecting after backoff"); + if !retry.sleep().await { + let timeout = backoff.timeout; + let msg = match last_error { + Some(err) => format!("reconnect timed out after {timeout:?}: {err}"), + None => format!("reconnect timed out after {timeout:?}"), + }; + return Err(Error::Reconnect(msg)); + } } } @@ -315,8 +336,9 @@ impl Reconnect { /// Poll whether the reconnect loop has stopped. /// - /// `Ready(Err)` if it permanently gave up (reconnect timeout exceeded), `Ready(Ok(()))` if - /// stopped by dropping the handle, `Pending` while it's still running. + /// `Ready(Err)` if it permanently gave up (a failure no retry can clear, or the backoff timeout + /// expiring), `Ready(Ok(()))` if stopped by dropping the handle, `Pending` while it's still + /// running. pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { ready!(self.state.poll_closed(waiter)); Poll::Ready(match &self.state.read().error { diff --git a/rs/moq-native/src/tcp.rs b/rs/moq-native/src/tcp.rs index 5ae9cf7194..4d59c66502 100644 --- a/rs/moq-native/src/tcp.rs +++ b/rs/moq-native/src/tcp.rs @@ -76,6 +76,28 @@ impl crate::failover::Aggregate for Error { } } +impl Error { + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + pub(crate) fn is_retryable(&self) -> bool { + match self { + // The TCP socket and DNS. `kind` tells a refused port from an unusable address. + Self::Io(err) => crate::error::io_retryable(err), + + // The qmux handshake, which is the exchange over an established socket. + Self::Connect(_) | Self::Accept(_) => true, + + // DNS answers propagate; an empty one now may not be empty in a minute. + Self::NoAddresses => true, + + // Retryable if any raced address is: one unroutable address must not retire the rest. + Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), + + // The URL is missing what `tcp://` requires, which no retry supplies. + Self::MissingHostname | Self::MissingPort => false, + } + } +} + type Result = std::result::Result; /// Dial a `tcp://host:port` URL, advertising `protocols` for in-band ALPN diff --git a/rs/moq-native/src/unix.rs b/rs/moq-native/src/unix.rs index 4a94bf658b..92938fa8d1 100644 --- a/rs/moq-native/src/unix.rs +++ b/rs/moq-native/src/unix.rs @@ -121,6 +121,23 @@ pub enum Error { type Result = std::result::Result; +impl Error { + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + pub(crate) fn is_retryable(&self) -> bool { + match self { + // The socket. A peer that hasn't created its socket yet reports `NotFound`, which is + // permanent as far as this layer knows: whoever starts it is the external change. + Self::Io(err) => crate::error::io_retryable(err), + + // The qmux handshake, which is the exchange over an established socket. + Self::Connect(_) | Self::Accept(_) => true, + + // The URL has no path, or the path is occupied by something we refuse to unlink. + Self::MissingPath | Self::NotASocket(_) => false, + } + } +} + /// Credentials of a connected Unix-socket peer. /// /// `pid` is `None` on platforms that don't report it (e.g. some macOS versions); diff --git a/rs/moq-native/src/websocket.rs b/rs/moq-native/src/websocket.rs index c1fb1d1433..1605aff7bd 100644 --- a/rs/moq-native/src/websocket.rs +++ b/rs/moq-native/src/websocket.rs @@ -223,6 +223,28 @@ impl Error { _ => None, } } + + /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. + pub(crate) fn is_retryable(&self) -> bool { + match self { + // The TCP socket. `kind` tells a refused port from a bind address this host can't use. + Self::Io(err) => crate::error::io_retryable(err), + + // A non-101 upgrade is the server's answer, so only the "ask again later" statuses are + // worth another try. Every other qmux failure is the TCP/TLS exchange. + Self::Connect(qmux::Error::Http(status)) => matches!(status, 408 | 429 | 502 | 503 | 504), + Self::Connect(_) | Self::Accept(_) | Self::WebSocketConnect(_) => true, + + // The server's settled answer on our credentials. + Self::ConnectRejected(_) => false, + + // Configuration: the fallback is switched off, or the URL can't carry WebSocket. + Self::Disabled | Self::MissingHostname | Self::UnsupportedScheme(_) => false, + + // The handshake request couldn't even be built, so there is nothing to send again. + Self::BuildRequest(_) | Self::ProtocolHeader(_) => false, + } + } } /// Listens for incoming WebSocket connections on a TCP port. diff --git a/rs/moq-native/tests/reconnect.rs b/rs/moq-native/tests/reconnect.rs new file mode 100644 index 0000000000..c041e777e2 --- /dev/null +++ b/rs/moq-native/tests/reconnect.rs @@ -0,0 +1,77 @@ +//! What the reconnect loop retries, and what it refuses to. +//! +//! Both cases dial over plain TCP (`tcp://`), which fails fast and locally: no TLS material, no +//! QUIC handshake, no server. That keeps the assertions about the *policy* rather than about how +//! long a particular backend takes to give up. + +#![cfg(feature = "tcp")] + +use std::time::Duration; + +/// A client whose reconnect loop escalates fast enough to assert on inside a test. +fn client(backoff: moq_native::Backoff) -> moq_native::Client { + let mut config = moq_native::ClientConfig::default(); + config.backoff = backoff; + config.init().expect("failed to init client") +} + +/// A failure no retry can clear must surface immediately. The initial delay is far longer than the +/// timeout below, so a single retry would blow the deadline: reaching the assertion at all is the +/// proof that exactly one attempt was made. +#[tokio::test] +async fn a_deterministic_failure_makes_one_attempt() { + let mut backoff = moq_native::Backoff::default(); + backoff.initial = Duration::from_secs(30); + + // `tcp://` has no default port, so this URL can never be dialed, however many times we try. + let url = "tcp://localhost".parse().expect("failed to parse url"); + let reconnect = client(backoff).reconnect(url); + + let err = tokio::time::timeout(Duration::from_secs(5), reconnect.closed()) + .await + .expect("reconnect loop retried a deterministic failure") + .expect_err("reconnect loop stopped without an error"); + + assert!(!err.is_retryable(), "gave up on a retryable error: {err}"); + assert!( + matches!(err, moq_native::Error::Tcp(_)), + "reported {err} instead of the failure that stopped it" + ); +} + +/// A transient failure is retried, escalating, until the budget runs out. The give-up error names +/// the underlying cause so an operator sees why rather than just "timed out". +#[tokio::test] +async fn a_transient_failure_retries_until_the_budget_runs_out() { + let mut backoff = moq_native::Backoff::default(); + backoff.initial = Duration::from_millis(20); + backoff.max = Duration::from_millis(40); + backoff.timeout = Duration::from_millis(200); + + // Nothing listens on port 1, so every attempt is refused: transient as far as this layer knows. + let url = "tcp://127.0.0.1:1".parse().expect("failed to parse url"); + let started = tokio::time::Instant::now(); + let reconnect = client(backoff).reconnect(url); + + let err = tokio::time::timeout(Duration::from_secs(10), reconnect.closed()) + .await + .expect("reconnect loop never gave up") + .expect_err("reconnect loop stopped without an error"); + + assert!( + matches!(err, moq_native::Error::Reconnect(_)), + "stopped with {err} rather than exhausting the budget" + ); + assert_ne!( + err.to_string(), + "reconnect timed out after 200ms", + "give-up error lost the underlying cause" + ); + // The budget is spent on sleeping between attempts, so reaching it takes at least most of it. + // Jitter draws each delay from the top half of its window, hence half rather than the whole. + assert!( + started.elapsed() >= Duration::from_millis(100), + "gave up after {:?} without retrying", + started.elapsed() + ); +} diff --git a/rs/moq-net/src/error.rs b/rs/moq-net/src/error.rs index f817ccd108..5d6b2a25b1 100644 --- a/rs/moq-net/src/error.rs +++ b/rs/moq-net/src/error.rs @@ -178,6 +178,61 @@ impl Error { } } + /// Whether repeating the failed operation could plausibly succeed with nothing else changing. + /// + /// True only for the failures a flaky link produces. Everything else is deterministic: a decode + /// failure, an auth rejection, or a version mismatch will fail identically on the next attempt, + /// so a loop that retries it burns the network and hides the real cause behind a warning. + /// + /// Retryable is the explicit case, never the fallback: a variant nobody has classified is + /// terminal. The match is exhaustive so adding one is a decision rather than an accident. + /// + /// This says nothing about *when* to retry. Pair it with [`retry::Backoff`](crate::retry::Backoff). + pub fn is_retryable(&self) -> bool { + match self { + // The link itself failed. A session that drops for any reason lands here (see + // [`Session::closed`](crate::Session::closed)), which is the case reconnect loops exist for. + Self::Transport(_) => true, + // A stream took too long to open or transmit, so the path was congested or black-holing. + Self::Timeout => true, + // Memory pressure dropped a group that is still inside the publisher's window, so a + // re-fetch can genuinely get it back. + Self::Evicted => true, + + // Deterministic protocol and coding failures: the same bytes fail the same way. + Self::Decode(_) + | Self::Encode(_) + | Self::BoundsExceeded(_) + | Self::Version + | Self::RequiredExtension + | Self::UnexpectedStream + | Self::UnexpectedMessage + | Self::ProtocolViolation + | Self::InvalidRole + | Self::TooManyParameters + | Self::Unsupported + | Self::UnknownAlpn(_) + | Self::WrongSize + | Self::FrameTooLarge + | Self::TimestampMismatch + | Self::Duplicate => false, + + // Authorization needs new credentials, not another attempt. + Self::Unauthorized => false, + + // Absent content. Retrying can only help once somebody publishes it, which is an + // external change the caller should wait on (an announcement) rather than poll for. + Self::NotFound | Self::Unroutable => false, + + // Lifecycle, not failure: the operation is over and there is nothing left to repeat. + Self::Cancel | Self::Closed | Self::Dropped | Self::Old | Self::Lagged => false, + + // Chosen by the application or the peer, so this layer can't say. Whoever assigned the + // code is the one that knows whether it's worth another try. + Self::App(_) | Self::Remote(_) => false, + } + } + /// Convert a transport error into an [Error], decoding stream reset codes. pub fn from_transport(err: impl web_transport_trait::Error) -> Self { match err.stream_error() { @@ -220,4 +275,29 @@ mod tests { assert_eq!(Error::App(404).to_code(), 468); assert_eq!(Error::Remote(468).to_code(), 468); } + + /// A dropped session always surfaces as `Transport`, so this is the classification a + /// reconnect loop actually depends on. + #[test] + fn transport_failures_are_retryable() { + assert!(Error::Transport("connection lost".to_string()).is_retryable()); + assert!(Error::Timeout.is_retryable()); + } + + /// The failures a retry can only repeat. Each of these was previously retried forever by at + /// least one loop in the workspace. + #[test] + fn deterministic_failures_are_not_retryable() { + for err in [ + Error::Unauthorized, + Error::Version, + Error::ProtocolViolation, + Error::Unsupported, + Error::UnknownAlpn("moqt-99".to_string()), + Error::NotFound, + Error::Cancel, + ] { + assert!(!err.is_retryable(), "{err} should be terminal"); + } + } } diff --git a/rs/moq-net/src/lib.rs b/rs/moq-net/src/lib.rs index 25cbd48991..ece3e21c9e 100644 --- a/rs/moq-net/src/lib.rs +++ b/rs/moq-net/src/lib.rs @@ -84,6 +84,7 @@ mod setup; mod util; mod version; +pub mod retry; pub mod stats; pub use client::*; diff --git a/rs/moq-net/src/retry.rs b/rs/moq-net/src/retry.rs new file mode 100644 index 0000000000..2f62adf1bd --- /dev/null +++ b/rs/moq-net/src/retry.rs @@ -0,0 +1,278 @@ +//! The retry schedule shared by every loop that re-attempts a failed operation. +//! +//! Two halves, kept apart on purpose. [`Error::is_retryable`](crate::Error::is_retryable) (and its +//! counterparts in the crates above) answers *whether* an attempt is worth repeating; [`Backoff`] +//! answers *when*. A loop that only has the second half retries deterministic failures forever, which +//! is the bug this module exists to prevent, so classify first and back off second. +//! +//! ```no_run +//! # async fn example() -> Result<(), moq_net::Error> { +//! # async fn attempt() -> Result<(), moq_net::Error> { Ok(()) } +//! let mut backoff = moq_net::retry::Backoff::default(); +//! loop { +//! match attempt().await { +//! Ok(()) => return Ok(()), +//! // Deterministic: the next attempt fails the same way, so surface it now. +//! Err(err) if !err.is_retryable() => return Err(err), +//! // Transient, but the budget is spent: stop rather than retry forever. +//! Err(err) if !backoff.sleep().await => return Err(err), +//! Err(_) => continue, +//! } +//! } +//! # } +//! ``` + +use kio::time::{Duration, Instant}; +use rand::RngExt; + +/// Whether an OS-level failure is worth another attempt. +/// +/// Configuration mistakes reach a caller as [`std::io::Error`] too: a path that doesn't exist, a +/// port another process holds, an address this host can't bind. Those repeat forever. What's left +/// (refused, unreachable, reset, timed out) is the network being the network. +pub fn io_retryable(err: &std::io::Error) -> bool { + !matches!( + err.kind(), + std::io::ErrorKind::NotFound + | std::io::ErrorKind::PermissionDenied + | std::io::ErrorKind::AddrInUse + | std::io::ErrorKind::AddrNotAvailable + | std::io::ErrorKind::InvalidInput + | std::io::ErrorKind::InvalidData + | std::io::ErrorKind::Unsupported + ) +} + +/// Whether an HTTP response status means "ask again later". +/// +/// A response that arrived is the server's answer, and only this narrow set invites another attempt: +/// request timeout, rate limit, and the gateway/overload statuses. Every other status, `404` and +/// `403` included, is settled. A request that got *no* response is a transport failure and doesn't +/// come through here. +pub fn status_retryable(status: u16) -> bool { + matches!(status, 408 | 429 | 502 | 503 | 504) +} + +/// How long to wait between attempts, and how long to keep making them. +/// +/// The defaults suit a long-lived connection: a second before the first retry, doubling to a +/// half-minute ceiling, giving up after five minutes. A one-shot request wants a much smaller +/// [`timeout`](Self::timeout); a supervisor that must never stop wants a zero one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct Config { + /// Delay before the first retry. + pub initial: Duration, + + /// Multiplier applied to the delay after each failure. + pub multiplier: u32, + + /// Ceiling on the delay, however many failures have piled up. + pub max: Duration, + + /// How long to keep retrying before giving up, measured from the first delay after a + /// [`reset`](Backoff::reset). [`Duration::ZERO`] retries forever, which only belongs in a + /// supervisor whose job is to outlive an outage. + pub timeout: Duration, +} + +impl Default for Config { + fn default() -> Self { + Self { + initial: Duration::from_secs(1), + multiplier: 2, + max: Duration::from_secs(30), + timeout: Duration::from_secs(300), + } + } +} + +/// A capped exponential backoff with jitter and a give-up budget. +/// +/// Each delay is drawn from the top half of the current window (equal jitter), so a fleet that fails +/// together doesn't retry together, while still waiting at least half the escalating delay. The +/// window doubles per failure up to [`Config::max`], and [`Config::timeout`] bounds the whole +/// sequence. +/// +/// Call [`sleep`](Self::sleep) (or [`delay`](Self::delay), if the caller owns the waiting) after each +/// failure and [`reset`](Self::reset) after a success worth trusting. Nothing else may own a competing +/// schedule for the same operation: an outer supervisor that rebuilds an inner loop restarts its +/// backoff at the initial delay and the escalation never happens. +#[derive(Debug)] +pub struct Backoff { + config: Config, + + /// The current window's upper bound, doubled per failure. + window: Duration, + + /// When the budget runs out, or `None` while the sequence hasn't started (or never expires). + deadline: Option, +} + +impl Backoff { + /// A backoff following `config`, with a full budget. + pub fn new(config: Config) -> Self { + Self { + window: config.initial, + config, + deadline: None, + } + } + + /// How long to wait before the next attempt, or `None` once the budget is spent. + /// + /// For callers that do their own waiting (a blocking thread, a poll loop with other arms). + /// Everything else wants [`sleep`](Self::sleep). + pub fn delay(&mut self) -> Option { + // An unlimited budget never reads the clock, which is what lets a blocking thread with its + // own [`std::time::Instant`] bookkeeping drive this too. + if !self.config.timeout.is_zero() { + let now = Instant::now(); + match self.deadline { + // Started already: stop once the budget is gone. + Some(deadline) if now >= deadline => return None, + Some(_) => {} + // The first delay of a sequence starts the clock. Deferred to here rather than to + // `new`/`reset` so a loop that runs healthy for hours still gets its full budget + // when it finally does fail. + None => self.deadline = now.checked_add(self.config.timeout), + } + } + + let delay = self.jitter(self.window); + self.window = self + .window + .saturating_mul(self.config.multiplier.max(1)) + .min(self.config.max); + + Some(delay) + } + + /// Wait out the next delay, returning `false` once the budget is spent. + /// + /// A `false` means stop retrying: the caller should surface the failure that got it here rather + /// than loop again. + pub async fn sleep(&mut self) -> bool { + let Some(delay) = self.delay() else { return false }; + web_async::time::sleep(delay).await; + true + } + + /// Start over: the next delay is [`Config::initial`] again and the budget is full. + /// + /// Only call this after an outcome that says the earlier failures no longer describe reality: a + /// session that stayed up, a request that succeeded, a changed destination. Resetting on an + /// attempt that failed immediately turns the escalation into a tight loop. + pub fn reset(&mut self) { + self.window = self.config.initial; + self.deadline = None; + } + + /// Draw the actual delay from the top half of `window`, so peers that failed together spread out. + fn jitter(&self, window: Duration) -> Duration { + let half = window / 2; + match half.is_zero() { + true => window, + false => half + Duration::from_nanos(rand::rng().random_range(0..half.as_nanos() as u64)), + } + } +} + +impl Default for Backoff { + fn default() -> Self { + Self::new(Config::default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> Config { + Config { + initial: Duration::from_secs(1), + multiplier: 2, + max: Duration::from_secs(8), + timeout: Duration::ZERO, + } + } + + /// The window doubles per failure and stops at the cap, and jitter keeps every draw inside the + /// top half of its window. + #[tokio::test(start_paused = true)] + async fn escalates_to_the_cap_within_the_jitter_band() { + let mut backoff = Backoff::new(config()); + + for expected in [1, 2, 4, 8, 8, 8].map(Duration::from_secs) { + let delay = backoff.delay().expect("unlimited budget"); + assert!( + delay >= expected / 2 && delay <= expected, + "{delay:?} outside the jitter band for {expected:?}" + ); + } + } + + /// Two backoffs with the same settings must not step in lockstep, or a fleet that failed + /// together retries together. + #[tokio::test(start_paused = true)] + async fn jitter_separates_identical_schedules() { + let mut a = Backoff::new(config()); + let mut b = Backoff::new(config()); + + // One shared draw could collide by chance; a run of them colliding means no jitter at all. + let differs = (0..8).any(|_| a.delay() != b.delay()); + assert!(differs, "identical backoffs produced identical delays"); + } + + #[tokio::test(start_paused = true)] + async fn reset_returns_to_the_initial_window() { + let mut backoff = Backoff::new(config()); + for _ in 0..4 { + backoff.delay(); + } + + backoff.reset(); + let delay = backoff.delay().expect("unlimited budget"); + assert!(delay <= Duration::from_secs(1), "{delay:?} did not return to initial"); + } + + /// The budget is a wall-clock deadline over the whole sequence, not a per-attempt one. + #[tokio::test(start_paused = true)] + async fn gives_up_once_the_budget_is_spent() { + let mut backoff = Backoff::new(Config { + timeout: Duration::from_secs(10), + ..config() + }); + + let mut slept = Duration::ZERO; + while let Some(delay) = backoff.delay() { + slept += delay; + tokio::time::sleep(delay).await; + assert!(slept < Duration::from_secs(60), "budget never ran out"); + } + + assert!(slept >= Duration::from_secs(10), "gave up after only {slept:?}"); + } + + /// A zero timeout is the supervisor case: keep retrying however long the outage lasts. + #[tokio::test(start_paused = true)] + async fn a_zero_timeout_never_gives_up() { + let mut backoff = Backoff::new(config()); + for _ in 0..64 { + assert!(backoff.sleep().await); + } + } + + /// The budget covers the retry sequence, so a reset after a healthy stretch buys a fresh one. + #[tokio::test(start_paused = true)] + async fn reset_refills_the_budget() { + let mut backoff = Backoff::new(Config { + timeout: Duration::from_secs(10), + ..config() + }); + + while backoff.sleep().await {} + backoff.reset(); + assert!(backoff.sleep().await, "reset did not refill the budget"); + } +} diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index d272312c02..60c5fde0e1 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -917,33 +917,35 @@ impl Cluster { url.query_pairs_mut().append_pair("jwt", &token); } - let base_backoff = tokio::time::Duration::from_secs(1); - let max_backoff = tokio::time::Duration::from_secs(300); + // A peer is supervised for the life of the relay, so there is no give-up deadline: one that + // is unreachable for an hour still has to be redialed when it comes back. What ends the loop + // is classification, not a budget. + let mut config = moq_net::retry::Config::default(); + config.max = tokio::time::Duration::from_secs(300); + config.timeout = tokio::time::Duration::ZERO; + let mut backoff = moq_net::retry::Backoff::new(config); + // Sessions shorter than this are treated as churn: we keep backing off // instead of resetting, otherwise a peer that rejects us instantly would // turn into a tight reconnect loop. let stable_threshold = tokio::time::Duration::from_secs(10); - let mut backoff = base_backoff; - loop { let started = tokio::time::Instant::now(); let result = self.run_remote_once(&url, cost).await; let elapsed = started.elapsed(); match result { - Ok(()) if elapsed >= stable_threshold => backoff = base_backoff, - Ok(()) => { - tracing::warn!(?elapsed, "cluster peer session closed cleanly but quickly; backing off"); - backoff = (backoff * 2).min(max_backoff); - } - Err(err) => { - tracing::warn!(%err, "cluster peer error; will retry"); - backoff = (backoff * 2).min(max_backoff); - } + Ok(()) if elapsed >= stable_threshold => backoff.reset(), + Ok(()) => tracing::warn!(?elapsed, "cluster peer session closed cleanly but quickly; backing off"), + // A rejected token, an ALPN neither side speaks, a URL this build can't dial: every + // redial produces the same failure, so stop and let the operator see it. The peer + // comes back when something external changes and re-announces it. + Err(err) if !peer_is_retryable(&err) => return Err(err.context("cluster peer rejected us")), + Err(err) => tracing::warn!(%err, "cluster peer error; will retry"), } - tokio::time::sleep(backoff).await; + backoff.sleep().await; } } @@ -986,6 +988,21 @@ impl Cluster { } } +/// Whether a failed peer dial is worth repeating. +/// +/// The dial and the session both report typed errors that classify themselves; anything else is an +/// internal invariant or a malformed peer entry, which no redial fixes. +fn peer_is_retryable(err: &anyhow::Error) -> bool { + if let Some(err) = err.downcast_ref::() { + return err.is_retryable(); + } + if let Some(err) = err.downcast_ref::() { + return err.is_retryable(); + } + + false +} + /// Extract and remove the `cost` query param from a peer URL. /// /// The param is dial-side configuration, not something the peer reads off the @@ -1119,6 +1136,23 @@ mod tests { use super::*; use crate::Config; + /// A dropped session is what a redial exists for; a rejected token is not. Both reach the loop + /// as `anyhow`, wrapped in the `.context` the dial adds, so the classification has to survive + /// that wrapping (it's the whole reason this helper exists rather than a `matches!`). + #[test] + fn peer_retries_only_transient_failures() { + let dropped = anyhow::Error::from(moq_net::Error::Transport("connection lost".to_string())); + assert!(peer_is_retryable(&dropped)); + + let rejected = anyhow::Error::from(moq_native::Error::from(moq_native::ConnectError::Unauthorized)) + .context("failed to connect to cluster peer"); + assert!(!peer_is_retryable(&rejected)); + + // Nothing typed to go on: an internal invariant, or a peer entry that never parsed. + let internal = anyhow::anyhow!("cluster peer dial without an attached QUIC client"); + assert!(!peer_is_retryable(&internal)); + } + /// The publish task holds only a `Weak` to its producer, so it stops when the /// last `moq_stats::Producer` clone drops. Attaching one must therefore hand /// its lifetime to the cluster: an embedder driving its own loop takes the diff --git a/rs/moq-rtmp/src/server.rs b/rs/moq-rtmp/src/server.rs index 412e77790a..1309987540 100644 --- a/rs/moq-rtmp/src/server.rs +++ b/rs/moq-rtmp/src/server.rs @@ -233,17 +233,32 @@ pub struct Server { /// In-flight handshakes; each resolves to a ready [`Request`], or `None` if /// the connection closed or errored before issuing a publish or play. pending: FuturesUnordered>>>, + + /// Escalating delay after a failed `accept`. Lives on the server rather than inside + /// [`accept`](Self::accept) so consecutive failures keep escalating across calls, and resets on + /// the next connection that does come in. + accept_backoff: moq_net::retry::Backoff, } impl Server { /// Bind an RTMP listener on `addr` (RTMP's well-known port is 1935). pub async fn bind(addr: SocketAddr) -> Result { let listener = TcpListener::bind(addr).await?; + + // The listener is supervised for the process's lifetime, so there is no give-up budget: the + // descriptor pressure or firewall rule behind a failed accept clears on its own, and the + // next connection resets the escalation. + let mut backoff = moq_net::retry::Config::default(); + backoff.initial = Duration::from_millis(100); + backoff.max = Duration::from_secs(5); + backoff.timeout = Duration::ZERO; + Ok(Self { listener, #[cfg(feature = "tls")] tls: None, pending: FuturesUnordered::new(), + accept_backoff: moq_net::retry::Backoff::new(backoff), }) } @@ -280,6 +295,8 @@ impl Server { // A new TCP connection: start its (TLS +) handshake concurrently. res = self.listener.accept(), if self.pending.len() < MAX_PENDING_REQUESTS => match res { Ok((stream, peer)) => { + // A connection got through, so whatever the last failure was has cleared. + self.accept_backoff.reset(); configure_socket(&stream, peer); #[cfg(feature = "tls")] let tls = self.tls.clone(); @@ -319,10 +336,13 @@ impl Server { } } Err(err) => { - // A failed accept must not take the listener down; back off so a - // persistent error doesn't busy-spin. + // A failed accept must not take the listener down: the usual causes + // (descriptor exhaustion, a connection the firewall dropped mid-handshake) + // are per-connection or clear on their own, and none of them is a reason to + // stop serving. Escalate the wait so a persistent one stops busy-spinning + // instead of retrying ten times a second forever. tracing::warn!(%err, "failed to accept RTMP connection; continuing"); - tokio::time::sleep(Duration::from_millis(100)).await; + self.accept_backoff.sleep().await; } }, } From 78592e953a2a23d580140ef238dfa6e9544e0164 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 17:17:10 -0700 Subject: [PATCH 02/13] fix(native): honor the CONNECT status; keep waiting for a late catalog Two findings from review. A WebTransport CONNECT the server actually answered is its settled response, but only 401/403 were being peeled into `ConnectRejected`; everything else stayed inside the generic client-error arm, which the new classification marked retryable. A wrong path (404) or an endpoint that doesn't speak WebTransport (405) therefore burned the whole reconnect budget instead of surfacing. Each backend now extracts the status once, and both consumers read it: the auth classification and `is_retryable`. The HLS exporter's catalog wait was using `is_retryable`, which reads `NotFound` as needing an external change before another attempt can help. That is true in general and wrong here: the external change is the publisher writing the catalog track, which is exactly what the loop waits for. An exporter that subscribed between the announcement and the first catalog write would give up and leave the broadcaster permanently empty. Waiting is now keyed on the failure being moq-level at all, with the broadcast closing still ending the wait. Co-Authored-By: Claude Opus 5 --- rs/moq-hls/src/export/mod.rs | 34 ++++++++++++++-- rs/moq-native/src/noq.rs | 35 ++++++++++++----- rs/moq-native/src/quiche.rs | 42 ++++++++++++-------- rs/moq-native/src/quinn.rs | 75 +++++++++++++++++++++++++++++------- 4 files changed, 144 insertions(+), 42 deletions(-) diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 2bcd326564..15ab51cdee 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -178,6 +178,20 @@ impl Drop for Broadcaster { } } +/// Whether a failed catalog subscription is worth waiting on. +/// +/// Any moq-level failure means "not yet": the publisher announced the broadcast before creating its +/// catalog track, the route is still resolving, the session blipped. Waiting is the whole point of +/// [`watch_catalog`]'s first loop, and the broadcast closing is what ends the wait. +/// +/// Deliberately *not* [`moq_net::Error::is_retryable`], which reads `NotFound` as needing an +/// external change before another attempt can help. That is true in general, but here the external +/// change is the publisher writing the track, which is precisely what this loop exists to wait for. +/// Everything else is a catalog this build cannot read, and no amount of waiting fixes that. +fn catalog_pending(err: &moq_mux::Error) -> bool { + matches!(err, moq_mux::Error::Moq(_)) +} + async fn watch_catalog( source: moq_mux::Source, broadcast: moq_net::broadcast::Consumer, @@ -189,10 +203,7 @@ async fn watch_catalog( let mut consumer = loop { match catalog::Consumer::<()>::new(&broadcast, CatalogFormat::Hang).await { Ok(consumer) => break consumer, - // The catalog exists but this build can't read it, or the session is gone. Waiting - // changes neither, and a broadcast with no servable renditions is what an empty - // rendition set already means. - Err(err) if !err.is_retryable() => { + Err(err) if !catalog_pending(&err) => { tracing::warn!(%err, "cannot subscribe to broadcast catalog"); renditions.close(); return; @@ -229,6 +240,21 @@ async fn watch_catalog( mod tests { use super::*; + /// The startup race this loop exists for: an exporter that subscribes between the announcement + /// and the publisher creating `catalog.json` sees the track as absent, and has to keep waiting. + /// Treating that as terminal leaves the broadcaster permanently empty with no error anywhere. + #[test] + fn a_missing_catalog_track_keeps_waiting() { + assert!(catalog_pending(&moq_net::Error::NotFound.into())); + assert!(catalog_pending(&moq_net::Error::Unroutable.into())); + assert!(catalog_pending( + &moq_net::Error::Transport("connection lost".to_string()).into() + )); + + // A catalog that arrived and could not be understood is not a waiting problem. + assert!(!catalog_pending(&moq_mux::Error::UnknownFormat("mystery".to_string()))); + } + fn frame(micros: u64, keyframe: bool) -> moq_mux::container::Frame { moq_mux::container::Frame { timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index cfa942fcf9..eee1aa24b8 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -416,8 +416,14 @@ impl Error { crate::error::http_retryable(err) } - // The QUIC exchange itself: handshake, established connection, WebTransport CONNECT. - Self::Connection(_) | Self::Establish(_) | Self::Client(_) | Self::Server(_) | Self::RecvRequest(_) => true, + // A CONNECT the server actually answered is its settled response unless the status says + // otherwise, so a wrong path (404) or an endpoint that doesn't speak WebTransport (405) + // surfaces now instead of burning the whole reconnect budget. + Self::Client(err) => client_status(err).is_none_or(moq_net::retry::status_retryable), + + // The rest of the QUIC exchange: handshake, established connection, and the server side + // of a CONNECT. + Self::Connection(_) | Self::Establish(_) | Self::Server(_) | Self::RecvRequest(_) => true, // Retryable if any raced address is: one unroutable address must not retire the rest. Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), @@ -460,26 +466,35 @@ fn map_client_error(err: web_transport_noq::ClientError) -> Error { } fn classify_client_error(err: &web_transport_noq::ClientError) -> Option { + client_status(err).and_then(crate::ConnectError::from_status_u16) +} + +/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at +/// all (as opposed to the connection failing underneath the request). +/// +/// Both classifications read this: [`classify_client_error`] turns an auth status into a +/// [`crate::ConnectError`], and [`Error::is_retryable`] decides whether the status invites another +/// attempt. A `404` or `405` is the server's settled answer, so retrying it just burns the reconnect +/// budget on a URL that will never work. +fn client_status(err: &web_transport_noq::ClientError) -> Option { match err { - web_transport_noq::ClientError::HttpError(err) => classify_connect_error(err), + web_transport_noq::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn classify_connect_error(err: &web_transport_noq::ConnectError) -> Option { +fn connect_status(err: &web_transport_noq::ConnectError) -> Option { match err { - web_transport_noq::ConnectError::ErrorStatus(status) => crate::ConnectError::from_status_u16(status.as_u16()), - web_transport_noq::ConnectError::ProtoError(err) => classify_proto_error(err), + web_transport_noq::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web_transport_noq::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn classify_proto_error(err: &web_transport_noq::proto::ConnectError) -> Option { +fn proto_status(err: &web_transport_noq::proto::ConnectError) -> Option { match err { web_transport_noq::proto::ConnectError::ErrorStatus(status) - | web_transport_noq::proto::ConnectError::WrongStatus(Some(status)) => { - crate::ConnectError::from_status_u16(status.as_u16()) - } + | web_transport_noq::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), _ => None, } } diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index a2fccf97d3..59d5342b28 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -495,13 +495,16 @@ impl Error { crate::error::http_retryable(err) } - // The QUIC exchange itself: handshake, established connection, WebTransport CONNECT. - Self::Connection(_) - | Self::Establish(_) - | Self::ClientConnect(_) - | Self::AcceptRequest(_) - | Self::Accept(_) - | Self::Reject(_) => true, + // A CONNECT the server actually answered is its settled response unless the status says + // otherwise, so a wrong path (404) or an endpoint that doesn't speak WebTransport (405) + // surfaces now instead of burning the whole reconnect budget. + Self::ClientConnect(err) => client_status(err).is_none_or(moq_net::retry::status_retryable), + + // The rest of the QUIC exchange: handshake, established connection, and the server side + // of a CONNECT. + Self::Connection(_) | Self::Establish(_) | Self::AcceptRequest(_) | Self::Accept(_) | Self::Reject(_) => { + true + } // Retryable if any raced address is: one unroutable address must not retire the rest. Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), @@ -537,26 +540,35 @@ fn map_client_error(err: web_transport_quiche::ClientError) -> Error { } fn classify_client_error(err: &web_transport_quiche::ClientError) -> Option { + client_status(err).and_then(crate::ConnectError::from_status_u16) +} + +/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at +/// all (as opposed to the connection failing underneath the request). +/// +/// Both classifications read this: [`classify_client_error`] turns an auth status into a +/// [`crate::ConnectError`], and [`Error::is_retryable`] decides whether the status invites another +/// attempt. A `404` or `405` is the server's settled answer, so retrying it just burns the reconnect +/// budget on a URL that will never work. +fn client_status(err: &web_transport_quiche::ClientError) -> Option { match err { - web_transport_quiche::ClientError::Connect(err) => classify_connect_error(err), + web_transport_quiche::ClientError::Connect(err) => connect_status(err), _ => None, } } -fn classify_connect_error(err: &web_transport_quiche::h3::ConnectError) -> Option { +fn connect_status(err: &web_transport_quiche::h3::ConnectError) -> Option { match err { - web_transport_quiche::h3::ConnectError::Status(status) => crate::ConnectError::from_status_u16(status.as_u16()), - web_transport_quiche::h3::ConnectError::Proto(err) => classify_proto_error(err), + web_transport_quiche::h3::ConnectError::Status(status) => Some(status.as_u16()), + web_transport_quiche::h3::ConnectError::Proto(err) => proto_status(err), _ => None, } } -fn classify_proto_error(err: &web_transport_quiche::proto::ConnectError) -> Option { +fn proto_status(err: &web_transport_quiche::proto::ConnectError) -> Option { match err { web_transport_quiche::proto::ConnectError::ErrorStatus(status) - | web_transport_quiche::proto::ConnectError::WrongStatus(Some(status)) => { - crate::ConnectError::from_status_u16(status.as_u16()) - } + | web_transport_quiche::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), _ => None, } } diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 63edfac776..6c6e191637 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -429,11 +429,16 @@ impl Error { crate::error::http_retryable(err) } - // The QUIC exchange itself: handshake, established connection, WebTransport CONNECT. - // Deliberately not decomposed. A rejected certificate arrives as a closed connection - // here and is retried until the give-up budget expires, which is the right call while - // certificates rotate underneath a long-lived publisher. - Self::Connection(_) | Self::Establish(_) | Self::Client(_) | Self::Server(_) | Self::RecvRequest(_) => true, + // A CONNECT the server actually answered is its settled response unless the status says + // otherwise, so a wrong path (404) or an endpoint that doesn't speak WebTransport (405) + // surfaces now instead of burning the whole reconnect budget. + Self::Client(err) => client_status(err).is_none_or(moq_net::retry::status_retryable), + + // The rest of the QUIC exchange: handshake, established connection, and the server side + // of a CONNECT. Deliberately not decomposed. A rejected certificate arrives as a closed + // connection here and is retried until the give-up budget expires, which is the right + // call while certificates rotate underneath a long-lived publisher. + Self::Connection(_) | Self::Establish(_) | Self::Server(_) | Self::RecvRequest(_) => true, // Retryable if any raced address is: one unroutable address must not retire the rest. Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), @@ -478,26 +483,35 @@ fn map_client_error(err: web_transport_quinn::ClientError) -> Error { } fn classify_client_error(err: &web_transport_quinn::ClientError) -> Option { + client_status(err).and_then(crate::ConnectError::from_status_u16) +} + +/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at +/// all (as opposed to the connection failing underneath the request). +/// +/// Both classifications read this: [`classify_client_error`] turns an auth status into a +/// [`crate::ConnectError`], and [`Error::is_retryable`] decides whether the status invites another +/// attempt. A `404` or `405` is the server's settled answer, so retrying it just burns the reconnect +/// budget on a URL that will never work. +fn client_status(err: &web_transport_quinn::ClientError) -> Option { match err { - web_transport_quinn::ClientError::HttpError(err) => classify_connect_error(err), + web_transport_quinn::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn classify_connect_error(err: &web_transport_quinn::ConnectError) -> Option { +fn connect_status(err: &web_transport_quinn::ConnectError) -> Option { match err { - web_transport_quinn::ConnectError::ErrorStatus(status) => crate::ConnectError::from_status_u16(status.as_u16()), - web_transport_quinn::ConnectError::ProtoError(err) => classify_proto_error(err), + web_transport_quinn::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web_transport_quinn::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn classify_proto_error(err: &web_transport_quinn::proto::ConnectError) -> Option { +fn proto_status(err: &web_transport_quinn::proto::ConnectError) -> Option { match err { web_transport_quinn::proto::ConnectError::ErrorStatus(status) - | web_transport_quinn::proto::ConnectError::WrongStatus(Some(status)) => { - crate::ConnectError::from_status_u16(status.as_u16()) - } + | web_transport_quinn::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), _ => None, } } @@ -743,6 +757,41 @@ impl quinn::ConnectionIdGenerator for ServerIdGenerator { mod tests { use super::*; + fn connect_rejected(status: u16) -> Error { + Error::Client(web_transport_quinn::ClientError::HttpError( + web_transport_quinn::ConnectError::ErrorStatus( + web_transport_quinn::http::StatusCode::from_u16(status).unwrap(), + ), + )) + } + + /// A CONNECT the relay answered is its settled response: a wrong path or an endpoint that + /// doesn't speak WebTransport must surface immediately rather than after the whole reconnect + /// budget. Only the "ask again later" statuses buy another attempt. + #[test] + fn a_rejected_connect_status_is_terminal() { + for status in [400, 404, 405, 410, 501] { + assert!( + !connect_rejected(status).is_retryable(), + "{status} should stop the reconnect loop" + ); + } + + for status in [408, 429, 502, 503, 504] { + assert!(connect_rejected(status).is_retryable(), "{status} should be retried"); + } + + // Auth is peeled off into its own variant before reaching the generic client arm. + assert_eq!( + connect_rejected(401).connect_error(), + Some(crate::ConnectError::Unauthorized) + ); + assert_eq!( + connect_rejected(403).connect_error(), + Some(crate::ConnectError::Forbidden) + ); + } + /// Build a controller from each family's factory and downcast it to the /// concrete quinn implementation it must map to. #[test] From eedf1a69be08da36e16855aa7714a94b11c5eba0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 17:24:37 -0700 Subject: [PATCH 03/13] fix: address review; stop stalling on a backoff that isn't ours - The RTMP accept backoff was slept on inline, so a handshake that completed during it waited out the whole delay. It is a deadline now, and the loop keeps polling pending handshakes through the pause. - A cluster peer session that ran healthy and then dropped never reset its backoff: `run_remote_session` reports even a clean close as an error, so the reset keyed on `Ok(())` could not fire and a peer up for hours redialed on a stale five-minute window. Key it on how long the session lasted instead. - `Backoff::jitter` narrowed the sample span to `u64`, which a `max` past ~584 years truncates to an empty range and panics on. `max` comes from a caller-supplied humantime string, so saturate. - `Reload.closed` rejects unprompted on a terminal failure, and a caller is free to never await it. Mark the rejection handled so it doesn't surface as an `unhandledrejection`, and document the new cause. - The JS backoff now clamps its multiplier to 1, matching the Rust twin; below that the window shrinks per failure into a tight loop. - `moq_mux::Error::is_retryable` is exhaustive, and the WebSocket backend reuses `status_retryable` rather than its own status list. - Docs and tests: the guides said `next()` after the rename to `delay()`; the JS budget test no longer depends on sub-millisecond clock resolution; the capture test constants are named and the constraint test says what the fake actually exercises. Co-Authored-By: Claude Opus 5 --- js/CLAUDE.md | 2 +- js/net/src/connection/reload.ts | 12 +++++++++- js/net/src/retry.test.ts | 8 ++++--- js/net/src/retry.ts | 3 ++- js/publish/src/source/retry.test.ts | 24 +++++++++++++------- rs/CLAUDE.md | 2 +- rs/moq-hls/src/export/mod.rs | 10 ++++++--- rs/moq-mux/src/error.rs | 34 +++++++++++++++++++++++++++- rs/moq-native/src/websocket.rs | 2 +- rs/moq-net/src/retry.rs | 29 +++++++++++++++++++++--- rs/moq-relay/src/cluster.rs | 11 ++++++++- rs/moq-rtmp/src/server.rs | 35 +++++++++++++++++++++++++---- 12 files changed, 144 insertions(+), 28 deletions(-) diff --git a/js/CLAUDE.md b/js/CLAUDE.md index edf56f32e1..d5cd803261 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -80,7 +80,7 @@ Plain custom elements built directly on `@moq/signals`, no framework (except moq ## Conventions -- **Retries go through `@moq/net`'s `Retry`** (root Retries explains the why). `Retry.Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget); `next()` returns the delay to hand to `effect.timer`, or `undefined` once the budget is spent, and `reset()` starts a fresh sequence. Never hand-roll a fixed delay in a failure path. Classification inverts the Rust rule: the platform throws untyped errors here, so `Retry.isRetryable` treats everything as retryable except `Retry.Terminal`. Throw a `Terminal` (rather than a plain `Error`) wherever the next attempt is provably identical to the one that just failed: an ALPN this build can't speak, a certificate that won't parse, an option set that leaves no usable transport. +- **Retries go through `@moq/net`'s `Retry`** (root Retries explains the why). `Retry.Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget); `delay()` returns the wait to hand to `effect.timer`, or `undefined` once the budget is spent, and `reset()` starts a fresh sequence. Never hand-roll a fixed delay in a failure path. Classification inverts the Rust rule: the platform throws untyped errors here, so `Retry.isRetryable` treats everything as retryable except `Retry.Terminal`. Throw a `Terminal` (rather than a plain `Error`) wherever the next attempt is provably identical to the one that just failed: an ALPN this build can't speak, a certificate that won't parse, an option set that leaves no usable transport. - **Avoid callback parameters.** A function taking a `fn`/`create`/`onXxx` to invoke later reads poorly and hides control flow. Prefer returning a value the caller acts on, exposing a method or getter, or splitting into a couple of small calls the caller sequences itself (e.g. a cache `get()` then `insert(value)`, not `getOrCreate(key, () => value)`). Reserve callbacks for genuine event/subscription sinks where there is no alternative (`effect.subscribe`, DOM listeners, `Signal` subscriptions). - ESM only (`"type": "module"`). Relative imports include the `.ts`/`.tsx` extension in the lower-level packages (`net`, `signals`, `hang`); `rewriteRelativeImportExtensions` in `tsconfig.json` rewrites them to `.js` on build. Some higher-level packages (watch/publish) still omit extensions, so match the file you are editing. - Document every exported symbol and add a top-of-file `@module` doc block to each entrypoint (root convention; the published JSR/`.d.ts` docs render these). Use `@public` on the load-bearing classes. diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 07bd0ce9b7..c18586f322 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -87,7 +87,12 @@ export class Reload { /** The reactive effect scope driving the connect loop; closed by {@link Reload.close}. */ #signals = new Effect(); - /** Resolves when the reconnect loop stops via {@link Reload.close} or the retry timeout. */ + /** + * Resolves when the reconnect loop stops via {@link Reload.close}. + * + * Rejects when the loop gives up instead: the retry window expired, or the failure was one no + * retry can clear (see {@link isRetryable}). + */ closed: Promise; #closedResolve!: () => void; #closedReject!: (err: Error) => void; @@ -118,6 +123,11 @@ export class Reload { this.#closedReject = reject; }); + // A caller is free to never await `closed`, and giving up rejects it unprompted. Marking the + // rejection handled here keeps that from surfacing as an `unhandledrejection`; a consumer + // awaiting the same promise still receives it. + this.closed.catch(() => {}); + if (typeof window !== "undefined" && typeof document !== "undefined") { this.#signals.event(window, "pagehide", () => this.#suspended.set(true)); this.#signals.event(window, "pageshow", () => this.#suspended.set(false)); diff --git a/js/net/src/retry.test.ts b/js/net/src/retry.test.ts index c4c2f79085..70adda14f0 100644 --- a/js/net/src/retry.test.ts +++ b/js/net/src/retry.test.ts @@ -34,11 +34,13 @@ test("a zero timeout never gives up", () => { for (let i = 0; i < 64; i++) expect(backoff.delay()).toBeDefined(); }); -test("the budget is a deadline over the whole sequence", () => { - // Already expired by the time the second call reads the clock. - const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 0.0001 }); +test("the budget is a deadline over the whole sequence", async () => { + const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 5 }); + // The first delay starts the clock; outliving the budget is what stops the sequence. Sleeping + // past it rather than shrinking the timeout keeps the test off `performance.now()`'s resolution. expect(backoff.delay()).toBeDefined(); + await Bun.sleep(15); expect(backoff.delay()).toBeUndefined(); // A reset says the earlier failures no longer describe reality, so the budget refills. diff --git a/js/net/src/retry.ts b/js/net/src/retry.ts index ab8b79b1d1..009221c308 100644 --- a/js/net/src/retry.ts +++ b/js/net/src/retry.ts @@ -106,7 +106,8 @@ export class Backoff { constructor(props?: BackoffProps) { this.#initial = props?.initial ?? DEFAULT_INITIAL; - this.#multiplier = props?.multiplier ?? DEFAULT_MULTIPLIER; + // Below 1 the window would shrink per failure, turning the escalation into a tight loop. + this.#multiplier = Math.max(props?.multiplier ?? DEFAULT_MULTIPLIER, 1); this.#max = props?.max ?? DEFAULT_MAX; this.#timeout = props?.timeout ?? DEFAULT_TIMEOUT; this.#window = this.#initial; diff --git a/js/publish/src/source/retry.test.ts b/js/publish/src/source/retry.test.ts index e47602dad7..4a604d98b9 100644 --- a/js/publish/src/source/retry.test.ts +++ b/js/publish/src/source/retry.test.ts @@ -122,10 +122,10 @@ async function settle(times = 20): Promise { * deciding the outcome. */ async function waitUntil(pred: () => boolean): Promise { - const deadline = Date.now() + 10000; + const deadline = Date.now() + WAIT_TIMEOUT; while (!pred()) { if (Date.now() > deadline) throw new Error("timed out waiting for condition"); - await new Promise((resolve) => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL)); } } @@ -137,7 +137,7 @@ async function waitUntil(pred: () => boolean): Promise { * stopped rather than mid-wait. */ async function waitSpent(media: FakeMediaDevices): Promise { - const quiet = (Retry.DELAY.max ?? 0) + 100; + const quiet = (Retry.DELAY.max ?? 0) + QUIET_MARGIN; let seen = -1; while (seen !== media.attempts) { @@ -147,8 +147,14 @@ async function waitSpent(media: FakeMediaDevices): Promise { await settle(); } -// Burning the whole budget waits out every backoff, which outlasts the default per-test timeout. -const SPENT_TIMEOUT = 30000; +/** How long a `waitUntil` polls before calling the condition unreachable. */ +const WAIT_TIMEOUT = 10_000; +/** How often `waitUntil` re-checks; short, since everything it watches is in-process. */ +const POLL_INTERVAL = 5; +/** Quiet time past the largest backoff that proves the capture really stopped reopening. */ +const QUIET_MARGIN = 100; +/** Burning the whole budget waits out every backoff, which outlasts the default per-test timeout. */ +const SPENT_TIMEOUT = 30_000; /** The track a source published, or undefined. */ function published(source: { track: MediaStreamTrack } | MediaStreamTrack | undefined): unknown { @@ -295,15 +301,15 @@ test( ); test( - "fixing a constraint revives a capture whose retries all failed", + "changing the constraints revives a capture whose retries all failed", async () => { const media = install(new FakeMediaDevices()); const mic = new Microphone({ enabled: true, constraints: { channelCount: 99 } }); await settle(); - // An impossible constraint fails instantly on every attempt, which is the quickest way to spend - // the whole budget. + // Spend the whole budget. The fake ignores constraints, so `missing` is what makes every + // attempt fail; the constraint edit only moves the settings the budget is keyed to. media.missing = true; mic.constraints.set({ channelCount: 98 }); await waitSpent(media); @@ -311,6 +317,8 @@ test( const spent = media.tracks.length; expect(mic.out.source.peek()).toBeUndefined(); + // Editing a constraint is new intent rather than another go at the same thing, so it buys a + // fresh budget. Nothing else here would rerun the capture: the device list never changed. media.missing = false; mic.constraints.set({ channelCount: 1 }); await waitUntil(() => media.tracks.length > spent); diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index be4400769e..658859c0b3 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,7 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions -- **Retries go through `moq_net::retry`** (root Retries explains the why). `retry::Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `next()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `retry::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm. Pair it with the error's `is_retryable()`, and add that method to any error type a retry loop branches on, as an exhaustive `match` with no wildcard so a new variant has to be classified. `retry::io_retryable` and `retry::status_retryable` cover the two shapes (`std::io::Error`, an HTTP status) that recur across crates. +- **Retries go through `moq_net::retry`** (root Retries explains the why). `retry::Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `delay()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `retry::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm. Pair it with the error's `is_retryable()`, and add that method to any error type a retry loop branches on, as an exhaustive `match` with no wildcard so a new variant has to be classified. `retry::io_retryable` and `retry::status_retryable` cover the two shapes (`std::io::Error`, an HTTP status) that recur across crates. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 15ab51cdee..39eaafa38f 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -34,9 +34,13 @@ pub use rendition::{Kind, Rendition}; /// Backoff for the initial catalog subscription. /// /// The usual failure is a publisher that has announced its broadcast but not yet written its -/// catalog track, so this waits for external state rather than repeating a failed request. Unbounded -/// for that reason, but escalating: a source that stays silent for an hour must not be polled four -/// times a second for an hour. The broadcast closing is what ends the wait. +/// catalog track, so this waits for external state rather than repeating a failed request. Escalating +/// for that reason: a source that stays silent for an hour must not be polled four times a second +/// for an hour. +/// +/// Deliberately no give-up budget. The broadcast closing is what ends the wait, and a relay-side +/// broadcast outlives its publisher's session, so any deadline here is a window in which a publisher +/// outage leaves the broadcaster permanently empty with nothing to recover it. fn catalog_backoff() -> moq_net::retry::Backoff { let mut config = moq_net::retry::Config::default(); config.initial = Duration::from_millis(250); diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 539cf2df32..27b3f1bbc8 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -150,7 +150,39 @@ impl Error { match self { Self::Moq(err) => err.is_retryable(), Self::Io(err) => moq_net::retry::io_retryable(err), - _ => false, + + // Container and catalog parsing. Exhaustive rather than a catch-all so a variant that + // is genuinely transient has to say so instead of inheriting this by accident. + Self::Hang(_) + | Self::Json(_) + | Self::Cmaf(_) + | Self::Mkv(_) + | Self::Msf(_) + | Self::Loc(_) + | Self::Mp4(_) + | Self::UnknownFormat(_) + | Self::UnsupportedContainer(_) + | Self::ReservedSection(_) + | Self::InvalidTimescale(_) => false, + + // Codec bitstream parsing. + Self::Annexb(_) + | Self::Aac(_) + | Self::Opus(_) + | Self::Flac(_) + | Self::Mp3(_) + | Self::H264(_) + | Self::H265(_) + | Self::Av1(_) + | Self::Vp8(_) + | Self::Vp9(_) + | Self::Legacy(_) => false, + + // Timing and framing that the bytes themselves determine. + Self::TimestampOverflow(_) | Self::MissingKeyframe(_) | Self::NegativeFlvPts { .. } => false, + + // A URL that isn't one, and the untyped `anyhow` catch-all: nothing to classify on. + Self::Url(_) | Self::Other(_) => false, } } } diff --git a/rs/moq-native/src/websocket.rs b/rs/moq-native/src/websocket.rs index 1605aff7bd..cdd0452272 100644 --- a/rs/moq-native/src/websocket.rs +++ b/rs/moq-native/src/websocket.rs @@ -232,7 +232,7 @@ impl Error { // A non-101 upgrade is the server's answer, so only the "ask again later" statuses are // worth another try. Every other qmux failure is the TCP/TLS exchange. - Self::Connect(qmux::Error::Http(status)) => matches!(status, 408 | 429 | 502 | 503 | 504), + Self::Connect(qmux::Error::Http(status)) => moq_net::retry::status_retryable(*status), Self::Connect(_) | Self::Accept(_) | Self::WebSocketConnect(_) => true, // The server's settled answer on our credentials. diff --git a/rs/moq-net/src/retry.rs b/rs/moq-net/src/retry.rs index 2f62adf1bd..3dbe5f936c 100644 --- a/rs/moq-net/src/retry.rs +++ b/rs/moq-net/src/retry.rs @@ -171,10 +171,16 @@ impl Backoff { /// Draw the actual delay from the top half of `window`, so peers that failed together spread out. fn jitter(&self, window: Duration) -> Duration { let half = window / 2; - match half.is_zero() { - true => window, - false => half + Duration::from_nanos(rand::rng().random_range(0..half.as_nanos() as u64)), + + // A window past ~584 years holds more nanoseconds than a `u64`, and truncating one would + // hand `random_range` an empty range to panic on. `max` is caller-configurable (a humantime + // string on the CLI), so saturate rather than trust it to be sane. + let span = u64::try_from(half.as_nanos()).unwrap_or(u64::MAX); + if span == 0 { + return window; } + + half.saturating_add(Duration::from_nanos(rand::rng().random_range(0..span))) } } @@ -224,6 +230,23 @@ mod tests { assert!(differs, "identical backoffs produced identical delays"); } + /// `max` comes from a caller-supplied humantime string, so an absurd one has to degrade rather + /// than panic: a window past ~584 years has more nanoseconds than the jitter sample can hold. + #[tokio::test(start_paused = true)] + async fn an_absurd_window_does_not_panic() { + let mut backoff = Backoff::new(Config { + initial: Duration::new(36_893_488_147, 419_103_232), + max: Duration::MAX, + ..config() + }); + + let delay = backoff.delay().expect("unlimited budget"); + assert!( + delay >= Duration::new(18_446_744_073, 709_551_616), + "{delay:?} below half" + ); + } + #[tokio::test(start_paused = true)] async fn reset_returns_to_the_initial_window() { let mut backoff = Backoff::new(config()); diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 60c5fde0e1..0c8215d7dd 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -935,8 +935,17 @@ impl Cluster { let result = self.run_remote_once(&url, cost).await; let elapsed = started.elapsed(); + // A session that lasted is a healthy peer, however it ended: clear the escalation so a + // one-off drop redials promptly. Keyed on how long it ran rather than on the outcome, + // because `run_remote_session` reports even a clean close as an error (it hands back the + // session's close reason). An outcome-keyed reset would therefore never fire, and a peer + // that had been up for hours would redial on a stale five-minute window. + if elapsed >= stable_threshold { + backoff.reset(); + } + match result { - Ok(()) if elapsed >= stable_threshold => backoff.reset(), + Ok(()) if elapsed >= stable_threshold => {} Ok(()) => tracing::warn!(?elapsed, "cluster peer session closed cleanly but quickly; backing off"), // A rejected token, an ALPN neither side speaks, a URL this build can't dial: every // redial produces the same failure, so stop and let the operator see it. The peer diff --git a/rs/moq-rtmp/src/server.rs b/rs/moq-rtmp/src/server.rs index 1309987540..7f9635938e 100644 --- a/rs/moq-rtmp/src/server.rs +++ b/rs/moq-rtmp/src/server.rs @@ -238,6 +238,11 @@ pub struct Server { /// [`accept`](Self::accept) so consecutive failures keep escalating across calls, and resets on /// the next connection that does come in. accept_backoff: moq_net::retry::Backoff, + + /// While set, `accept` stops asking the listener until this instant. In-flight handshakes keep + /// being polled meanwhile: a connection that already got through must not wait out a backoff + /// earned by a different one. + accept_retry: Option, } impl Server { @@ -259,6 +264,7 @@ impl Server { tls: None, pending: FuturesUnordered::new(), accept_backoff: moq_net::retry::Backoff::new(backoff), + accept_retry: None, }) } @@ -285,6 +291,9 @@ impl Server { /// if the listener itself stops (it currently never does). pub async fn accept(&mut self) -> Option> { loop { + // Copied out so the timer arm below doesn't borrow `self` alongside the other two. + let retry = self.accept_retry; + tokio::select! { // A handshake finished: yield its request, or skip a dead connection. Some(maybe) = self.pending.next(), if !self.pending.is_empty() => { @@ -292,8 +301,12 @@ impl Server { return Some(request); } } - // A new TCP connection: start its (TLS +) handshake concurrently. - res = self.listener.accept(), if self.pending.len() < MAX_PENDING_REQUESTS => match res { + // A failed accept's backoff elapsed: start asking the listener again. + () = sleep_until(retry), if retry.is_some() => self.accept_retry = None, + + // A new TCP connection: start its (TLS +) handshake concurrently. Paused while a + // failed accept is backing off, so a persistent error doesn't busy-spin. + res = self.listener.accept(), if retry.is_none() && self.pending.len() < MAX_PENDING_REQUESTS => match res { Ok((stream, peer)) => { // A connection got through, so whatever the last failure was has cleared. self.accept_backoff.reset(); @@ -340,9 +353,12 @@ impl Server { // (descriptor exhaustion, a connection the firewall dropped mid-handshake) // are per-connection or clear on their own, and none of them is a reason to // stop serving. Escalate the wait so a persistent one stops busy-spinning - // instead of retrying ten times a second forever. + // instead of retrying ten times a second forever. Recorded as a deadline + // rather than slept on here, so the loop keeps serving in-flight handshakes + // through the pause. tracing::warn!(%err, "failed to accept RTMP connection; continuing"); - self.accept_backoff.sleep().await; + let delay = self.accept_backoff.delay().expect("unlimited accept budget"); + self.accept_retry = Some(tokio::time::Instant::now() + delay); } }, } @@ -350,6 +366,17 @@ impl Server { } } +/// Sleep until `at`, or park forever when there is nothing to wait for. +/// +/// The `select!` arm that uses this is guarded on `at` being set; the pending branch keeps the arm +/// well-formed rather than leaving the macro with a `None` to unwrap. +async fn sleep_until(at: Option) { + match at { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending().await, + } +} + /// Tune an RTMP TCP socket (accepted by the server or dialed by the client): /// Nagle off for latency, keepalive on so a dead peer is reaped rather than /// pinning a broadcast forever. From d863172202b25ff27ce2fc2c0dd4ee04e9793e6d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 19:10:48 -0700 Subject: [PATCH 04/13] refactor: drop retryable/non-retryable error classification The retry loops no longer ask an error whether it is worth repeating. Deciding that means guessing, the guess has to stay correct as every wrapped error type evolves, and getting it wrong either strands a connection a retry would have recovered or hammers a dead one. The backoff budget already bounds the damage; classification only bought a faster path to surfacing a configuration error. Removes `is_retryable` from `moq_net::Error`, `moq_native::Error` and all seven backend error enums, `moq_mux::Error`, and `moq_hls::Error`, along with `retry::io_retryable`, the `Terminal` marker class in `@moq/net`, and the `catalog_pending` and `peer_is_retryable` helpers. What survives is an answer a peer actually gave, where the protocol defines the meaning rather than us inferring it. `retry::status_retryable` stays, and `moq_native::Error::status` / `moq_hls::Error::status` report the HTTP status a server sent so a caller can consult it: the reconnect loop still stops on a CONNECT the relay answered with a `404`, and the HLS import still stops on a `404` playlist. The reconnect loop also keeps its pre-existing `is_auth` guard. Everything else now ends on its budget: the cluster peer loop, the HLS export catalog wait, and the JS reconnect loop retry whatever they get. The schedules, jitter, budgets, and the RTMP and cluster fixes from earlier in this branch are unchanged. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 9 ++- js/CLAUDE.md | 2 +- js/net/src/connection/connect.ts | 11 ++-- js/net/src/connection/handshake.ts | 3 +- js/net/src/connection/reload.test.ts | 31 ---------- js/net/src/connection/reload.ts | 23 +++----- js/net/src/retry.test.ts | 16 +---- js/net/src/retry.ts | 44 ++------------ rs/CLAUDE.md | 2 +- rs/moq-gst/src/sink/session.rs | 16 ++--- rs/moq-hls/src/error.rs | 63 +++++--------------- rs/moq-hls/src/export/mod.rs | 41 ++----------- rs/moq-hls/src/import.rs | 20 ++++--- rs/moq-mux/src/error.rs | 46 --------------- rs/moq-native/src/error.rs | 74 ++++------------------- rs/moq-native/src/iroh.rs | 34 ----------- rs/moq-native/src/noq.rs | 69 +++++----------------- rs/moq-native/src/quiche.rs | 67 +++++---------------- rs/moq-native/src/quinn.rs | 87 +++++++--------------------- rs/moq-native/src/reconnect.rs | 37 +++++++----- rs/moq-native/src/tcp.rs | 22 ------- rs/moq-native/src/unix.rs | 17 ------ rs/moq-native/src/websocket.rs | 25 +++----- rs/moq-native/tests/reconnect.rs | 32 ++-------- rs/moq-net/src/error.rs | 80 ------------------------- rs/moq-net/src/retry.rs | 33 +++-------- rs/moq-relay/src/cluster.rs | 42 +------------- 27 files changed, 171 insertions(+), 775 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 086540aab6..47e7bf8630 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,13 +99,16 @@ The rename/removal rationale lives in the commit message and PR description, not ## Retries -Retrying is the reflex that hides bugs, so a new retry loop has to answer four questions in the code, not in the reviewer's head. +Retrying is the reflex that hides bugs, so a new retry loop has to answer three questions in the code, not in the reviewer's head. -- **What is worth retrying?** Only failures a retry could plausibly clear: a dropped connection, a timed-out request, an OS error that isn't a missing file or an occupied port, and the narrow "ask again later" HTTP set (408, 429, 502, 503, 504). Auth rejections, parse and transmux failures, unsupported features, invalid URLs, and configuration errors fail identically forever; surface them instead. Retryable is the explicit case, never the fallback, so classification lives in an exhaustive `match` on the error type and a new variant forces a decision. Rust: `moq_net::Error::is_retryable` and the same method on `moq_native`, `moq_mux`, and `moq_hls` errors. TypeScript is the deliberate exception (the browser throws untyped platform errors, and defaulting those to terminal would strand recoverable connections), so it inverts: `@moq/net`'s `Retry.Terminal` marks what is settled and everything else is retried. - **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Use the shared primitive rather than a hand-rolled `sleep`: `moq_net::retry::Backoff` in Rust, `Retry.Backoff` from `@moq/net` in TypeScript. -- **When does it stop?** A deadline or an attempt budget. Unlimited retries belong only to a supervisor whose job is to outlive an outage (a reconnecting publisher, a cluster peer, a listener), and only once classification is doing the stopping. +- **When does it stop?** A deadline or an attempt budget, and that budget is what ends the loop. Unlimited retries belong only to a supervisor whose job is to outlive an outage (a reconnecting publisher, a cluster peer, a listener), where the escalating delay is what keeps a permanently-dead target cheap. - **Who owns the budget?** Exactly one layer. An outer supervisor that rebuilds an inner retry loop resets its backoff to the initial delay, so the escalation never happens and a fixed-interval hammer wears an exponential costume. Watch the inner loop's terminal signal instead of restarting it. +**Don't classify errors as retryable.** It is tempting to give an error type an `is_retryable()` and skip the wait when the answer is no. Resist it: deciding whether a failure is permanent means guessing, the guess has to stay correct as every wrapped error type evolves, and getting it wrong either strands a connection a retry would have recovered or hammers a dead one. The budget already bounds the damage; the only thing classification buys is surfacing a config error sooner. + +The exception is an answer a peer actually gave, where the protocol defines the meaning. An HTTP status is the one we have: `moq_net::retry::status_retryable` covers `408`, `429`, `502`, `503`, and `504`, and `moq_native::Error::status` / `moq_hls::Error::status` report the status a server sent so a caller can consult it. That is reading a response, not inferring intent from a failure. + Resetting a backoff is its own claim: only after an outcome that says the earlier failures no longer describe reality (a session that stayed healthy, a request that succeeded, a changed destination). Resetting on an attempt that failed immediately turns escalation into a tight loop. Not every wait is a retry. Periodic refreshes, readiness probes, stream reads, alternate-address races, and test synchronization don't repeat a failed operation, so none of this applies to them. diff --git a/js/CLAUDE.md b/js/CLAUDE.md index d5cd803261..d5c86f1587 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -80,7 +80,7 @@ Plain custom elements built directly on `@moq/signals`, no framework (except moq ## Conventions -- **Retries go through `@moq/net`'s `Retry`** (root Retries explains the why). `Retry.Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget); `delay()` returns the wait to hand to `effect.timer`, or `undefined` once the budget is spent, and `reset()` starts a fresh sequence. Never hand-roll a fixed delay in a failure path. Classification inverts the Rust rule: the platform throws untyped errors here, so `Retry.isRetryable` treats everything as retryable except `Retry.Terminal`. Throw a `Terminal` (rather than a plain `Error`) wherever the next attempt is provably identical to the one that just failed: an ALPN this build can't speak, a certificate that won't parse, an option set that leaves no usable transport. +- **Retries go through `@moq/net`'s `Retry`** (root Retries explains the why). `Retry.Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget); `delay()` returns the wait to hand to `effect.timer`, or `undefined` once the budget is spent, and `reset()` starts a fresh sequence. Never hand-roll a fixed delay in a failure path, and don't try to classify which thrown values are worth retrying: the platform hands back `WebTransportError`, `DOMException`, `AggregateError`, and bare `Error`s interchangeably, so the budget is what stops the loop. - **Avoid callback parameters.** A function taking a `fn`/`create`/`onXxx` to invoke later reads poorly and hides control flow. Prefer returning a value the caller acts on, exposing a method or getter, or splitting into a couple of small calls the caller sequences itself (e.g. a cache `get()` then `insert(value)`, not `getOrCreate(key, () => value)`). Reserve callbacks for genuine event/subscription sinks where there is no alternative (`effect.subscribe`, DOM listeners, `Signal` subscriptions). - ESM only (`"type": "module"`). Relative imports include the `.ts`/`.tsx` extension in the lower-level packages (`net`, `signals`, `hang`); `rewriteRelativeImportExtensions` in `tsconfig.json` rewrites them to `.js` on build. Some higher-level packages (watch/publish) still omit extensions, so match the file you are editing. - Document every exported symbol and add a top-of-file `@module` doc block to each entrypoint (root convention; the published JSR/`.d.ts` docs render these). Use `@public` on the load-bearing classes. diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index cda901316d..ca7efdcb3b 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -1,7 +1,6 @@ import Session, { type Version as QmuxVersion } from "@moq/qmux"; import * as Ietf from "../ietf/index.ts"; import * as Lite from "../lite/index.ts"; -import { Terminal } from "../retry.ts"; import { Stream } from "../stream.ts"; import * as Hex from "../util/hex.ts"; import { isWebTransportSupported } from "./browser.ts"; @@ -156,7 +155,7 @@ async function connectInner(url: URL, props: ConnectProps | undefined, abort: Pr : undefined; if (!websocket && !webtransport) { - throw new Terminal("no transport available; WebTransport not supported and WebSocket is disabled"); + throw new Error("no transport available; WebTransport not supported and WebSocket is disabled"); } // Race the available transports, using `.any` to ignore if one participant has an error. @@ -221,7 +220,7 @@ async function connectTransport(url: URL, session: WebTransport, discovery: bool } else if (protocol === Lite.ALPN || protocol === "" || protocol === undefined) { setupVersion = Ietf.Version.DRAFT_14; } else { - throw new Terminal(`unsupported WebTransport protocol: ${protocol}`); + throw new Error(`unsupported WebTransport protocol: ${protocol}`); } const stream = await Stream.open(session); @@ -246,7 +245,7 @@ async function connectTransport(url: URL, session: WebTransport, discovery: bool const serverCompat = await stream.reader.u53(); if (serverCompat !== Lite.StreamId.ServerCompat) { - throw new Terminal(`unsupported server message type: ${serverCompat.toString()}`); + throw new Error(`unsupported server message type: ${serverCompat.toString()}`); } const server = await Ietf.ServerSetup.decode(stream.reader, setupVersion); @@ -271,7 +270,7 @@ async function connectTransport(url: URL, session: WebTransport, discovery: bool version: server.version as Ietf.IetfVersion, }); } else { - throw new Terminal(`unsupported server version: ${server.version.toString()}`); + throw new Error(`unsupported server version: ${server.version.toString()}`); } } @@ -306,7 +305,7 @@ type WebTransportHash = NonNullable { const match = pem.match(/-----BEGIN CERTIFICATE-----([\s\S]+?)-----END CERTIFICATE-----/); if (!match) { - throw new Terminal("invalid PEM certificate: missing -----BEGIN/END CERTIFICATE----- armor"); + throw new Error("invalid PEM certificate: missing -----BEGIN/END CERTIFICATE----- armor"); } const binary = atob(match[1].replace(/\s+/g, "")); diff --git a/js/net/src/connection/handshake.ts b/js/net/src/connection/handshake.ts index 3e00a0afa6..59d822e923 100644 --- a/js/net/src/connection/handshake.ts +++ b/js/net/src/connection/handshake.ts @@ -1,5 +1,4 @@ import * as Ietf from "../ietf/index.ts"; -import { Terminal } from "../retry.ts"; import { Reader, Stream, Writer } from "../stream.ts"; /** @@ -59,7 +58,7 @@ async function receiveSetup( const streamType = await reader.u53(); if (streamType !== Ietf.Setup.id) { - throw new Terminal(`unexpected stream type on setup uni: 0x${streamType.toString(16)}`); + throw new Error(`unexpected stream type on setup uni: 0x${streamType.toString(16)}`); } await Ietf.Setup.decode(reader, version); diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 3855848cb8..18d7ae7d18 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -118,37 +118,6 @@ test("a peer that severs immediately keeps escalating the backoff", async () => } }); -test("a failure no retry can clear stops after one attempt", async () => { - const original = globalThis.WebTransport; - const url = new URL("https://example.com/"); - let connects = 0; - - // The relay answers with an ALPN this build doesn't speak, which `connect` reports as - // `Terminal`. Redialing produces the same answer forever, so the loop has to stop. - const stub = function StubWebTransport() { - connects++; - return createMockTransportPair("moq-from-the-future").client; - }; - globalThis.WebTransport = stub as unknown as typeof WebTransport; - - // A delay far longer than the test's patience: reaching the rejection at all proves nothing - // was scheduled, and the count proves it wasn't retried. - const reload = new Reload({ - enabled: true, - url, - websocket: { enabled: false }, - delay: { initial: 60000, multiplier: 2, max: 60000 }, - }); - - try { - await expect(reload.closed).rejects.toThrow(/unsupported WebTransport protocol/); - expect(connects).toBe(1); - } finally { - reload.close(); - globalThis.WebTransport = original; - } -}); - // Polls until `pred` holds, so a regression fails the test instead of hanging it. async function waitUntil(pred: () => boolean): Promise { for (let i = 0; i < 500; i++) { diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index c18586f322..0880615498 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -3,7 +3,7 @@ import * as Announce from "../announced.ts"; import { error } from "../error.ts"; import type * as Path from "../path.ts"; import { empty as emptyPath } from "../path.ts"; -import { Backoff, isRetryable } from "../retry.ts"; +import { Backoff } from "../retry.ts"; import { type ConnectProps, connect, type WebSocketOptions, type WebTransportProps } from "./connect.ts"; import type { Established } from "./established.ts"; import type { Probe, Stats } from "./stats.ts"; @@ -12,7 +12,7 @@ import type { Probe, Stats } from "./stats.ts"; * Exponential backoff settings for {@link Reload}'s reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Only failures a retry could clear are retried at all; see {@link isRetryable}. + * lockstep. Every failure is retried; {@link ReloadDelay.timeout} is what stops the loop. */ export type ReloadDelay = { /** The delay in milliseconds before reconnecting (default: 1000). */ @@ -90,8 +90,8 @@ export class Reload { /** * Resolves when the reconnect loop stops via {@link Reload.close}. * - * Rejects when the loop gives up instead: the retry window expired, or the failure was one no - * retry can clear (see {@link isRetryable}). + * Rejects when the loop gives up instead, carrying the failure that was in flight when the + * retry window expired. */ closed: Promise; #closedResolve!: () => void; @@ -204,10 +204,9 @@ export class Reload { } /** - * Schedule the next connect attempt after the current backoff, or stop when the failure isn't - * one a retry can clear and when the retry window has expired. `connected` is when the dead - * session was established, if it ever was, and `cause` the error that killed it, if it died - * with one. + * Schedule the next connect attempt after the current backoff, or stop once the retry window + * has expired. `connected` is when the dead session was established, if it ever was, and + * `cause` the error that killed it, if it died with one. */ #retry(effect: Effect, connected: DOMHighResTimeStamp | undefined, cause?: unknown): void { // Any session is dead now: report disconnected during the backoff rather than @@ -215,14 +214,6 @@ export class Reload { this.established.set(undefined); this.status.set("disconnected"); - // A relay speaking a protocol this build doesn't, a certificate that won't parse, no usable - // transport at all: every attempt produces the same failure, so surface it instead of - // hiding it behind a console warning every few seconds. - if (cause !== undefined && !isRetryable(cause)) { - this.#closedReject(error(cause)); - return; - } - // A session that outlived the initial delay was healthy, so clear the backoff and // start a fresh retry window: a one-off drop should reconnect promptly. Anything // shorter is a peer that accepts and immediately severs, which has to keep diff --git a/js/net/src/retry.test.ts b/js/net/src/retry.test.ts index 70adda14f0..16b2253227 100644 --- a/js/net/src/retry.test.ts +++ b/js/net/src/retry.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { Backoff, isRetryable, Terminal } from "./retry.ts"; +import { Backoff } from "./retry.ts"; test("the window escalates to the cap, each delay inside its jitter band", () => { const backoff = new Backoff({ initial: 1000, multiplier: 2, max: 8000, timeout: 0 }); @@ -47,17 +47,3 @@ test("the budget is a deadline over the whole sequence", async () => { backoff.reset(); expect(backoff.delay()).toBeDefined(); }); - -test("only a Terminal failure stops the retry", () => { - expect(isRetryable(new Terminal("unsupported WebTransport protocol: moq-99"))).toBe(false); - - // The browser hands back untyped failures, and those are overwhelmingly the network. - expect(isRetryable(new Error("connection lost"))).toBe(true); - expect(isRetryable(new DOMException("closed", "AbortError"))).toBe(true); - - // A lost transport race: worth repeating if any half of it was. - expect(isRetryable(new AggregateError([new Terminal("no WebSocket"), new Error("timed out")]))).toBe(true); - expect(isRetryable(new AggregateError([new Terminal("no WebSocket"), new Terminal("no WebTransport")]))).toBe( - false, - ); -}); diff --git a/js/net/src/retry.ts b/js/net/src/retry.ts index 009221c308..7d7165fcda 100644 --- a/js/net/src/retry.ts +++ b/js/net/src/retry.ts @@ -1,10 +1,10 @@ /** * The retry schedule shared by every loop that re-attempts a failed operation. * - * Two halves, kept apart on purpose. {@link isRetryable} answers *whether* an attempt is worth - * repeating; {@link Backoff} answers *when*. A loop with only the second half retries deterministic - * failures forever, which is the bug this module exists to prevent, so classify first and back off - * second. + * {@link Backoff} answers *when* to try again, and its budget is what ends a loop. There is + * deliberately no counterpart answering *whether* a given error is worth repeating: the browser + * hands back whatever the platform threw, and guessing wrong either strands a connection a retry + * would have recovered or hammers a dead one. The budget bounds the damage instead. * * @module */ @@ -42,42 +42,6 @@ export type BackoffProps = { timeout?: DOMHighResTimeStamp; }; -/** - * A failure that a retry cannot clear. - * - * Throw this instead of a plain `Error` when the next attempt is byte-for-byte the same as the one - * that just failed: a relay speaking a protocol this build doesn't, a certificate that won't parse, - * an option combination that leaves no usable transport. {@link isRetryable} reports `false` for it, - * so a reconnect loop surfaces it rather than repeating it every few seconds. - * - * @public - */ -export class Terminal extends Error { - constructor(message: string, options?: { cause?: unknown }) { - super(message, options); - this.name = "Terminal"; - } -} - -/** - * Whether repeating the failed operation could plausibly succeed with nothing else changing. - * - * Only {@link Terminal} is treated as settled. Unlike the Rust side, which classifies an error enum - * variant by variant, the browser hands back whatever the platform threw: a `WebTransportError`, a - * `DOMException`, an `AggregateError` wrapping a lost race, a bare `Error` from a relay. Defaulting - * those to terminal would strand a connection that a retry would have recovered, so the burden sits - * on whoever *knows* a failure is settled to say so. - */ -export function isRetryable(err: unknown): boolean { - if (err instanceof Terminal) return false; - - // `Promise.any` rejects with every transport's failure at once; the attempt is worth repeating - // if any of them was. - if (err instanceof AggregateError) return err.errors.some(isRetryable); - - return true; -} - /** * A capped exponential backoff with jitter and a give-up budget. * diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 658859c0b3..ea3dee542f 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,7 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions -- **Retries go through `moq_net::retry`** (root Retries explains the why). `retry::Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `delay()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `retry::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm. Pair it with the error's `is_retryable()`, and add that method to any error type a retry loop branches on, as an exhaustive `match` with no wildcard so a new variant has to be classified. `retry::io_retryable` and `retry::status_retryable` cover the two shapes (`std::io::Error`, an HTTP status) that recur across crates. +- **Retries go through `moq_net::retry`** (root Retries explains the why). `retry::Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `delay()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `retry::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm, and don't add an `is_retryable()` to an error type: the budget is what stops a loop. The one thing worth reading off a failure is a status a peer actually sent, via `retry::status_retryable` and the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index 73e75f82f0..89289bfec6 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -29,7 +29,7 @@ pub(crate) static CAT: LazyLock = /// The publish connection's lifecycle, surfaced as the `status` property. /// /// Bundles what a bare `connected` bool can't: `Failed` (a terminal give-up) is distinct from -/// `Disconnected` (a transient drop the reconnect loop is still retrying), so a consumer watching +/// `Disconnected` (a drop the reconnect loop is still retrying), so a consumer watching /// `notify::status` learns when a connection is newly established or permanently rejected. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, glib::Enum)] #[enum_type(name = "GstMoqSinkConnectionStatus")] @@ -42,7 +42,8 @@ pub enum ConnectionStatus { /// A session is connected and publishing. #[enum_value(name = "Connected: session established", nick = "connected")] Connected, - /// The reconnect loop gave up permanently (a non-retryable error, e.g. auth rejection). Terminal. + /// The reconnect loop gave up permanently (an auth rejection, or a CONNECT status that isn't an + /// invitation to retry). Terminal. #[enum_value(name = "Failed: connection rejected, gave up", nick = "failed")] Failed, } @@ -137,11 +138,12 @@ impl Session { // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC - // outages of any length. Safe to leave unbounded because the loop only retries what a retry - // can fix (`moq_native::Error::is_retryable`): a rejected token, unusable TLS material, or a - // URL no backend can dial still ends it, posting the bus error below. During an outage the pad - // threads keep writing (bounded by moq-net's per-group eviction) and the relay catches up from - // a group boundary on reconnect. A bounded policy is available via `ClientConfig::backoff`. + // outages of any length, which is the trade this element wants: a pipeline nobody is watching + // should still be publishing when the relay comes back. The loop still ends on the two answers + // a server states outright (an auth rejection, or a CONNECT status that isn't an invitation to + // retry), posting the bus error below. During an outage the pad threads keep writing (bounded + // by moq-net's per-group eviction) and the relay catches up from a group boundary on + // reconnect. A bounded policy is available via `ClientConfig::backoff`. let mut config = moq_native::ClientConfig::default(); config.tls.disable_verify = Some(settings.tls_disable_verify); config.backoff.timeout = std::time::Duration::ZERO; diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index 9d0db20f83..a5131dbab3 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -125,39 +125,15 @@ pub enum Error { } impl Error { - /// Whether repeating the failed operation could plausibly succeed with nothing else changing. - /// See [`moq_net::Error::is_retryable`]. + /// The HTTP status the origin answered with, if it answered with one at all. /// - /// The gateway sits between an HTTP origin and a MoQ relay, so both halves can be transient. A - /// playlist that didn't parse, a segment whose byte range didn't add up, or a URL that isn't one - /// will fail identically on the next pass: those end the import instead of looping on it. - pub fn is_retryable(&self) -> bool { + /// The import loop reads this through [`moq_net::retry::status_retryable`]: a `503` on a playlist + /// fetch is worth another pass, a `404` is the origin's settled answer. Nothing else here is + /// classified; a failure with no status falls through to the backoff budget. + pub fn status(&self) -> Option { match self { - Self::Moq(err) => err.is_retryable(), - Self::Mux(err) => err.is_retryable(), - Self::Io(err) => moq_net::retry::io_retryable(err), - // No response at all is the network; a response that arrived is the origin's answer. - Self::Reqwest(err) => err - .status() - .is_none_or(|status| moq_net::retry::status_retryable(status.as_u16())), - - // The playlist, its URLs, or the segments it points at are malformed. - Self::InvalidPlaylistUrl - | Self::InvalidFilePath - | Self::InvalidFileUrl - | Self::UrlParse(_) - | Self::ParsePlaylist(_) - | Self::NoVariants - | Self::MissingMap - | Self::EmptySegmentUri - | Self::MissingByteRangeOffset { .. } - | Self::InvalidByteRange { .. } - | Self::ByteRangeLengthMismatch { .. } - | Self::ByteRangeResponseMismatch { .. } - | Self::SequenceOverflow { .. } => false, - - // Untyped, so there is nothing to classify on. - Self::Other(_) => false, + Self::Reqwest(err) => err.status().map(|status| status.as_u16()), + _ => None, } } } @@ -187,25 +163,12 @@ pub type Result = std::result::Result; mod tests { use super::*; - /// The import loop retries on this classification, so a malformed playlist ending up on the - /// retryable side is an infinite loop that publishes nothing. + /// The import loop consults this, so an origin's settled answer has to reach it intact. #[test] - fn only_transient_failures_are_retryable() { - assert!(Error::Moq(moq_net::Error::Transport("connection lost".to_string())).is_retryable()); - assert!(Error::from(std::io::Error::from(std::io::ErrorKind::ConnectionReset)).is_retryable()); - - for err in [ - Error::ParsePlaylist("not a playlist".to_string()), - Error::NoVariants, - Error::MissingMap, - Error::InvalidPlaylistUrl, - Error::SequenceOverflow { - kind: SequenceKind::Media, - value: u64::MAX, - }, - Error::from(std::io::Error::from(std::io::ErrorKind::NotFound)), - ] { - assert!(!err.is_retryable(), "{err} should be terminal"); - } + fn an_http_failure_reports_its_status() { + // A failure the origin never answered carries no status, so the budget decides instead. + assert_eq!(Error::NoVariants.status(), None); + assert_eq!(Error::ParsePlaylist("not a playlist".to_string()).status(), None); + assert_eq!(Error::Moq(moq_net::Error::Transport("lost".to_string())).status(), None); } } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 39eaafa38f..c39a966252 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -38,9 +38,10 @@ pub use rendition::{Kind, Rendition}; /// for that reason: a source that stays silent for an hour must not be polled four times a second /// for an hour. /// -/// Deliberately no give-up budget. The broadcast closing is what ends the wait, and a relay-side -/// broadcast outlives its publisher's session, so any deadline here is a window in which a publisher -/// outage leaves the broadcaster permanently empty with nothing to recover it. +/// Deliberately no give-up budget, and no attempt to judge which failures are worth waiting on. The +/// broadcast closing is what ends the wait, and a relay-side broadcast outlives its publisher's +/// session, so any deadline here is a window in which a publisher outage leaves the broadcaster +/// permanently empty with nothing to recover it. fn catalog_backoff() -> moq_net::retry::Backoff { let mut config = moq_net::retry::Config::default(); config.initial = Duration::from_millis(250); @@ -182,20 +183,6 @@ impl Drop for Broadcaster { } } -/// Whether a failed catalog subscription is worth waiting on. -/// -/// Any moq-level failure means "not yet": the publisher announced the broadcast before creating its -/// catalog track, the route is still resolving, the session blipped. Waiting is the whole point of -/// [`watch_catalog`]'s first loop, and the broadcast closing is what ends the wait. -/// -/// Deliberately *not* [`moq_net::Error::is_retryable`], which reads `NotFound` as needing an -/// external change before another attempt can help. That is true in general, but here the external -/// change is the publisher writing the track, which is precisely what this loop exists to wait for. -/// Everything else is a catalog this build cannot read, and no amount of waiting fixes that. -fn catalog_pending(err: &moq_mux::Error) -> bool { - matches!(err, moq_mux::Error::Moq(_)) -} - async fn watch_catalog( source: moq_mux::Source, broadcast: moq_net::broadcast::Consumer, @@ -207,11 +194,6 @@ async fn watch_catalog( let mut consumer = loop { match catalog::Consumer::<()>::new(&broadcast, CatalogFormat::Hang).await { Ok(consumer) => break consumer, - Err(err) if !catalog_pending(&err) => { - tracing::warn!(%err, "cannot subscribe to broadcast catalog"); - renditions.close(); - return; - } Err(err) => { tracing::warn!(%err, "failed to subscribe to broadcast catalog, retrying"); tokio::select! { @@ -244,21 +226,6 @@ async fn watch_catalog( mod tests { use super::*; - /// The startup race this loop exists for: an exporter that subscribes between the announcement - /// and the publisher creating `catalog.json` sees the track as absent, and has to keep waiting. - /// Treating that as terminal leaves the broadcaster permanently empty with no error anywhere. - #[test] - fn a_missing_catalog_track_keeps_waiting() { - assert!(catalog_pending(&moq_net::Error::NotFound.into())); - assert!(catalog_pending(&moq_net::Error::Unroutable.into())); - assert!(catalog_pending( - &moq_net::Error::Transport("connection lost".to_string()).into() - )); - - // A catalog that arrived and could not be understood is not a waiting problem. - assert!(!catalog_pending(&moq_mux::Error::UnknownFormat("mystery".to_string()))); - } - fn frame(micros: u64, keyframe: bool) -> moq_mux::container::Frame { moq_mux::container::Frame { timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index de64a7e732..44781edf86 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -599,12 +599,10 @@ impl Import { /// Run the import loop until cancelled. /// - /// A transient step failure (an origin 503, a dropped connection) is logged and retried with - /// escalating backoff. A failure that says the source is broken rather than briefly unavailable - /// (a playlist that doesn't parse, a segment whose byte range doesn't add up) ends the import: - /// the next pass reads the same bytes and fails the same way, so looping on it only hides the - /// cause. The import also ends once the backoff budget is spent, so a permanently unreachable - /// origin surfaces instead of being retried forever. + /// A failed step is logged and retried with escalating backoff, and the import ends once the + /// backoff budget is spent, so a broken source surfaces instead of looping forever. The one + /// shortcut is an HTTP status the origin actually sent: a `404` playlist ends the import + /// immediately, since no amount of waiting turns it into a `200`. pub async fn run(&mut self) -> Result<()> { let mut backoff = error_backoff(); @@ -614,7 +612,15 @@ impl Import { backoff.reset(); outcome } - Err(err) if !err.is_retryable() => return Err(err), + // A status the origin actually sent is its answer: a 404 playlist is not going to + // become a 200 on the next pass. Everything else rides the backoff budget. + Err(err) + if err + .status() + .is_some_and(|status| !moq_net::retry::status_retryable(status)) => + { + return Err(err); + } Err(err) => { warn!(%err, "HLS import step failed, retrying"); if !backoff.sleep().await { diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index 27b3f1bbc8..9ca1979840 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -139,52 +139,6 @@ impl Error { pub(crate) fn unsupported_container(container: &hang::catalog::UnknownContainer) -> Self { Self::UnsupportedContainer(container.kind().unwrap_or("").to_string()) } - - /// Whether repeating the failed operation could plausibly succeed with nothing else changing. - /// See [`moq_net::Error::is_retryable`]. - /// - /// Only the transport and local I/O can be transient here. Everything else is a property of the - /// bytes themselves: a container, codec, or catalog that failed to parse once parses to the same - /// failure every time, so a loop that retries it never converges. - pub fn is_retryable(&self) -> bool { - match self { - Self::Moq(err) => err.is_retryable(), - Self::Io(err) => moq_net::retry::io_retryable(err), - - // Container and catalog parsing. Exhaustive rather than a catch-all so a variant that - // is genuinely transient has to say so instead of inheriting this by accident. - Self::Hang(_) - | Self::Json(_) - | Self::Cmaf(_) - | Self::Mkv(_) - | Self::Msf(_) - | Self::Loc(_) - | Self::Mp4(_) - | Self::UnknownFormat(_) - | Self::UnsupportedContainer(_) - | Self::ReservedSection(_) - | Self::InvalidTimescale(_) => false, - - // Codec bitstream parsing. - Self::Annexb(_) - | Self::Aac(_) - | Self::Opus(_) - | Self::Flac(_) - | Self::Mp3(_) - | Self::H264(_) - | Self::H265(_) - | Self::Av1(_) - | Self::Vp8(_) - | Self::Vp9(_) - | Self::Legacy(_) => false, - - // Timing and framing that the bytes themselves determine. - Self::TimestampOverflow(_) | Self::MissingKeyframe(_) | Self::NegativeFlvPts { .. } => false, - - // A URL that isn't one, and the untyped `anyhow` catch-all: nothing to classify on. - Self::Url(_) | Self::Other(_) => false, - } - } } impl From for Error { diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index 6dc1359beb..dd3bd12d72 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -146,78 +146,28 @@ impl Error { self.connect_error().is_some_and(|err| err.is_auth()) } - /// Whether reconnecting could plausibly succeed with nothing else changing. + /// The HTTP status a server answered a connection attempt with, if it answered with one at all. /// - /// A reconnect loop should call this before every retry. Half of what can go wrong here is - /// configuration (an unbuildable TLS config, a URL no compiled-in backend can dial, a flag the - /// backend doesn't support) or credentials, and those fail identically forever: the loop has to - /// surface them instead of hiding them behind a warning every few seconds. - /// - /// Retryable is the explicit case, never the fallback. The match is exhaustive so a new variant - /// is a decision rather than an accident. - pub fn is_retryable(&self) -> bool { + /// `None` covers everything else: a dial that never got a response, a QUIC handshake that + /// failed, a URL we couldn't parse. Only a status the peer actually sent shows up here, and + /// [`moq_net::retry::status_retryable`] is what decides whether it invites another attempt. This + /// deliberately does not try to say whether some *other* kind of failure is worth retrying; + /// that's a guess, and the caller's backoff budget bounds it instead. + pub fn status(&self) -> Option { match self { - // The OS refused a socket or a file. `kind` separates a refused port from a missing - // certificate, which is the difference between a retry and a typo. - Self::Io(err) => io_retryable(err), - - // The MoQ session's own classification, once the transport was up. - Self::MoqNet(err) => err.is_retryable(), - - // Every backend gave up, or the dial plus handshake outlived its deadline. Both are the - // network failing to answer. - Self::ConnectFailed | Self::ConnectTimeout(_) => true, - - // The race is retryable if either half is: one transport being permanently unusable - // (say, no WebSocket route) shouldn't retire the other. - #[cfg(feature = "websocket")] - Self::TransportRace { quic, websocket } => quic.is_retryable() || websocket.is_retryable(), - #[cfg(feature = "quinn")] - Self::Quinn(err) => err.is_retryable(), + Self::Quinn(err) => err.status(), #[cfg(feature = "noq")] - Self::Noq(err) => err.is_retryable(), + Self::Noq(err) => err.status(), #[cfg(feature = "quiche")] - Self::Quiche(err) => err.is_retryable(), - #[cfg(feature = "iroh")] - Self::Iroh(err) => err.is_retryable(), + Self::Quiche(err) => err.status(), #[cfg(feature = "websocket")] - Self::WebSocket(err) => err.is_retryable(), - #[cfg(feature = "tcp")] - Self::Tcp(err) => err.is_retryable(), - #[cfg(all(feature = "uds", unix))] - Self::Unix(err) => err.is_retryable(), - - // The server rejected our credentials. Retrying needs a new token, not a new attempt. - Self::Connect(_) => false, - - // Build and configuration failures: nothing about the next attempt differs. - Self::NoBackend(_) | Self::QlogUnsupported | Self::MtlsUnsupported | Self::InvalidStatusCode => false, - #[cfg(feature = "iroh")] - Self::IrohDisabled => false, - Self::Tls(_) => false, - - // Process setup, reached long before any connect. - Self::Directive(_) | Self::SetSubscriber(_) | Self::Logcat(_) => false, - - // A reconnect loop already gave up here. Retrying it is the nested-retry bug. - Self::Reconnect(_) => false, + Self::WebSocket(err) => err.status(), + _ => None, } } } -pub(crate) use moq_net::retry::io_retryable; - -/// Whether an HTTP failure is worth another attempt. -/// -/// No response at all is the network failing; a response that did arrive is the server's answer, so -/// only [`moq_net::retry::status_retryable`] statuses invite another try. -#[cfg(any(feature = "quinn", feature = "noq", feature = "quiche"))] -pub(crate) fn http_retryable(err: &reqwest::Error) -> bool { - err.status() - .is_none_or(|status| moq_net::retry::status_retryable(status.as_u16())) -} - // The wrapped sources aren't `Clone`, so `#[from]` can't store them behind `Arc` // directly. These hand-written conversions keep `?` ergonomic at the call sites. impl From for Error { diff --git a/rs/moq-native/src/iroh.rs b/rs/moq-native/src/iroh.rs index 88dff389d3..872eda047d 100644 --- a/rs/moq-native/src/iroh.rs +++ b/rs/moq-native/src/iroh.rs @@ -115,40 +115,6 @@ pub enum Error { type Result = std::result::Result; -impl Error { - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - pub(crate) fn is_retryable(&self) -> bool { - match self { - // Reading or writing the secret key file. `kind` tells a transient failure from a path - // that isn't there. - Self::Io(err) => crate::error::io_retryable(err), - - // The endpoint's UDP socket, which relay discovery and hole punching sit on top of. - Self::Bind(_) => true, - - // The exchange with the peer: dial, handshake, established connection, WebTransport. - Self::Connect(_) - | Self::Connecting(_) - | Self::Alpn(_) - | Self::Connection(_) - | Self::Client(_) - | Self::Server(_) - | Self::RecvRequest(_) => true, - - // Configuration: the key, the bind address, or a URL that isn't an endpoint id. - Self::Secret(_) - | Self::BindAddr(_) - | Self::MissingHost - | Self::InvalidEndpointId(_) - | Self::InvalidUrl - | Self::Url(_) => false, - - // Negotiation produced something we can't speak, and GSO can't be turned off here. - Self::DecodeAlpn(_) | Self::UnsupportedAlpn(_) | Self::GsoUnsupported => false, - } - } -} - /// Settings for the shared iroh endpoint, used by both the client and server. #[derive(clap::Args, Clone, Debug, Default, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields, default)] diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index eee1aa24b8..ab0057c956 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -398,61 +398,20 @@ impl Error { } } - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - pub(crate) fn is_retryable(&self) -> bool { + /// The HTTP status a server answered with, if it answered with one at all. + /// + /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the + /// WebTransport CONNECT response. See [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { match self { - // Local socket and endpoint setup. `kind` tells a port already in use (permanent until - // something else moves) from a transient failure to allocate one. - Self::BindSocket(err) | Self::CreateEndpoint(err) | Self::LocalAddr(err) | Self::ResolveBind(err) => { - crate::error::io_retryable(err) - } - - // DNS is a service like any other: a lookup failure, or an answer that hasn't - // propagated yet, resolves on its own. - Self::DnsLookup(_) | Self::NoDnsEntries => true, - - // The `http://` fingerprint bootstrap, which is a plain HTTP request to the relay. Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { - crate::error::http_retryable(err) + err.status().map(|status| status.as_u16()) } - - // A CONNECT the server actually answered is its settled response unless the status says - // otherwise, so a wrong path (404) or an endpoint that doesn't speak WebTransport (405) - // surfaces now instead of burning the whole reconnect budget. - Self::Client(err) => client_status(err).is_none_or(moq_net::retry::status_retryable), - - // The rest of the QUIC exchange: handshake, established connection, and the server side - // of a CONNECT. - Self::Connection(_) | Self::Establish(_) | Self::Server(_) | Self::RecvRequest(_) => true, - - // Retryable if any raced address is: one unroutable address must not retire the rest. - Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), - - // noq refused before a packet left the machine, so the next attempt is identical. - Self::Connect(_) => false, - - // The server's settled answer on our credentials. - Self::ConnectRejected(_) => false, - - // Configuration: the URL, the QUIC-LB sizing, the TLS material, or a missing runtime. - Self::NoRuntime - | Self::InvalidDnsName - | Self::InvalidFingerprint(_) - | Self::InvalidScheme - | Self::UnsupportedScheme(_) - | Self::QuicLbNonceTooSmall - | Self::QuicLbCidTooLong(_) - | Self::ClientVerifier(_) - | Self::NoInitialCipherSuite(_) - | Self::Tls(_) => false, - - // Negotiation produced something we can't speak. Both ends have to change first. - Self::MissingHandshake - | Self::MissingAlpn - | Self::DecodeAlpn(_) - | Self::UnsupportedAlpn(_) - | Self::MissingServerName - | Self::BuildUrl(_) => false, + Self::Client(err) => client_status(err), + // One address answering is not the set answering, so a raced dial reports nothing + // rather than letting a single response speak for the rest. + Self::Failover(_) => None, + _ => None, } } } @@ -473,9 +432,9 @@ fn classify_client_error(err: &web_transport_noq::ClientError) -> Option Option { match err { web_transport_noq::ClientError::HttpError(err) => connect_status(err), diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index 59d5342b28..9199999e93 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -474,59 +474,20 @@ impl Error { } } - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - // The deprecated variants are never constructed, but an exhaustive match is what makes a new - // variant a deliberate classification rather than a silent "not retryable". - #[allow(deprecated)] - pub(crate) fn is_retryable(&self) -> bool { + /// The HTTP status a server answered with, if it answered with one at all. + /// + /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the + /// WebTransport CONNECT response. See [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { match self { - // Sockets and local addresses. `kind` tells a port already in use (permanent until - // something else moves) from a transient failure to allocate one. - Self::Io(err) | Self::ResolveBind(err) | Self::Connect(err) | Self::ServerBuild(err) => { - crate::error::io_retryable(err) - } - - // DNS is a service like any other: a lookup failure, or an answer that hasn't - // propagated yet, resolves on its own. - Self::DnsLookup(_) | Self::NoDnsEntries => true, - - // The `http://` fingerprint bootstrap, which is a plain HTTP request to the relay. Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { - crate::error::http_retryable(err) - } - - // A CONNECT the server actually answered is its settled response unless the status says - // otherwise, so a wrong path (404) or an endpoint that doesn't speak WebTransport (405) - // surfaces now instead of burning the whole reconnect budget. - Self::ClientConnect(err) => client_status(err).is_none_or(moq_net::retry::status_retryable), - - // The rest of the QUIC exchange: handshake, established connection, and the server side - // of a CONNECT. - Self::Connection(_) | Self::Establish(_) | Self::AcceptRequest(_) | Self::Accept(_) | Self::Reject(_) => { - true + err.status().map(|status| status.as_u16()) } - - // Retryable if any raced address is: one unroutable address must not retire the rest. - Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), - - // The server's settled answer on our credentials. - Self::ConnectRejected(_) => false, - - // Configuration: the URL, the certificates, the fingerprint, or an unbound server. - Self::InvalidDnsName - | Self::InvalidFingerprint(_) - | Self::FingerprintLength(_) - | Self::InvalidScheme - | Self::NoLocalAddr - | Self::CertRequired - | Self::CertPairMismatch - | Self::Tls(_) => false, - - // Negotiation produced something we can't speak. Both ends have to change first. - Self::MissingAlpn | Self::DecodeAlpn(_) | Self::UnsupportedAlpn(_) => false, - - // Unsupported build/flag combinations, kept only so old code still compiles. - Self::FingerprintUnsupported | Self::HostNameUnsupported | Self::GsoUnsupported => false, + Self::ClientConnect(err) => client_status(err), + // One address answering is not the set answering, so a raced dial reports nothing + // rather than letting a single response speak for the rest. + Self::Failover(_) => None, + _ => None, } } } @@ -547,9 +508,9 @@ fn classify_client_error(err: &web_transport_quiche::ClientError) -> Option Option { match err { web_transport_quiche::ClientError::Connect(err) => connect_status(err), diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 6c6e191637..66c089e1fd 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -411,65 +411,20 @@ impl Error { } } - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - pub(crate) fn is_retryable(&self) -> bool { + /// The HTTP status a server answered with, if it answered with one at all. + /// + /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the + /// WebTransport CONNECT response. See [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { match self { - // Local socket and endpoint setup. `kind` tells a port already in use (permanent until - // something else moves) from a transient failure to allocate one. - Self::BindSocket(err) | Self::CreateEndpoint(err) | Self::LocalAddr(err) | Self::ResolveBind(err) => { - crate::error::io_retryable(err) - } - - // DNS is a service like any other: a lookup failure, or an answer that hasn't - // propagated yet, resolves on its own. - Self::DnsLookup(_) | Self::NoDnsEntries => true, - - // The `http://` fingerprint bootstrap, which is a plain HTTP request to the relay. Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { - crate::error::http_retryable(err) + err.status().map(|status| status.as_u16()) } - - // A CONNECT the server actually answered is its settled response unless the status says - // otherwise, so a wrong path (404) or an endpoint that doesn't speak WebTransport (405) - // surfaces now instead of burning the whole reconnect budget. - Self::Client(err) => client_status(err).is_none_or(moq_net::retry::status_retryable), - - // The rest of the QUIC exchange: handshake, established connection, and the server side - // of a CONNECT. Deliberately not decomposed. A rejected certificate arrives as a closed - // connection here and is retried until the give-up budget expires, which is the right - // call while certificates rotate underneath a long-lived publisher. - Self::Connection(_) | Self::Establish(_) | Self::Server(_) | Self::RecvRequest(_) => true, - - // Retryable if any raced address is: one unroutable address must not retire the rest. - Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), - - // Quinn refused before a packet left the machine, so the next attempt is identical. - Self::Connect(_) => false, - - // The server's settled answer on our credentials. - Self::ConnectRejected(_) => false, - - // Configuration: the URL, the qlog directory, the QUIC-LB sizing, the TLS material, - // or a runtime that isn't there. - Self::CreateQlog(_) - | Self::NoRuntime - | Self::InvalidDnsName - | Self::InvalidFingerprint(_) - | Self::InvalidScheme - | Self::UnsupportedScheme(_) - | Self::QuicLbNonceTooSmall - | Self::QuicLbCidTooLong(_) - | Self::ClientVerifier(_) - | Self::NoInitialCipherSuite(_) - | Self::Tls(_) => false, - - // Negotiation produced something we can't speak. Both ends have to change first. - Self::MissingHandshake - | Self::MissingAlpn - | Self::DecodeAlpn(_) - | Self::UnsupportedAlpn(_) - | Self::MissingServerName - | Self::BuildUrl(_) => false, + Self::Client(err) => client_status(err), + // One address answering is not the set answering, so a raced dial reports nothing + // rather than letting a single response speak for the rest. + Self::Failover(_) => None, + _ => None, } } } @@ -490,9 +445,9 @@ fn classify_client_error(err: &web_transport_quinn::ClientError) -> Option Option { match err { web_transport_quinn::ClientError::HttpError(err) => connect_status(err), @@ -765,20 +720,22 @@ mod tests { )) } - /// A CONNECT the relay answered is its settled response: a wrong path or an endpoint that - /// doesn't speak WebTransport must surface immediately rather than after the whole reconnect - /// budget. Only the "ask again later" statuses buy another attempt. + /// A CONNECT the relay answered carries its status through to the caller, so a wrong path or an + /// endpoint that doesn't speak WebTransport can surface immediately rather than after the whole + /// reconnect budget. #[test] - fn a_rejected_connect_status_is_terminal() { + fn a_rejected_connect_reports_its_status() { for status in [400, 404, 405, 410, 501] { + assert_eq!(connect_rejected(status).status(), Some(status)); assert!( - !connect_rejected(status).is_retryable(), + !moq_net::retry::status_retryable(status), "{status} should stop the reconnect loop" ); } for status in [408, 429, 502, 503, 504] { - assert!(connect_rejected(status).is_retryable(), "{status} should be retried"); + assert_eq!(connect_rejected(status).status(), Some(status)); + assert!(moq_net::retry::status_retryable(status), "{status} should be retried"); } // Auth is peeled off into its own variant before reaching the generic client arm. diff --git a/rs/moq-native/src/reconnect.rs b/rs/moq-native/src/reconnect.rs index 3b6cd69dd7..e5fa22f70c 100644 --- a/rs/moq-native/src/reconnect.rs +++ b/rs/moq-native/src/reconnect.rs @@ -10,9 +10,13 @@ use crate::{Client, Error}; /// Exponential backoff configuration for reconnection attempts. /// -/// Only failures that could plausibly clear on their own are retried at all -/// ([`Error::is_retryable`]); this decides how long to wait between those retries and when to stop. -/// The delays carry jitter, so a fleet knocked offline together doesn't reconnect in lockstep. +/// This decides how long to wait between reconnect attempts and when to give up. The delays carry +/// jitter, so a fleet knocked offline together doesn't reconnect in lockstep. +/// +/// [`timeout`](Self::timeout) is what ends a hopeless loop, not a judgment about the error: the only +/// failures short-circuited are the ones a server states outright (an auth rejection, or a CONNECT +/// status that isn't an invitation to retry). A zero timeout removes that backstop, so it belongs +/// only where an unattended process must outlive an outage of any length. #[derive(Clone, Debug, clap::Args, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] @@ -114,8 +118,8 @@ struct State { status: Option, /// The negotiated MoQ version of the live session, or `None` when disconnected. version: Option, - /// Set when the reconnect loop permanently gives up: a failure no retry can clear, or the - /// backoff timeout expiring. + /// Set when the reconnect loop permanently gives up: the backoff timeout expiring, or a server + /// answer that redialing cannot change. error: Option, /// The currently-connected session, or `None` while reconnecting. Read by /// [`ConnectionStatsReader`] to snapshot live connection stats. @@ -141,9 +145,9 @@ impl ConnectionStatsReader { /// Handle to a background reconnect loop. /// /// Spawns a tokio task that connects, waits for session close, then reconnects with exponential -/// backoff. This loop is the only retry owner for the connection: a caller that rebuilds it on -/// failure restarts the backoff from its initial delay, which turns the escalation back into a tight -/// loop. Watch [`closed`](Self::closed) instead. +/// backoff until [`Backoff::timeout`] runs out. This loop is the only retry owner for the connection: +/// a caller that rebuilds it on failure restarts the backoff from its initial delay, which turns the +/// escalation back into a tight loop. Watch [`closed`](Self::closed) instead. /// /// The read surface mirrors [`moq_net::Session`] so a caller can treat it like a session /// that transparently reconnects: [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), @@ -241,9 +245,6 @@ impl Reconnect { // sleep below so repeated flaps escalate instead of spinning the CPU. if let Err(err) = closed { let err = Error::from(err); - if !err.is_retryable() { - return Err(err); - } tracing::warn!(%url, %err, "session severed immediately, retrying"); last_error = Some(err); } else { @@ -252,10 +253,16 @@ impl Reconnect { } } Err(err) => { - // Auth, TLS material, an unsupported flag, a URL no backend can dial: the next - // dial is byte-for-byte the same, so surface it instead of hiding it behind a - // warning every few seconds. - if !err.is_retryable() { + // The two answers a server can give that redialing cannot change: it rejected our + // credentials, or it answered the CONNECT with a status that isn't an invitation + // to come back. Everything else falls through to the backoff, whose budget is + // what eventually stops the loop. + if err.is_auth() { + return Err(err); + } + if let Some(status) = err.status() + && !moq_net::retry::status_retryable(status) + { return Err(err); } last_error = Some(err); diff --git a/rs/moq-native/src/tcp.rs b/rs/moq-native/src/tcp.rs index 4d59c66502..5ae9cf7194 100644 --- a/rs/moq-native/src/tcp.rs +++ b/rs/moq-native/src/tcp.rs @@ -76,28 +76,6 @@ impl crate::failover::Aggregate for Error { } } -impl Error { - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - pub(crate) fn is_retryable(&self) -> bool { - match self { - // The TCP socket and DNS. `kind` tells a refused port from an unusable address. - Self::Io(err) => crate::error::io_retryable(err), - - // The qmux handshake, which is the exchange over an established socket. - Self::Connect(_) | Self::Accept(_) => true, - - // DNS answers propagate; an empty one now may not be empty in a minute. - Self::NoAddresses => true, - - // Retryable if any raced address is: one unroutable address must not retire the rest. - Self::Failover(failures) => failures.iter().any(|failure| failure.error.is_retryable()), - - // The URL is missing what `tcp://` requires, which no retry supplies. - Self::MissingHostname | Self::MissingPort => false, - } - } -} - type Result = std::result::Result; /// Dial a `tcp://host:port` URL, advertising `protocols` for in-band ALPN diff --git a/rs/moq-native/src/unix.rs b/rs/moq-native/src/unix.rs index 92938fa8d1..4a94bf658b 100644 --- a/rs/moq-native/src/unix.rs +++ b/rs/moq-native/src/unix.rs @@ -121,23 +121,6 @@ pub enum Error { type Result = std::result::Result; -impl Error { - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - pub(crate) fn is_retryable(&self) -> bool { - match self { - // The socket. A peer that hasn't created its socket yet reports `NotFound`, which is - // permanent as far as this layer knows: whoever starts it is the external change. - Self::Io(err) => crate::error::io_retryable(err), - - // The qmux handshake, which is the exchange over an established socket. - Self::Connect(_) | Self::Accept(_) => true, - - // The URL has no path, or the path is occupied by something we refuse to unlink. - Self::MissingPath | Self::NotASocket(_) => false, - } - } -} - /// Credentials of a connected Unix-socket peer. /// /// `pid` is `None` on platforms that don't report it (e.g. some macOS versions); diff --git a/rs/moq-native/src/websocket.rs b/rs/moq-native/src/websocket.rs index cdd0452272..f80a262077 100644 --- a/rs/moq-native/src/websocket.rs +++ b/rs/moq-native/src/websocket.rs @@ -224,25 +224,14 @@ impl Error { } } - /// Whether another dial could plausibly succeed. See [`crate::Error::is_retryable`]. - pub(crate) fn is_retryable(&self) -> bool { + /// The HTTP status the server answered the upgrade with, if it answered with one at all. + /// + /// qmux surfaces a non-101 WebSocket upgrade response as `Http(status)`. See + /// [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { match self { - // The TCP socket. `kind` tells a refused port from a bind address this host can't use. - Self::Io(err) => crate::error::io_retryable(err), - - // A non-101 upgrade is the server's answer, so only the "ask again later" statuses are - // worth another try. Every other qmux failure is the TCP/TLS exchange. - Self::Connect(qmux::Error::Http(status)) => moq_net::retry::status_retryable(*status), - Self::Connect(_) | Self::Accept(_) | Self::WebSocketConnect(_) => true, - - // The server's settled answer on our credentials. - Self::ConnectRejected(_) => false, - - // Configuration: the fallback is switched off, or the URL can't carry WebSocket. - Self::Disabled | Self::MissingHostname | Self::UnsupportedScheme(_) => false, - - // The handshake request couldn't even be built, so there is nothing to send again. - Self::BuildRequest(_) | Self::ProtocolHeader(_) => false, + Self::Connect(qmux::Error::Http(status)) => Some(*status), + _ => None, } } } diff --git a/rs/moq-native/tests/reconnect.rs b/rs/moq-native/tests/reconnect.rs index c041e777e2..2e45ff0774 100644 --- a/rs/moq-native/tests/reconnect.rs +++ b/rs/moq-native/tests/reconnect.rs @@ -1,8 +1,8 @@ -//! What the reconnect loop retries, and what it refuses to. +//! What ends the reconnect loop. //! -//! Both cases dial over plain TCP (`tcp://`), which fails fast and locally: no TLS material, no -//! QUIC handshake, no server. That keeps the assertions about the *policy* rather than about how -//! long a particular backend takes to give up. +//! Dials over plain TCP (`tcp://`), which fails fast and locally: no TLS material, no QUIC +//! handshake, no server. That keeps the assertion about the *budget* rather than about how long a +//! particular backend takes to give up. #![cfg(feature = "tcp")] @@ -15,30 +15,6 @@ fn client(backoff: moq_native::Backoff) -> moq_native::Client { config.init().expect("failed to init client") } -/// A failure no retry can clear must surface immediately. The initial delay is far longer than the -/// timeout below, so a single retry would blow the deadline: reaching the assertion at all is the -/// proof that exactly one attempt was made. -#[tokio::test] -async fn a_deterministic_failure_makes_one_attempt() { - let mut backoff = moq_native::Backoff::default(); - backoff.initial = Duration::from_secs(30); - - // `tcp://` has no default port, so this URL can never be dialed, however many times we try. - let url = "tcp://localhost".parse().expect("failed to parse url"); - let reconnect = client(backoff).reconnect(url); - - let err = tokio::time::timeout(Duration::from_secs(5), reconnect.closed()) - .await - .expect("reconnect loop retried a deterministic failure") - .expect_err("reconnect loop stopped without an error"); - - assert!(!err.is_retryable(), "gave up on a retryable error: {err}"); - assert!( - matches!(err, moq_native::Error::Tcp(_)), - "reported {err} instead of the failure that stopped it" - ); -} - /// A transient failure is retried, escalating, until the budget runs out. The give-up error names /// the underlying cause so an operator sees why rather than just "timed out". #[tokio::test] diff --git a/rs/moq-net/src/error.rs b/rs/moq-net/src/error.rs index 5d6b2a25b1..f817ccd108 100644 --- a/rs/moq-net/src/error.rs +++ b/rs/moq-net/src/error.rs @@ -178,61 +178,6 @@ impl Error { } } - /// Whether repeating the failed operation could plausibly succeed with nothing else changing. - /// - /// True only for the failures a flaky link produces. Everything else is deterministic: a decode - /// failure, an auth rejection, or a version mismatch will fail identically on the next attempt, - /// so a loop that retries it burns the network and hides the real cause behind a warning. - /// - /// Retryable is the explicit case, never the fallback: a variant nobody has classified is - /// terminal. The match is exhaustive so adding one is a decision rather than an accident. - /// - /// This says nothing about *when* to retry. Pair it with [`retry::Backoff`](crate::retry::Backoff). - pub fn is_retryable(&self) -> bool { - match self { - // The link itself failed. A session that drops for any reason lands here (see - // [`Session::closed`](crate::Session::closed)), which is the case reconnect loops exist for. - Self::Transport(_) => true, - // A stream took too long to open or transmit, so the path was congested or black-holing. - Self::Timeout => true, - // Memory pressure dropped a group that is still inside the publisher's window, so a - // re-fetch can genuinely get it back. - Self::Evicted => true, - - // Deterministic protocol and coding failures: the same bytes fail the same way. - Self::Decode(_) - | Self::Encode(_) - | Self::BoundsExceeded(_) - | Self::Version - | Self::RequiredExtension - | Self::UnexpectedStream - | Self::UnexpectedMessage - | Self::ProtocolViolation - | Self::InvalidRole - | Self::TooManyParameters - | Self::Unsupported - | Self::UnknownAlpn(_) - | Self::WrongSize - | Self::FrameTooLarge - | Self::TimestampMismatch - | Self::Duplicate => false, - - // Authorization needs new credentials, not another attempt. - Self::Unauthorized => false, - - // Absent content. Retrying can only help once somebody publishes it, which is an - // external change the caller should wait on (an announcement) rather than poll for. - Self::NotFound | Self::Unroutable => false, - - // Lifecycle, not failure: the operation is over and there is nothing left to repeat. - Self::Cancel | Self::Closed | Self::Dropped | Self::Old | Self::Lagged => false, - - // Chosen by the application or the peer, so this layer can't say. Whoever assigned the - // code is the one that knows whether it's worth another try. - Self::App(_) | Self::Remote(_) => false, - } - } - /// Convert a transport error into an [Error], decoding stream reset codes. pub fn from_transport(err: impl web_transport_trait::Error) -> Self { match err.stream_error() { @@ -275,29 +220,4 @@ mod tests { assert_eq!(Error::App(404).to_code(), 468); assert_eq!(Error::Remote(468).to_code(), 468); } - - /// A dropped session always surfaces as `Transport`, so this is the classification a - /// reconnect loop actually depends on. - #[test] - fn transport_failures_are_retryable() { - assert!(Error::Transport("connection lost".to_string()).is_retryable()); - assert!(Error::Timeout.is_retryable()); - } - - /// The failures a retry can only repeat. Each of these was previously retried forever by at - /// least one loop in the workspace. - #[test] - fn deterministic_failures_are_not_retryable() { - for err in [ - Error::Unauthorized, - Error::Version, - Error::ProtocolViolation, - Error::Unsupported, - Error::UnknownAlpn("moqt-99".to_string()), - Error::NotFound, - Error::Cancel, - ] { - assert!(!err.is_retryable(), "{err} should be terminal"); - } - } } diff --git a/rs/moq-net/src/retry.rs b/rs/moq-net/src/retry.rs index 3dbe5f936c..8f9de00203 100644 --- a/rs/moq-net/src/retry.rs +++ b/rs/moq-net/src/retry.rs @@ -1,9 +1,12 @@ //! The retry schedule shared by every loop that re-attempts a failed operation. //! -//! Two halves, kept apart on purpose. [`Error::is_retryable`](crate::Error::is_retryable) (and its -//! counterparts in the crates above) answers *whether* an attempt is worth repeating; [`Backoff`] -//! answers *when*. A loop that only has the second half retries deterministic failures forever, which -//! is the bug this module exists to prevent, so classify first and back off second. +//! [`Backoff`] answers *when* to try again, and its budget is what ends a loop. There is deliberately +//! no counterpart answering *whether* a given error is worth repeating: a transport layer can't tell +//! a permanent failure from a temporary one without guessing, and a wrong guess either strands a +//! recoverable connection or hammers a dead one. The budget bounds the damage instead. +//! +//! The one exception is [`status_retryable`], where a peer sent an HTTP status whose meaning the +//! protocol defines. That's reading an answer, not inferring one. //! //! ```no_run //! # async fn example() -> Result<(), moq_net::Error> { @@ -12,9 +15,7 @@ //! loop { //! match attempt().await { //! Ok(()) => return Ok(()), -//! // Deterministic: the next attempt fails the same way, so surface it now. -//! Err(err) if !err.is_retryable() => return Err(err), -//! // Transient, but the budget is spent: stop rather than retry forever. +//! // Out of budget: surface the failure instead of looping on it. //! Err(err) if !backoff.sleep().await => return Err(err), //! Err(_) => continue, //! } @@ -25,24 +26,6 @@ use kio::time::{Duration, Instant}; use rand::RngExt; -/// Whether an OS-level failure is worth another attempt. -/// -/// Configuration mistakes reach a caller as [`std::io::Error`] too: a path that doesn't exist, a -/// port another process holds, an address this host can't bind. Those repeat forever. What's left -/// (refused, unreachable, reset, timed out) is the network being the network. -pub fn io_retryable(err: &std::io::Error) -> bool { - !matches!( - err.kind(), - std::io::ErrorKind::NotFound - | std::io::ErrorKind::PermissionDenied - | std::io::ErrorKind::AddrInUse - | std::io::ErrorKind::AddrNotAvailable - | std::io::ErrorKind::InvalidInput - | std::io::ErrorKind::InvalidData - | std::io::ErrorKind::Unsupported - ) -} - /// Whether an HTTP response status means "ask again later". /// /// A response that arrived is the server's answer, and only this narrow set invites another attempt: diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 0c8215d7dd..ac5a066a56 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -917,9 +917,9 @@ impl Cluster { url.query_pairs_mut().append_pair("jwt", &token); } - // A peer is supervised for the life of the relay, so there is no give-up deadline: one that - // is unreachable for an hour still has to be redialed when it comes back. What ends the loop - // is classification, not a budget. + // A peer is supervised for the life of the relay, so there is no give-up deadline: one that is + // unreachable for an hour still has to be redialed when it comes back. Nothing ends this + // loop; the escalating delay is what keeps a permanently-dead peer cheap. let mut config = moq_net::retry::Config::default(); config.max = tokio::time::Duration::from_secs(300); config.timeout = tokio::time::Duration::ZERO; @@ -947,10 +947,6 @@ impl Cluster { match result { Ok(()) if elapsed >= stable_threshold => {} Ok(()) => tracing::warn!(?elapsed, "cluster peer session closed cleanly but quickly; backing off"), - // A rejected token, an ALPN neither side speaks, a URL this build can't dial: every - // redial produces the same failure, so stop and let the operator see it. The peer - // comes back when something external changes and re-announces it. - Err(err) if !peer_is_retryable(&err) => return Err(err.context("cluster peer rejected us")), Err(err) => tracing::warn!(%err, "cluster peer error; will retry"), } @@ -997,21 +993,6 @@ impl Cluster { } } -/// Whether a failed peer dial is worth repeating. -/// -/// The dial and the session both report typed errors that classify themselves; anything else is an -/// internal invariant or a malformed peer entry, which no redial fixes. -fn peer_is_retryable(err: &anyhow::Error) -> bool { - if let Some(err) = err.downcast_ref::() { - return err.is_retryable(); - } - if let Some(err) = err.downcast_ref::() { - return err.is_retryable(); - } - - false -} - /// Extract and remove the `cost` query param from a peer URL. /// /// The param is dial-side configuration, not something the peer reads off the @@ -1145,23 +1126,6 @@ mod tests { use super::*; use crate::Config; - /// A dropped session is what a redial exists for; a rejected token is not. Both reach the loop - /// as `anyhow`, wrapped in the `.context` the dial adds, so the classification has to survive - /// that wrapping (it's the whole reason this helper exists rather than a `matches!`). - #[test] - fn peer_retries_only_transient_failures() { - let dropped = anyhow::Error::from(moq_net::Error::Transport("connection lost".to_string())); - assert!(peer_is_retryable(&dropped)); - - let rejected = anyhow::Error::from(moq_native::Error::from(moq_native::ConnectError::Unauthorized)) - .context("failed to connect to cluster peer"); - assert!(!peer_is_retryable(&rejected)); - - // Nothing typed to go on: an internal invariant, or a peer entry that never parsed. - let internal = anyhow::anyhow!("cluster peer dial without an attached QUIC client"); - assert!(!peer_is_retryable(&internal)); - } - /// The publish task holds only a `Weak` to its producer, so it stops when the /// last `moq_stats::Producer` clone drops. Attaching one must therefore hand /// its lifetime to the cluster: an embedder driving its own loop takes the From 66ca2ea8834b57cd168ec9306be83b5ba9ccf8cb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 19:31:19 -0700 Subject: [PATCH 05/13] fix: four holes the removal opened up - The cluster loop timed the whole dial, not the session, so a peer that blackholed until the connect timeout looked "stable" and reset the backoff on every attempt. The escalation therefore never happened. `run_remote_session` now reports when the session actually came up, and the reset keys on the session's own lifetime. - A `Backoff` could sleep past its deadline by a whole window, since the jittered delay ignored what was left of the budget. An `initial` larger than `timeout` meant the first wait alone blew through it. Both the Rust and JS twins now cap the delay to the remaining budget, so the sequence lands on the deadline instead of overshooting it. - `moq_native::Error::status` dropped the status when a transport race lost both halves, so a `404` answered over both QUIC and WebSocket read as a transient failure. It now reports one, but only when both halves were answered and neither answer invites a retry. - The HLS import reset its backoff on any `Ok` step, and `step` returns `Ok` even when every rendition failed (it swallows per-rendition errors so one bad variant doesn't drop the rest). A source returning 404 forever would spin at the refresh cadence publishing nothing. `StepOutcome` now carries the failure, and a pass that imported nothing while a rendition was failing is treated as the failure it is. Co-Authored-By: Claude Opus 5 --- js/net/src/retry.test.ts | 10 +++++ js/net/src/retry.ts | 20 ++++++---- rs/moq-hls/src/import.rs | 28 +++++++++++++- rs/moq-native/src/error.rs | 13 +++++++ rs/moq-net/src/retry.rs | 68 +++++++++++++++++++++++++-------- rs/moq-relay/src/cluster.rs | 76 +++++++++++++++++++++++++++---------- 6 files changed, 169 insertions(+), 46 deletions(-) diff --git a/js/net/src/retry.test.ts b/js/net/src/retry.test.ts index 16b2253227..b944f50f8e 100644 --- a/js/net/src/retry.test.ts +++ b/js/net/src/retry.test.ts @@ -34,6 +34,16 @@ test("a zero timeout never gives up", () => { for (let i = 0; i < 64; i++) expect(backoff.delay()).toBeDefined(); }); +test("a delay never outlives the budget", () => { + // An initial delay longer than the whole budget must not sleep past it: the budget is the + // promise, and one oversized window would blow through it before a single retry lands. + const backoff = new Backoff({ initial: 60000, multiplier: 2, max: 60000, timeout: 50 }); + + const delay = backoff.delay(); + expect(delay).toBeDefined(); + expect(delay).toBeLessThanOrEqual(50); +}); + test("the budget is a deadline over the whole sequence", async () => { const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 5 }); diff --git a/js/net/src/retry.ts b/js/net/src/retry.ts index 7d7165fcda..d6748e87e7 100644 --- a/js/net/src/retry.ts +++ b/js/net/src/retry.ts @@ -79,22 +79,26 @@ export class Backoff { /** How long to wait before the next attempt, or undefined once the budget is spent. */ delay(): DOMHighResTimeStamp | undefined { + let remaining = Number.POSITIVE_INFINITY; + if (this.#timeout > 0) { const now = performance.now(); - if (this.#deadline === undefined) { - // The first delay of a sequence starts the clock, so a loop that ran healthy for - // hours still gets its full budget when it finally does fail. - this.#deadline = now + this.#timeout; - } else if (now >= this.#deadline) { - return undefined; - } + // The first delay of a sequence starts the clock, so a loop that ran healthy for hours + // still gets its full budget when it finally does fail. + this.#deadline ??= now + this.#timeout; + + remaining = this.#deadline - now; + if (remaining <= 0) return undefined; } // Equal jitter: at least half the window, never more than all of it. const delay = this.#window / 2 + Math.random() * (this.#window / 2); this.#window = Math.min(this.#window * this.#multiplier, this.#max); - return delay; + // Never sleep past the deadline: the budget says how long to keep retrying, so overshooting + // it by a whole window would spend more than the caller asked for and skip the attempt that + // still fit. A truncated final delay is the point, not a rounding error. + return Math.min(delay, remaining); } /** diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 44781edf86..6e138f7e11 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -96,6 +96,12 @@ struct StepOutcome { wrote_segments: usize, /// Target segment duration (in seconds) from the playlist, if known. target_duration: Option, + /// The last rendition failure of this step, if any. + /// + /// [`OnError::Warn`] keeps the other renditions going after one fails, so a step can report + /// `Ok` having imported nothing at all. The loop needs to tell that apart from a quiet playlist + /// with no new segments, or it treats a permanently broken source as steady progress. + failed: Option, } /// What a step does when a rendition fails. @@ -607,7 +613,20 @@ impl Import { let mut backoff = error_backoff(); loop { - let outcome = match self.step(OnError::Warn).await { + // A step that imported nothing while a rendition was failing is a failed pass wearing an + // `Ok`: `step` swallows per-rendition errors so one bad variant doesn't drop the rest. + // Letting it through would reset the backoff every pass, so a source that returns 404 + // forever would spin at the refresh cadence while publishing nothing. + let stepped = match self.step(OnError::Warn).await { + Ok(StepOutcome { + wrote_segments: 0, + failed: Some(err), + .. + }) => Err(err), + stepped => stepped, + }; + + let outcome = match stepped { Ok(outcome) => { backoff.reset(); outcome @@ -672,6 +691,7 @@ impl Import { let mut wrote_segments = 0; let mut target_duration = None; + let mut failed = None; for track in self.video.iter_mut().chain(self.audio.iter_mut()) { match track.ingest(&self.fetcher, &mut target_duration).await { @@ -680,7 +700,10 @@ impl Import { OnError::Fail => return Err(err), // Keep the other renditions going: one bad variant or segment shouldn't // drop the rest or abort the whole step. - OnError::Warn => warn!(label = %track.label, %err, "rendition import step failed, will retry"), + OnError::Warn => { + warn!(label = %track.label, %err, "rendition import step failed, will retry"); + failed = Some(err); + } }, } } @@ -688,6 +711,7 @@ impl Import { Ok(StepOutcome { wrote_segments, target_duration, + failed, }) } diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index dd3bd12d72..fa244e6303 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -155,6 +155,19 @@ impl Error { /// that's a guess, and the caller's backoff budget bounds it instead. pub fn status(&self) -> Option { match self { + // A race is only settled when both halves were answered, and answered with something not + // worth repeating: one transport being refused says nothing about the other, so a `404` + // over QUIC alongside a dead WebSocket is still just a failed dial. + #[cfg(feature = "websocket")] + Self::TransportRace { quic, websocket } => match (quic.status(), websocket.status()) { + (Some(quic), Some(websocket)) + if !moq_net::retry::status_retryable(quic) && !moq_net::retry::status_retryable(websocket) => + { + Some(quic) + } + _ => None, + }, + #[cfg(feature = "quinn")] Self::Quinn(err) => err.status(), #[cfg(feature = "noq")] diff --git a/rs/moq-net/src/retry.rs b/rs/moq-net/src/retry.rs index 8f9de00203..2c94e8d8ab 100644 --- a/rs/moq-net/src/retry.rs +++ b/rs/moq-net/src/retry.rs @@ -109,16 +109,26 @@ impl Backoff { pub fn delay(&mut self) -> Option { // An unlimited budget never reads the clock, which is what lets a blocking thread with its // own [`std::time::Instant`] bookkeeping drive this too. + let mut remaining = None; if !self.config.timeout.is_zero() { let now = Instant::now(); - match self.deadline { + // The first delay of a sequence starts the clock. Deferred to here rather than to + // `new`/`reset` so a loop that runs healthy for hours still gets its full budget when it + // finally does fail. An unrepresentable deadline is treated as no deadline. + let deadline = match self.deadline { + Some(deadline) => Some(deadline), + None => { + self.deadline = now.checked_add(self.config.timeout); + self.deadline + } + }; + + if let Some(deadline) = deadline { // Started already: stop once the budget is gone. - Some(deadline) if now >= deadline => return None, - Some(_) => {} - // The first delay of a sequence starts the clock. Deferred to here rather than to - // `new`/`reset` so a loop that runs healthy for hours still gets its full budget - // when it finally does fail. - None => self.deadline = now.checked_add(self.config.timeout), + if now >= deadline { + return None; + } + remaining = Some(deadline - now); } } @@ -128,7 +138,13 @@ impl Backoff { .saturating_mul(self.config.multiplier.max(1)) .min(self.config.max); - Some(delay) + // Never sleep past the deadline: the budget says how long to keep retrying, so overshooting + // it by a whole window would spend more than the caller asked for and skip the attempt that + // still fit. A truncated final delay is the point, not a rounding error. + Some(match remaining { + Some(remaining) => delay.min(remaining), + None => delay, + }) } /// Wait out the next delay, returning `false` once the budget is spent. @@ -242,22 +258,44 @@ mod tests { assert!(delay <= Duration::from_secs(1), "{delay:?} did not return to initial"); } - /// The budget is a wall-clock deadline over the whole sequence, not a per-attempt one. + /// An initial delay longer than the whole budget must not sleep past it: the budget is the + /// promise, and one oversized window would blow through it before a single retry lands. #[tokio::test(start_paused = true)] - async fn gives_up_once_the_budget_is_spent() { + async fn a_delay_never_outlives_the_budget() { let mut backoff = Backoff::new(Config { - timeout: Duration::from_secs(10), + initial: Duration::from_secs(60), + timeout: Duration::from_millis(50), ..config() }); - let mut slept = Duration::ZERO; + let delay = backoff.delay().expect("budget available"); + assert!(delay <= Duration::from_millis(50), "{delay:?} outlived the budget"); + } + + /// The budget is a wall-clock deadline over the whole sequence, not a per-attempt one, and the + /// sequence lands on it rather than overshooting by a whole window. + #[tokio::test(start_paused = true)] + async fn gives_up_once_the_budget_is_spent() { + let timeout = Duration::from_secs(10); + let mut backoff = Backoff::new(Config { timeout, ..config() }); + + let started = tokio::time::Instant::now(); while let Some(delay) = backoff.delay() { - slept += delay; tokio::time::sleep(delay).await; - assert!(slept < Duration::from_secs(60), "budget never ran out"); + assert!(started.elapsed() < Duration::from_secs(60), "budget never ran out"); } - assert!(slept >= Duration::from_secs(10), "gave up after only {slept:?}"); + // Tokio's paused clock rounds each sleep to its timer granularity, so the sequence can land a + // hair either side of the deadline it aimed for. + let elapsed = started.elapsed(); + assert!( + elapsed >= timeout - Duration::from_millis(10), + "gave up after only {elapsed:?}" + ); + assert!( + elapsed < timeout + config().max, + "overshot the budget by a whole window: {elapsed:?}" + ); } /// A zero timeout is the supervisor case: keep retrying however long the outage lasts. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index ac5a066a56..54751a3e75 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -931,30 +931,29 @@ impl Cluster { let stable_threshold = tokio::time::Duration::from_secs(10); loop { - let started = tokio::time::Instant::now(); - let result = self.run_remote_once(&url, cost).await; - let elapsed = started.elapsed(); - - // A session that lasted is a healthy peer, however it ended: clear the escalation so a - // one-off drop redials promptly. Keyed on how long it ran rather than on the outcome, - // because `run_remote_session` reports even a clean close as an error (it hands back the - // session's close reason). An outcome-keyed reset would therefore never fire, and a peer - // that had been up for hours would redial on a stale five-minute window. - if elapsed >= stable_threshold { + let attempt = self.run_remote_once(&url, cost).await; + + // A session that came up and lasted is a healthy peer: clear the escalation so a one-off + // drop redials promptly. Keyed on the session's own lifetime, not on how long the call + // took, because a peer that blackholes until the connect timeout would otherwise look + // stable and reset the backoff on every attempt, so the escalation would never happen. + let stable = attempt.connected.is_some_and(|at| at.elapsed() >= stable_threshold); + if stable { backoff.reset(); } - match result { - Ok(()) if elapsed >= stable_threshold => {} - Ok(()) => tracing::warn!(?elapsed, "cluster peer session closed cleanly but quickly; backing off"), - Err(err) => tracing::warn!(%err, "cluster peer error; will retry"), + if let Err(err) = attempt.result { + match stable { + true => tracing::warn!(%err, "cluster peer session closed; reconnecting"), + false => tracing::warn!(%err, "cluster peer error; will retry"), + } } backoff.sleep().await; } } - async fn run_remote_once(&self, url: &Url, cost: Option) -> anyhow::Result<()> { + async fn run_remote_once(&self, url: &Url, cost: Option) -> Attempt { // Each attempt is its own session, so it gets its own id. Matches the span an // accepted connection runs under, so both directions log the same way. let id = self.next_connection_id(); @@ -963,16 +962,20 @@ impl Cluster { .await } - async fn run_remote_session(&self, id: u64, url: &Url, cost: Option) -> anyhow::Result<()> { + async fn run_remote_session(&self, id: u64, url: &Url, cost: Option) -> Attempt { let mut log_url = url.clone(); log_url.set_query(None); tracing::info!(url = %log_url, "dialing cluster peer"); // Checked at the start of `run`; per-peer tasks inherit that guarantee. - let client = self + let client = match self .client .clone() - .context("internal: cluster peer dial without an attached QUIC client")?; + .context("internal: cluster peer dial without an attached QUIC client") + { + Ok(client) => client, + Err(err) => return Attempt::failed(err), + }; // Cluster dials use their configured stats tier. Cluster peers carry no auth // root, so presence is keyed under the empty root within the cluster tier. @@ -983,13 +986,44 @@ impl Cluster { if let Some(cost) = cost { client = client.with_cost(cost); } - let cs = client + let cs = match client .connect(url.clone()) .await - .context("failed to connect to cluster peer")?; + .context("failed to connect to cluster peer") + { + Ok(cs) => cs, + Err(err) => return Attempt::failed(err), + }; + + let connected = tokio::time::Instant::now(); let _connection = self.nodes.connect_outbound(id, log_url.to_string()); - Err(cs.closed().await.into()) + Attempt { + connected: Some(connected), + result: Err(cs.closed().await.into()), + } + } +} + +/// One peer dial: when its session came up, and how it ended. +/// +/// The instant is what the backoff reset keys on, which is why the dial and the session are timed +/// separately. A session always ends in an `Err` (it hands back its own close reason), so the +/// outcome alone can't tell a healthy peer from a dead one. +struct Attempt { + /// When `connect` handed back a live session, or `None` if the dial never got that far. + connected: Option, + /// How the attempt ended. + result: anyhow::Result<()>, +} + +impl Attempt { + /// An attempt that never established a session. + fn failed(err: anyhow::Error) -> Self { + Self { + connected: None, + result: Err(err), + } } } From 88a356db7c10c0f47039af1f91515ef946c25201 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 19:49:54 -0700 Subject: [PATCH 06/13] refactor(kio): move Backoff out of moq-net's public surface A backoff schedule is a utility, not part of the wire layer's published API. It now lives in `kio::time` next to `Deadline`, where the crates that need it (moq-native, moq-relay, moq-hls, moq-rtmp, moq-audio) already reach it. `moq_net::retry` is gone. `status_retryable` moves with its callers rather than being shared: moq-native and moq-hls each keep a private copy of the five-status list, so nothing about retry policy is exported from either. Three review findings folded in: - The budget now runs from construction or the last `reset` rather than from the first delay, so it covers the attempts as well as the waits between them. A reconnect whose every dial hung until the connect timeout could previously outlive its budget by that much, which also put it past the linger window `Client::consume` sizes from it. - When several HLS renditions fail in one pass, the step keeps the failure another pass could clear rather than whichever came last, so a single permanently-dead variant doesn't end an import the others could still serve. Keeping the last one made the outcome depend on rendition order. - `doc/bin/relay/cluster.md` still promised that a rejected cluster peer is given up on, which stopped being true when the classification came out. It documents the redial-forever behavior now. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 4 +- Cargo.lock | 5 + doc/bin/relay/cluster.md | 13 +- js/net/src/retry.ts | 29 +-- rs/CLAUDE.md | 2 +- rs/kio/Cargo.toml | 9 +- rs/kio/src/time.rs | 293 ++++++++++++++++++++++++- rs/moq-audio/Cargo.toml | 1 + rs/moq-audio/src/playback/driver.rs | 8 +- rs/moq-hls/src/error.rs | 11 +- rs/moq-hls/src/export/mod.rs | 6 +- rs/moq-hls/src/import.rs | 27 ++- rs/moq-hls/src/lib.rs | 1 + rs/moq-native/Cargo.toml | 1 + rs/moq-native/src/error.rs | 17 +- rs/moq-native/src/noq.rs | 2 +- rs/moq-native/src/quiche.rs | 2 +- rs/moq-native/src/quinn.rs | 6 +- rs/moq-native/src/reconnect.rs | 6 +- rs/moq-net/src/lib.rs | 1 - rs/moq-net/src/retry.rs | 322 ---------------------------- rs/moq-relay/Cargo.toml | 1 + rs/moq-relay/src/cluster.rs | 4 +- rs/moq-rtmp/Cargo.toml | 1 + rs/moq-rtmp/src/server.rs | 6 +- 25 files changed, 392 insertions(+), 386 deletions(-) delete mode 100644 rs/moq-net/src/retry.rs diff --git a/CLAUDE.md b/CLAUDE.md index 47e7bf8630..622cf41c88 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,13 +101,13 @@ The rename/removal rationale lives in the commit message and PR description, not Retrying is the reflex that hides bugs, so a new retry loop has to answer three questions in the code, not in the reviewer's head. -- **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Use the shared primitive rather than a hand-rolled `sleep`: `moq_net::retry::Backoff` in Rust, `Retry.Backoff` from `@moq/net` in TypeScript. +- **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Use the shared primitive rather than a hand-rolled `sleep`: `kio::time::Backoff` in Rust, `Retry.Backoff` from `@moq/net` in TypeScript. - **When does it stop?** A deadline or an attempt budget, and that budget is what ends the loop. Unlimited retries belong only to a supervisor whose job is to outlive an outage (a reconnecting publisher, a cluster peer, a listener), where the escalating delay is what keeps a permanently-dead target cheap. - **Who owns the budget?** Exactly one layer. An outer supervisor that rebuilds an inner retry loop resets its backoff to the initial delay, so the escalation never happens and a fixed-interval hammer wears an exponential costume. Watch the inner loop's terminal signal instead of restarting it. **Don't classify errors as retryable.** It is tempting to give an error type an `is_retryable()` and skip the wait when the answer is no. Resist it: deciding whether a failure is permanent means guessing, the guess has to stay correct as every wrapped error type evolves, and getting it wrong either strands a connection a retry would have recovered or hammers a dead one. The budget already bounds the damage; the only thing classification buys is surfacing a config error sooner. -The exception is an answer a peer actually gave, where the protocol defines the meaning. An HTTP status is the one we have: `moq_net::retry::status_retryable` covers `408`, `429`, `502`, `503`, and `504`, and `moq_native::Error::status` / `moq_hls::Error::status` report the status a server sent so a caller can consult it. That is reading a response, not inferring intent from a failure. +The exception is an answer a peer actually gave, where the protocol defines the meaning. An HTTP status is the one we have: `moq_native::Error::status` and `moq_hls::Error::status` report the status a server sent, and each crate decides what to do with it (`408`, `429`, `502`, `503`, and `504` are worth another try). That is reading a response, not inferring intent from a failure. Resetting a backoff is its own claim: only after an outcome that says the earlier failures no longer describe reality (a session that stayed healthy, a request that succeeded, a changed destination). Resetting on an attempt that failed immediately turns escalation into a tight loop. diff --git a/Cargo.lock b/Cargo.lock index 7efcd7ad4d..0b435974a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4013,6 +4013,7 @@ name = "kio" version = "0.5.4" dependencies = [ "loom", + "rand 0.10.2", "smallvec", "tokio", "web-async", @@ -4345,6 +4346,7 @@ dependencies = [ "dispatch2", "fixed-resample", "hang", + "kio", "moq-mux", "moq-net", "objc2", @@ -4578,6 +4580,7 @@ dependencies = [ "humantime", "humantime-serde", "jni 0.22.4", + "kio", "moq-net", "noq-proto", "notify", @@ -4663,6 +4666,7 @@ dependencies = [ "humantime", "humantime-serde", "jsonwebtoken", + "kio", "moq-native", "moq-net", "moq-stats", @@ -4720,6 +4724,7 @@ dependencies = [ "futures", "hang", "hmac 0.13.0", + "kio", "moq-mux", "moq-native", "moq-net", diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index f5a2760fc7..4c9db5079b 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -117,13 +117,12 @@ Cluster peers must authenticate to each other: See [Authentication](/bin/relay/auth) for the full setup. -A peer that is merely unreachable is redialed indefinitely, with exponential backoff and jitter so a -restarting cluster doesn't reconnect in lockstep. A peer that *rejects* us is not: a bad token, an -ALPN neither side speaks, or a URL this build can't dial produces the same failure on every dial, so -the relay logs `cluster peer rejected us` and gives up on that peer rather than hiding the cause -behind a warning every few seconds. After fixing the cause, the peer is dialed again once it leaves -and re-enters the dial set (it stops and resumes gossiping, or drops out of and back into the -`connect_api` list); a static `connect` entry needs a relay restart. +Peers are redialed indefinitely, with exponential backoff and jitter so a restarting cluster doesn't +reconnect in lockstep. That includes a peer that rejects us: a bad token logs `cluster peer error; +will retry` on every attempt rather than giving up, so watch for a peer that never reaches +`cluster peer session closed`. The delay escalates to five minutes, which is what keeps a +permanently-rejected peer cheap rather than noisy. A session that stays up for ten seconds is +treated as healthy and clears the escalation, so a peer that comes back redials promptly. ## Migration from older configs diff --git a/js/net/src/retry.ts b/js/net/src/retry.ts index d6748e87e7..f4033f40ee 100644 --- a/js/net/src/retry.ts +++ b/js/net/src/retry.ts @@ -35,9 +35,10 @@ export type BackoffProps = { max?: DOMHighResTimeStamp; /** - * How long to keep retrying before giving up, in milliseconds (default: 300000, five minutes). - * Measured from the first delay after a {@link Backoff.reset}. Zero retries forever, which only - * belongs in a supervisor whose job is to outlive an outage. + * How long to keep trying before giving up, in milliseconds (default: 300000, five minutes). + * Measured from construction or the last {@link Backoff.reset}, and covering the attempts + * themselves rather than just the waits between them. Zero retries forever, which only belongs + * in a supervisor whose job is to outlive an outage. */ timeout?: DOMHighResTimeStamp; }; @@ -65,7 +66,7 @@ export class Backoff { /** The current window's upper bound, grown per failure. */ #window: DOMHighResTimeStamp; - /** When the budget runs out, or undefined while the sequence hasn't started. */ + /** When the budget runs out, or undefined when there isn't one. */ #deadline: DOMHighResTimeStamp | undefined; constructor(props?: BackoffProps) { @@ -75,19 +76,23 @@ export class Backoff { this.#max = props?.max ?? DEFAULT_MAX; this.#timeout = props?.timeout ?? DEFAULT_TIMEOUT; this.#window = this.#initial; + this.#deadline = this.#budget(); + } + + /** When the budget runs out, or undefined when there isn't one. */ + #budget(): DOMHighResTimeStamp | undefined { + return this.#timeout > 0 ? performance.now() + this.#timeout : undefined; } /** How long to wait before the next attempt, or undefined once the budget is spent. */ delay(): DOMHighResTimeStamp | undefined { let remaining = Number.POSITIVE_INFINITY; - if (this.#timeout > 0) { - const now = performance.now(); - // The first delay of a sequence starts the clock, so a loop that ran healthy for hours - // still gets its full budget when it finally does fail. - this.#deadline ??= now + this.#timeout; - - remaining = this.#deadline - now; + if (this.#deadline !== undefined) { + // Out of budget. The clock runs from construction or the last `reset`, so it covers the + // attempts as well as the waits between them: a caller whose every attempt hangs for most + // of the budget would otherwise outlive it many times over. + remaining = this.#deadline - performance.now(); if (remaining <= 0) return undefined; } @@ -110,6 +115,6 @@ export class Backoff { */ reset(): void { this.#window = this.#initial; - this.#deadline = undefined; + this.#deadline = this.#budget(); } } diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index ea3dee542f..cdb4232bc5 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,7 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions -- **Retries go through `moq_net::retry`** (root Retries explains the why). `retry::Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `delay()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `retry::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm, and don't add an `is_retryable()` to an error type: the budget is what stops a loop. The one thing worth reading off a failure is a status a peer actually sent, via `retry::status_retryable` and the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. +- **Retries go through `kio::time::Backoff`** (root Retries explains the why). It's the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `delay()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `kio::time::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. It lives in kio rather than moq-net because a backoff schedule is a utility, not part of the wire layer's surface. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm, and don't add an `is_retryable()` to an error type: the budget is what stops a loop. The one thing worth reading off a failure is a status a peer actually sent, via the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/kio/Cargo.toml b/rs/kio/Cargo.toml index 079f3841ef..cd704cd403 100644 --- a/rs/kio/Cargo.toml +++ b/rs/kio/Cargo.toml @@ -13,12 +13,15 @@ keywords = ["async", "producer", "consumer", "state", "sync"] categories = ["asynchronous", "concurrency"] [features] -# Opt-in poll-driven wall-clock deadlines, backed by `web-async` (tokio on native, -# wasmtimer in the browser). Off by default so kio stays runtime-free. -time = ["dep:web-async"] +# Opt-in poll-driven wall-clock deadlines and retry schedules, backed by `web-async` +# (tokio on native, wasmtimer in the browser). Off by default so kio stays runtime-free. +# `rand` is here for `Backoff`'s jitter; a wasm consumer picks its own getrandom +# backend in the leaf binary, same as moq-net. +time = ["dep:web-async", "dep:rand"] tokio = ["dep:tokio"] [dependencies] +rand = { version = "0.10.1", optional = true } smallvec = "1.15" tokio = { workspace = true, features = ["time"], optional = true } web-async = { workspace = true, optional = true } diff --git a/rs/kio/src/time.rs b/rs/kio/src/time.rs index eaa8e4ec2e..0874ea2569 100644 --- a/rs/kio/src/time.rs +++ b/rs/kio/src/time.rs @@ -1,4 +1,7 @@ -//! Poll-driven wall-clock deadlines. +//! Poll-driven wall-clock deadlines and retry schedules. +//! +//! [`Deadline`] is a single instant to poll on. [`Backoff`] is the escalating delay a loop waits +//! between attempts at something that failed, with a budget that eventually stops it. //! //! Behind the `time` feature. Built on [`web_async::time`], which is `tokio::time` on //! native and `wasmtimer` in the browser, so the rest of kio stays runtime-free. @@ -10,6 +13,8 @@ use std::{pin::Pin, task::Poll}; +use rand::RngExt; + /// Re-exported from `web-async`, so a major bump of that crate is a breaking change /// for these types. pub use web_async::time::{Duration, Instant}; @@ -123,6 +128,165 @@ impl std::fmt::Debug for Deadline { } } +/// How long to wait between attempts, and how long to keep making them. +/// +/// The defaults suit a long-lived connection: a second before the first retry, doubling to a +/// half-minute ceiling, giving up after five minutes. A one-shot request wants a much smaller +/// [`timeout`](Self::timeout); a supervisor that must never stop wants a zero one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct Config { + /// Delay before the first retry. + pub initial: Duration, + + /// Multiplier applied to the delay after each failure. + pub multiplier: u32, + + /// Ceiling on the delay, however many failures have piled up. + pub max: Duration, + + /// How long to keep trying before giving up, measured from construction or the last + /// [`reset`](Backoff::reset). This covers the attempts themselves, not just the waits between + /// them, so a caller whose every attempt hangs still gives up on schedule. [`Duration::ZERO`] + /// retries forever, which only belongs in a supervisor whose job is to outlive an outage. + pub timeout: Duration, +} + +impl Default for Config { + fn default() -> Self { + Self { + initial: Duration::from_secs(1), + multiplier: 2, + max: Duration::from_secs(30), + timeout: Duration::from_secs(300), + } + } +} + +/// The escalating delay a loop waits between attempts at something that failed. +/// +/// Capped exponential backoff with jitter and a give-up budget. It answers *when* to try again and, +/// through the budget, when to stop. It deliberately has no opinion on *whether* a given failure is +/// worth repeating: deciding that means guessing, and a wrong guess either strands something a retry +/// would have recovered or hammers something already dead. The budget bounds the damage instead. +/// +/// Each delay is drawn from the top half of the current window (equal jitter), so a fleet that fails +/// together doesn't retry together, while still waiting at least half the escalating delay. The +/// window doubles per failure up to [`Config::max`], and [`Config::timeout`] bounds the whole +/// sequence. +/// +/// Call [`sleep`](Self::sleep) (or [`delay`](Self::delay), if the caller owns the waiting) after each +/// failure and [`reset`](Self::reset) after a success worth trusting. Nothing else may own a competing +/// schedule for the same operation: an outer supervisor that rebuilds an inner loop restarts its +/// backoff at the initial delay and the escalation never happens. +#[derive(Debug)] +pub struct Backoff { + config: Config, + + /// The current window's upper bound, doubled per failure. + window: Duration, + + /// When the budget runs out, or `None` when there isn't one. + deadline: Option, +} + +impl Backoff { + /// A backoff following `config`, with a full budget starting now. + pub fn new(config: Config) -> Self { + Self { + window: config.initial, + deadline: Self::deadline(&config), + config, + } + } + + /// When the budget runs out, or `None` when there isn't one. + /// + /// An unlimited budget never reads the clock, which is what lets a blocking thread with its own + /// [`std::time::Instant`] bookkeeping drive this too. + fn deadline(config: &Config) -> Option { + match config.timeout.is_zero() { + true => None, + // An unrepresentable deadline is treated as no deadline. + false => Instant::now().checked_add(config.timeout), + } + } + + /// How long to wait before the next attempt, or `None` once the budget is spent. + /// + /// For callers that do their own waiting (a blocking thread, a poll loop with other arms). + /// Everything else wants [`sleep`](Self::sleep). + pub fn delay(&mut self) -> Option { + let mut remaining = None; + if let Some(deadline) = self.deadline { + let now = Instant::now(); + + // Out of budget. The clock runs from construction or the last `reset`, so it covers the + // attempts as well as the waits between them: a caller whose every attempt hangs for + // most of the budget would otherwise outlive it many times over. + if now >= deadline { + return None; + } + remaining = deadline.checked_duration_since(now); + } + + let delay = self.jitter(self.window); + self.window = self + .window + .saturating_mul(self.config.multiplier.max(1)) + .min(self.config.max); + + // Never sleep past the deadline: the budget says how long to keep retrying, so overshooting + // it by a whole window would spend more than the caller asked for and skip the attempt that + // still fit. A truncated final delay is the point, not a rounding error. + Some(match remaining { + Some(remaining) => delay.min(remaining), + None => delay, + }) + } + + /// Wait out the next delay, returning `false` once the budget is spent. + /// + /// A `false` means stop retrying: the caller should surface the failure that got it here rather + /// than loop again. + pub async fn sleep(&mut self) -> bool { + let Some(delay) = self.delay() else { return false }; + web_async::time::sleep(delay).await; + true + } + + /// Start over: the next delay is [`Config::initial`] again and the budget is full. + /// + /// Only call this after an outcome that says the earlier failures no longer describe reality: a + /// session that stayed up, a request that succeeded, a changed destination. Resetting on an + /// attempt that failed immediately turns the escalation into a tight loop. + pub fn reset(&mut self) { + self.window = self.config.initial; + self.deadline = Self::deadline(&self.config); + } + + /// Draw the actual delay from the top half of `window`, so peers that failed together spread out. + fn jitter(&self, window: Duration) -> Duration { + let half = window / 2; + + // A window past ~584 years holds more nanoseconds than a `u64`, and truncating one would + // hand `random_range` an empty range to panic on. `max` is caller-configurable (a humantime + // string on the CLI), so saturate rather than trust it to be sane. + let span = u64::try_from(half.as_nanos()).unwrap_or(u64::MAX); + if span == 0 { + return window; + } + + half.saturating_add(Duration::from_nanos(rand::rng().random_range(0..span))) + } +} + +impl Default for Backoff { + fn default() -> Self { + Self::new(Config::default()) + } +} + #[cfg(all(test, not(loom)))] mod tests { use std::task::Waker; @@ -221,4 +385,131 @@ mod tests { assert!(Instant::now() >= at, "returned before the re-armed deadline"); } + + fn config() -> Config { + Config { + initial: Duration::from_secs(1), + multiplier: 2, + max: Duration::from_secs(8), + timeout: Duration::ZERO, + } + } + + /// The window doubles per failure and stops at the cap, and jitter keeps every draw inside the + /// top half of its window. + #[tokio::test(start_paused = true)] + async fn escalates_to_the_cap_within_the_jitter_band() { + let mut backoff = Backoff::new(config()); + + for expected in [1, 2, 4, 8, 8, 8].map(Duration::from_secs) { + let delay = backoff.delay().expect("unlimited budget"); + assert!( + delay >= expected / 2 && delay <= expected, + "{delay:?} outside the jitter band for {expected:?}" + ); + } + } + + /// Two backoffs with the same settings must not step in lockstep, or a fleet that failed + /// together retries together. + #[tokio::test(start_paused = true)] + async fn jitter_separates_identical_schedules() { + let mut a = Backoff::new(config()); + let mut b = Backoff::new(config()); + + // One shared draw could collide by chance; a run of them colliding means no jitter at all. + let differs = (0..8).any(|_| a.delay() != b.delay()); + assert!(differs, "identical backoffs produced identical delays"); + } + + /// `max` comes from a caller-supplied humantime string, so an absurd one has to degrade rather + /// than panic: a window past ~584 years has more nanoseconds than the jitter sample can hold. + #[tokio::test(start_paused = true)] + async fn an_absurd_window_does_not_panic() { + let mut backoff = Backoff::new(Config { + initial: Duration::new(36_893_488_147, 419_103_232), + max: Duration::MAX, + ..config() + }); + + let delay = backoff.delay().expect("unlimited budget"); + assert!( + delay >= Duration::new(18_446_744_073, 709_551_616), + "{delay:?} below half" + ); + } + + #[tokio::test(start_paused = true)] + async fn reset_returns_to_the_initial_window() { + let mut backoff = Backoff::new(config()); + for _ in 0..4 { + backoff.delay(); + } + + backoff.reset(); + let delay = backoff.delay().expect("unlimited budget"); + assert!(delay <= Duration::from_secs(1), "{delay:?} did not return to initial"); + } + + /// An initial delay longer than the whole budget must not sleep past it: the budget is the + /// promise, and one oversized window would blow through it before a single retry lands. + #[tokio::test(start_paused = true)] + async fn a_delay_never_outlives_the_budget() { + let mut backoff = Backoff::new(Config { + initial: Duration::from_secs(60), + timeout: Duration::from_millis(50), + ..config() + }); + + let delay = backoff.delay().expect("budget available"); + assert!(delay <= Duration::from_millis(50), "{delay:?} outlived the budget"); + } + + /// The budget is a wall-clock deadline over the whole sequence, not a per-attempt one, and the + /// sequence lands on it rather than overshooting by a whole window. + #[tokio::test(start_paused = true)] + async fn gives_up_once_the_budget_is_spent() { + let timeout = Duration::from_secs(10); + let mut backoff = Backoff::new(Config { timeout, ..config() }); + + let started = tokio::time::Instant::now(); + while let Some(delay) = backoff.delay() { + tokio::time::sleep(delay).await; + assert!(started.elapsed() < Duration::from_secs(60), "budget never ran out"); + } + + // Tokio's paused clock rounds each sleep to its timer granularity, so the sequence can land a + // hair either side of the deadline it aimed for. + let elapsed = started.elapsed(); + assert!( + elapsed >= timeout - Duration::from_millis(10), + "gave up after only {elapsed:?}" + ); + assert!( + elapsed < timeout + config().max, + "overshot the budget by a whole window: {elapsed:?}" + ); + } + + /// A zero timeout is the supervisor case: keep retrying however long the outage lasts. + #[tokio::test(start_paused = true)] + async fn a_zero_timeout_never_gives_up() { + let mut backoff = Backoff::new(config()); + for _ in 0..64 { + assert!(backoff.sleep().await); + } + } + + /// The budget covers the retry sequence, so a reset after a healthy stretch buys a fresh one. + #[tokio::test(start_paused = true)] + async fn reset_refills_the_budget() { + let mut backoff = Backoff::new(Config { + timeout: Duration::from_secs(10), + ..config() + }); + + while backoff.sleep().await {} + backoff.reset(); + assert!(backoff.sleep().await, "reset did not refill the budget"); + } } diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 688210210b..11e0ab2fff 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -55,6 +55,7 @@ cpal = { version = "0.18", optional = true } # is the ring buffer and `resampler` its rubato back end; the default # `fft-resampler` stays off, matching the sinc-only stance below. fixed-resample = { version = "0.12", optional = true, default-features = false, features = ["channel", "resampler"] } +kio = { workspace = true, features = ["time"] } hang = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } diff --git a/rs/moq-audio/src/playback/driver.rs b/rs/moq-audio/src/playback/driver.rs index 5a29fe41db..7525d39729 100644 --- a/rs/moq-audio/src/playback/driver.rs +++ b/rs/moq-audio/src/playback/driver.rs @@ -25,12 +25,12 @@ use crate::Error; /// No give-up budget: the engine outlives any one device, and the user plugging a headset back in is /// exactly the external change a retry is waiting for. Unlimited retries are also what keeps this /// clock-free, so the driver thread can stay on [`std::time::Instant`]. -fn retry_backoff() -> moq_net::retry::Backoff { - let mut config = moq_net::retry::Config::default(); +fn retry_backoff() -> kio::time::Backoff { + let mut config = kio::time::Config::default(); config.initial = Duration::from_millis(500); config.max = Duration::from_secs(4); config.timeout = Duration::ZERO; - moq_net::retry::Backoff::new(config) + kio::time::Backoff::new(config) } /// Problems tolerated in [`ERROR_WINDOW`] before the stream is rebuilt. @@ -426,7 +426,7 @@ struct Driver { /// from one the live stream raised. generation: u64, /// Escalating delay before reopening a device that would not start. - retry: moq_net::retry::Backoff, + retry: kio::time::Backoff, /// When a failed start may be retried, and what the command wait times out /// against. `None` while the stream is healthy. retry_at: Option, diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index a5131dbab3..552b397651 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -18,6 +18,15 @@ impl std::fmt::Display for SequenceKind { } } +/// Whether an HTTP response status means "ask again later". +/// +/// A response that arrived is the server's answer, and only this narrow set invites another +/// attempt: request timeout, rate limit, and the gateway/overload statuses. Every other status, +/// `404` and `403` included, is settled. +pub(crate) fn status_retryable(status: u16) -> bool { + matches!(status, 408 | 429 | 502 | 503 | 504) +} + /// Errors produced by the HLS <-> MoQ gateway (import and export). #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] @@ -127,7 +136,7 @@ pub enum Error { impl Error { /// The HTTP status the origin answered with, if it answered with one at all. /// - /// The import loop reads this through [`moq_net::retry::status_retryable`]: a `503` on a playlist + /// The import loop reads this through [`status_retryable`]: a `503` on a playlist /// fetch is worth another pass, a `404` is the origin's settled answer. Nothing else here is /// classified; a failure with no status falls through to the backoff budget. pub fn status(&self) -> Option { diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index c39a966252..6cee23b004 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -42,12 +42,12 @@ pub use rendition::{Kind, Rendition}; /// broadcast closing is what ends the wait, and a relay-side broadcast outlives its publisher's /// session, so any deadline here is a window in which a publisher outage leaves the broadcaster /// permanently empty with nothing to recover it. -fn catalog_backoff() -> moq_net::retry::Backoff { - let mut config = moq_net::retry::Config::default(); +fn catalog_backoff() -> kio::time::Backoff { + let mut config = kio::time::Config::default(); config.initial = Duration::from_millis(250); config.max = Duration::from_secs(5); config.timeout = Duration::ZERO; - moq_net::retry::Backoff::new(config) + kio::time::Backoff::new(config) } /// Export tuning shared across renditions. diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 6e138f7e11..84ec82adb2 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -36,11 +36,11 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// outage the caller should hear about, not one the import should paper over indefinitely while /// publishing nothing. The ceiling is lower than the default, since a live playlist window is /// measured in seconds and a longer wait would blow past it anyway. -fn error_backoff() -> moq_net::retry::Backoff { - let mut config = moq_net::retry::Config::default(); +fn error_backoff() -> kio::time::Backoff { + let mut config = kio::time::Config::default(); config.initial = Duration::from_secs(1); config.max = Duration::from_secs(10); - moq_net::retry::Backoff::new(config) + kio::time::Backoff::new(config) } /// How far back from the live edge to start when (re-)anchoring to a playlist window. @@ -96,11 +96,14 @@ struct StepOutcome { wrote_segments: usize, /// Target segment duration (in seconds) from the playlist, if known. target_duration: Option, - /// The last rendition failure of this step, if any. + /// A rendition failure from this step, if any. /// /// [`OnError::Warn`] keeps the other renditions going after one fails, so a step can report /// `Ok` having imported nothing at all. The loop needs to tell that apart from a quiet playlist /// with no new segments, or it treats a permanently broken source as steady progress. + /// + /// When several renditions fail, this is the one whose failure another pass could still clear, + /// so a single permanently-dead variant doesn't end an import the others could still serve. failed: Option, } @@ -633,11 +636,7 @@ impl Import { } // A status the origin actually sent is its answer: a 404 playlist is not going to // become a 200 on the next pass. Everything else rides the backoff budget. - Err(err) - if err - .status() - .is_some_and(|status| !moq_net::retry::status_retryable(status)) => - { + Err(err) if err.status().is_some_and(|status| !crate::status_retryable(status)) => { return Err(err); } Err(err) => { @@ -702,7 +701,15 @@ impl Import { // drop the rest or abort the whole step. OnError::Warn => { warn!(label = %track.label, %err, "rendition import step failed, will retry"); - failed = Some(err); + // Prefer a failure another pass could still clear. The import is worth + // continuing as long as *any* rendition might come back, even when another is + // permanently gone, so one dead variant must not end an import the rest could + // still serve. Keeping whichever error came last instead would make the + // outcome depend on rendition order. + let recoverable = err.status().is_none_or(|status| crate::status_retryable(status)); + if recoverable || failed.is_none() { + failed = Some(err); + } } }, } diff --git a/rs/moq-hls/src/lib.rs b/rs/moq-hls/src/lib.rs index 5b97dd6cda..1708142543 100644 --- a/rs/moq-hls/src/lib.rs +++ b/rs/moq-hls/src/lib.rs @@ -25,6 +25,7 @@ pub mod import; #[cfg(feature = "server")] pub mod server; +pub(crate) use error::status_retryable; pub use error::*; #[cfg(feature = "server")] pub use server::Server; diff --git a/rs/moq-native/Cargo.toml b/rs/moq-native/Cargo.toml index 70babdf4c7..5460607fce 100644 --- a/rs/moq-native/Cargo.toml +++ b/rs/moq-native/Cargo.toml @@ -46,6 +46,7 @@ hex = "0.4" humantime = "2.3" humantime-serde = "1.1" +kio = { workspace = true, features = ["time"] } moq-net = { workspace = true } # iroh runs on noq but re-exports only the ControllerFactory trait, not the concrete # congestion configs. Version matched to the copy iroh pulls in. diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index fa244e6303..1b05c51631 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -1,5 +1,14 @@ use std::sync::Arc; +/// Whether an HTTP response status means "ask again later". +/// +/// A response that arrived is the server's answer, and only this narrow set invites another +/// attempt: request timeout, rate limit, and the gateway/overload statuses. Every other status, +/// `404` and `403` included, is settled. +pub(crate) fn status_retryable(status: u16) -> bool { + matches!(status, 408 | 429 | 502 | 503 | 504) +} + /// Errors produced while configuring or establishing native MoQ connections. /// /// Backend-specific failures live in per-backend error types ([`crate::tls::Error`], @@ -150,7 +159,7 @@ impl Error { /// /// `None` covers everything else: a dial that never got a response, a QUIC handshake that /// failed, a URL we couldn't parse. Only a status the peer actually sent shows up here, and - /// [`moq_net::retry::status_retryable`] is what decides whether it invites another attempt. This + /// [`status_retryable`] is what decides whether it invites another attempt. This /// deliberately does not try to say whether some *other* kind of failure is worth retrying; /// that's a guess, and the caller's backoff budget bounds it instead. pub fn status(&self) -> Option { @@ -160,11 +169,7 @@ impl Error { // over QUIC alongside a dead WebSocket is still just a failed dial. #[cfg(feature = "websocket")] Self::TransportRace { quic, websocket } => match (quic.status(), websocket.status()) { - (Some(quic), Some(websocket)) - if !moq_net::retry::status_retryable(quic) && !moq_net::retry::status_retryable(websocket) => - { - Some(quic) - } + (Some(quic), Some(websocket)) if !status_retryable(quic) && !status_retryable(websocket) => Some(quic), _ => None, }, diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index ab0057c956..e333a1ff25 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -433,7 +433,7 @@ fn classify_client_error(err: &web_transport_noq::ClientError) -> Option Option { match err { diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index 9199999e93..955be8729d 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -509,7 +509,7 @@ fn classify_client_error(err: &web_transport_quiche::ClientError) -> Option Option { match err { diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 66c089e1fd..d57322935d 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -446,7 +446,7 @@ fn classify_client_error(err: &web_transport_quinn::ClientError) -> Option Option { match err { @@ -728,14 +728,14 @@ mod tests { for status in [400, 404, 405, 410, 501] { assert_eq!(connect_rejected(status).status(), Some(status)); assert!( - !moq_net::retry::status_retryable(status), + !crate::error::status_retryable(status), "{status} should stop the reconnect loop" ); } for status in [408, 429, 502, 503, 504] { assert_eq!(connect_rejected(status).status(), Some(status)); - assert!(moq_net::retry::status_retryable(status), "{status} should be retried"); + assert!(crate::error::status_retryable(status), "{status} should be retried"); } // Auth is peeled off into its own variant before reaching the generic client arm. diff --git a/rs/moq-native/src/reconnect.rs b/rs/moq-native/src/reconnect.rs index e5fa22f70c..6e6bb8d53b 100644 --- a/rs/moq-native/src/reconnect.rs +++ b/rs/moq-native/src/reconnect.rs @@ -73,7 +73,7 @@ impl Default for Backoff { } } -impl From<&Backoff> for moq_net::retry::Config { +impl From<&Backoff> for kio::time::Config { fn from(backoff: &Backoff) -> Self { let mut config = Self::default(); config.initial = backoff.initial; @@ -204,7 +204,7 @@ impl Reconnect { url: Url, backoff: Backoff, ) -> crate::Result<()> { - let mut retry = moq_net::retry::Backoff::new((&backoff).into()); + let mut retry = kio::time::Backoff::new((&backoff).into()); let mut last_error: Option = None; loop { @@ -261,7 +261,7 @@ impl Reconnect { return Err(err); } if let Some(status) = err.status() - && !moq_net::retry::status_retryable(status) + && !crate::error::status_retryable(status) { return Err(err); } diff --git a/rs/moq-net/src/lib.rs b/rs/moq-net/src/lib.rs index ece3e21c9e..25cbd48991 100644 --- a/rs/moq-net/src/lib.rs +++ b/rs/moq-net/src/lib.rs @@ -84,7 +84,6 @@ mod setup; mod util; mod version; -pub mod retry; pub mod stats; pub use client::*; diff --git a/rs/moq-net/src/retry.rs b/rs/moq-net/src/retry.rs deleted file mode 100644 index 2c94e8d8ab..0000000000 --- a/rs/moq-net/src/retry.rs +++ /dev/null @@ -1,322 +0,0 @@ -//! The retry schedule shared by every loop that re-attempts a failed operation. -//! -//! [`Backoff`] answers *when* to try again, and its budget is what ends a loop. There is deliberately -//! no counterpart answering *whether* a given error is worth repeating: a transport layer can't tell -//! a permanent failure from a temporary one without guessing, and a wrong guess either strands a -//! recoverable connection or hammers a dead one. The budget bounds the damage instead. -//! -//! The one exception is [`status_retryable`], where a peer sent an HTTP status whose meaning the -//! protocol defines. That's reading an answer, not inferring one. -//! -//! ```no_run -//! # async fn example() -> Result<(), moq_net::Error> { -//! # async fn attempt() -> Result<(), moq_net::Error> { Ok(()) } -//! let mut backoff = moq_net::retry::Backoff::default(); -//! loop { -//! match attempt().await { -//! Ok(()) => return Ok(()), -//! // Out of budget: surface the failure instead of looping on it. -//! Err(err) if !backoff.sleep().await => return Err(err), -//! Err(_) => continue, -//! } -//! } -//! # } -//! ``` - -use kio::time::{Duration, Instant}; -use rand::RngExt; - -/// Whether an HTTP response status means "ask again later". -/// -/// A response that arrived is the server's answer, and only this narrow set invites another attempt: -/// request timeout, rate limit, and the gateway/overload statuses. Every other status, `404` and -/// `403` included, is settled. A request that got *no* response is a transport failure and doesn't -/// come through here. -pub fn status_retryable(status: u16) -> bool { - matches!(status, 408 | 429 | 502 | 503 | 504) -} - -/// How long to wait between attempts, and how long to keep making them. -/// -/// The defaults suit a long-lived connection: a second before the first retry, doubling to a -/// half-minute ceiling, giving up after five minutes. A one-shot request wants a much smaller -/// [`timeout`](Self::timeout); a supervisor that must never stop wants a zero one. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[non_exhaustive] -pub struct Config { - /// Delay before the first retry. - pub initial: Duration, - - /// Multiplier applied to the delay after each failure. - pub multiplier: u32, - - /// Ceiling on the delay, however many failures have piled up. - pub max: Duration, - - /// How long to keep retrying before giving up, measured from the first delay after a - /// [`reset`](Backoff::reset). [`Duration::ZERO`] retries forever, which only belongs in a - /// supervisor whose job is to outlive an outage. - pub timeout: Duration, -} - -impl Default for Config { - fn default() -> Self { - Self { - initial: Duration::from_secs(1), - multiplier: 2, - max: Duration::from_secs(30), - timeout: Duration::from_secs(300), - } - } -} - -/// A capped exponential backoff with jitter and a give-up budget. -/// -/// Each delay is drawn from the top half of the current window (equal jitter), so a fleet that fails -/// together doesn't retry together, while still waiting at least half the escalating delay. The -/// window doubles per failure up to [`Config::max`], and [`Config::timeout`] bounds the whole -/// sequence. -/// -/// Call [`sleep`](Self::sleep) (or [`delay`](Self::delay), if the caller owns the waiting) after each -/// failure and [`reset`](Self::reset) after a success worth trusting. Nothing else may own a competing -/// schedule for the same operation: an outer supervisor that rebuilds an inner loop restarts its -/// backoff at the initial delay and the escalation never happens. -#[derive(Debug)] -pub struct Backoff { - config: Config, - - /// The current window's upper bound, doubled per failure. - window: Duration, - - /// When the budget runs out, or `None` while the sequence hasn't started (or never expires). - deadline: Option, -} - -impl Backoff { - /// A backoff following `config`, with a full budget. - pub fn new(config: Config) -> Self { - Self { - window: config.initial, - config, - deadline: None, - } - } - - /// How long to wait before the next attempt, or `None` once the budget is spent. - /// - /// For callers that do their own waiting (a blocking thread, a poll loop with other arms). - /// Everything else wants [`sleep`](Self::sleep). - pub fn delay(&mut self) -> Option { - // An unlimited budget never reads the clock, which is what lets a blocking thread with its - // own [`std::time::Instant`] bookkeeping drive this too. - let mut remaining = None; - if !self.config.timeout.is_zero() { - let now = Instant::now(); - // The first delay of a sequence starts the clock. Deferred to here rather than to - // `new`/`reset` so a loop that runs healthy for hours still gets its full budget when it - // finally does fail. An unrepresentable deadline is treated as no deadline. - let deadline = match self.deadline { - Some(deadline) => Some(deadline), - None => { - self.deadline = now.checked_add(self.config.timeout); - self.deadline - } - }; - - if let Some(deadline) = deadline { - // Started already: stop once the budget is gone. - if now >= deadline { - return None; - } - remaining = Some(deadline - now); - } - } - - let delay = self.jitter(self.window); - self.window = self - .window - .saturating_mul(self.config.multiplier.max(1)) - .min(self.config.max); - - // Never sleep past the deadline: the budget says how long to keep retrying, so overshooting - // it by a whole window would spend more than the caller asked for and skip the attempt that - // still fit. A truncated final delay is the point, not a rounding error. - Some(match remaining { - Some(remaining) => delay.min(remaining), - None => delay, - }) - } - - /// Wait out the next delay, returning `false` once the budget is spent. - /// - /// A `false` means stop retrying: the caller should surface the failure that got it here rather - /// than loop again. - pub async fn sleep(&mut self) -> bool { - let Some(delay) = self.delay() else { return false }; - web_async::time::sleep(delay).await; - true - } - - /// Start over: the next delay is [`Config::initial`] again and the budget is full. - /// - /// Only call this after an outcome that says the earlier failures no longer describe reality: a - /// session that stayed up, a request that succeeded, a changed destination. Resetting on an - /// attempt that failed immediately turns the escalation into a tight loop. - pub fn reset(&mut self) { - self.window = self.config.initial; - self.deadline = None; - } - - /// Draw the actual delay from the top half of `window`, so peers that failed together spread out. - fn jitter(&self, window: Duration) -> Duration { - let half = window / 2; - - // A window past ~584 years holds more nanoseconds than a `u64`, and truncating one would - // hand `random_range` an empty range to panic on. `max` is caller-configurable (a humantime - // string on the CLI), so saturate rather than trust it to be sane. - let span = u64::try_from(half.as_nanos()).unwrap_or(u64::MAX); - if span == 0 { - return window; - } - - half.saturating_add(Duration::from_nanos(rand::rng().random_range(0..span))) - } -} - -impl Default for Backoff { - fn default() -> Self { - Self::new(Config::default()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn config() -> Config { - Config { - initial: Duration::from_secs(1), - multiplier: 2, - max: Duration::from_secs(8), - timeout: Duration::ZERO, - } - } - - /// The window doubles per failure and stops at the cap, and jitter keeps every draw inside the - /// top half of its window. - #[tokio::test(start_paused = true)] - async fn escalates_to_the_cap_within_the_jitter_band() { - let mut backoff = Backoff::new(config()); - - for expected in [1, 2, 4, 8, 8, 8].map(Duration::from_secs) { - let delay = backoff.delay().expect("unlimited budget"); - assert!( - delay >= expected / 2 && delay <= expected, - "{delay:?} outside the jitter band for {expected:?}" - ); - } - } - - /// Two backoffs with the same settings must not step in lockstep, or a fleet that failed - /// together retries together. - #[tokio::test(start_paused = true)] - async fn jitter_separates_identical_schedules() { - let mut a = Backoff::new(config()); - let mut b = Backoff::new(config()); - - // One shared draw could collide by chance; a run of them colliding means no jitter at all. - let differs = (0..8).any(|_| a.delay() != b.delay()); - assert!(differs, "identical backoffs produced identical delays"); - } - - /// `max` comes from a caller-supplied humantime string, so an absurd one has to degrade rather - /// than panic: a window past ~584 years has more nanoseconds than the jitter sample can hold. - #[tokio::test(start_paused = true)] - async fn an_absurd_window_does_not_panic() { - let mut backoff = Backoff::new(Config { - initial: Duration::new(36_893_488_147, 419_103_232), - max: Duration::MAX, - ..config() - }); - - let delay = backoff.delay().expect("unlimited budget"); - assert!( - delay >= Duration::new(18_446_744_073, 709_551_616), - "{delay:?} below half" - ); - } - - #[tokio::test(start_paused = true)] - async fn reset_returns_to_the_initial_window() { - let mut backoff = Backoff::new(config()); - for _ in 0..4 { - backoff.delay(); - } - - backoff.reset(); - let delay = backoff.delay().expect("unlimited budget"); - assert!(delay <= Duration::from_secs(1), "{delay:?} did not return to initial"); - } - - /// An initial delay longer than the whole budget must not sleep past it: the budget is the - /// promise, and one oversized window would blow through it before a single retry lands. - #[tokio::test(start_paused = true)] - async fn a_delay_never_outlives_the_budget() { - let mut backoff = Backoff::new(Config { - initial: Duration::from_secs(60), - timeout: Duration::from_millis(50), - ..config() - }); - - let delay = backoff.delay().expect("budget available"); - assert!(delay <= Duration::from_millis(50), "{delay:?} outlived the budget"); - } - - /// The budget is a wall-clock deadline over the whole sequence, not a per-attempt one, and the - /// sequence lands on it rather than overshooting by a whole window. - #[tokio::test(start_paused = true)] - async fn gives_up_once_the_budget_is_spent() { - let timeout = Duration::from_secs(10); - let mut backoff = Backoff::new(Config { timeout, ..config() }); - - let started = tokio::time::Instant::now(); - while let Some(delay) = backoff.delay() { - tokio::time::sleep(delay).await; - assert!(started.elapsed() < Duration::from_secs(60), "budget never ran out"); - } - - // Tokio's paused clock rounds each sleep to its timer granularity, so the sequence can land a - // hair either side of the deadline it aimed for. - let elapsed = started.elapsed(); - assert!( - elapsed >= timeout - Duration::from_millis(10), - "gave up after only {elapsed:?}" - ); - assert!( - elapsed < timeout + config().max, - "overshot the budget by a whole window: {elapsed:?}" - ); - } - - /// A zero timeout is the supervisor case: keep retrying however long the outage lasts. - #[tokio::test(start_paused = true)] - async fn a_zero_timeout_never_gives_up() { - let mut backoff = Backoff::new(config()); - for _ in 0..64 { - assert!(backoff.sleep().await); - } - } - - /// The budget covers the retry sequence, so a reset after a healthy stretch buys a fresh one. - #[tokio::test(start_paused = true)] - async fn reset_refills_the_budget() { - let mut backoff = Backoff::new(Config { - timeout: Duration::from_secs(10), - ..config() - }); - - while backoff.sleep().await {} - backoff.reset(); - assert!(backoff.sleep().await, "reset did not refill the budget"); - } -} diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index eefb8f7ea0..3b41be1960 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -50,6 +50,7 @@ http-body = "1" http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-moka", "url-standard"], default-features = false } humantime = "2.3" humantime-serde = "1.1" +kio = { workspace = true, features = ["time"] } jsonwebtoken = "11" moq-native = { workspace = true, default-features = false, features = ["aws-lc-rs", "watch", "tcp"] } moq-net = { workspace = true } diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 54751a3e75..1399cfead7 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -920,10 +920,10 @@ impl Cluster { // A peer is supervised for the life of the relay, so there is no give-up deadline: one that is // unreachable for an hour still has to be redialed when it comes back. Nothing ends this // loop; the escalating delay is what keeps a permanently-dead peer cheap. - let mut config = moq_net::retry::Config::default(); + let mut config = kio::time::Config::default(); config.max = tokio::time::Duration::from_secs(300); config.timeout = tokio::time::Duration::ZERO; - let mut backoff = moq_net::retry::Backoff::new(config); + let mut backoff = kio::time::Backoff::new(config); // Sessions shorter than this are treated as churn: we keep backing off // instead of resetting, otherwise a peer that rejects us instantly would diff --git a/rs/moq-rtmp/Cargo.toml b/rs/moq-rtmp/Cargo.toml index 7d3a88d85b..e55f270cf2 100644 --- a/rs/moq-rtmp/Cargo.toml +++ b/rs/moq-rtmp/Cargo.toml @@ -33,6 +33,7 @@ anyhow = { version = "1", features = ["backtrace"] } byteorder = "1" bytes = "1" futures = "0.3" +kio = { workspace = true, features = ["time"] } hang = { workspace = true } hmac = "0.13" moq-mux = { workspace = true } diff --git a/rs/moq-rtmp/src/server.rs b/rs/moq-rtmp/src/server.rs index 7f9635938e..00b7d2f8ce 100644 --- a/rs/moq-rtmp/src/server.rs +++ b/rs/moq-rtmp/src/server.rs @@ -237,7 +237,7 @@ pub struct Server { /// Escalating delay after a failed `accept`. Lives on the server rather than inside /// [`accept`](Self::accept) so consecutive failures keep escalating across calls, and resets on /// the next connection that does come in. - accept_backoff: moq_net::retry::Backoff, + accept_backoff: kio::time::Backoff, /// While set, `accept` stops asking the listener until this instant. In-flight handshakes keep /// being polled meanwhile: a connection that already got through must not wait out a backoff @@ -253,7 +253,7 @@ impl Server { // The listener is supervised for the process's lifetime, so there is no give-up budget: the // descriptor pressure or firewall rule behind a failed accept clears on its own, and the // next connection resets the escalation. - let mut backoff = moq_net::retry::Config::default(); + let mut backoff = kio::time::Config::default(); backoff.initial = Duration::from_millis(100); backoff.max = Duration::from_secs(5); backoff.timeout = Duration::ZERO; @@ -263,7 +263,7 @@ impl Server { #[cfg(feature = "tls")] tls: None, pending: FuturesUnordered::new(), - accept_backoff: moq_net::retry::Backoff::new(backoff), + accept_backoff: kio::time::Backoff::new(backoff), accept_retry: None, }) } From 18e0cd75a2364035b887059763fa4a50ed5d922e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 20:00:31 -0700 Subject: [PATCH 07/13] fix: sort the new kio deps; don't end an import on a quiet pass `cargo sort` failed CI: the `kio` dependency went in at a convenient anchor rather than its alphabetical slot in moq-audio, moq-relay, and moq-rtmp. The HLS import promoted a step to a failure when no *segments* were written, but a healthy live playlist writes none between segments. A multi-rendition import with one permanently-404 variant and one healthy but quiet one therefore ended outright, which is exactly what `OnError::Warn` exists to prevent. It now keys on renditions that ingested without error, so a step is a failure only when nothing ingested at all. Co-Authored-By: Claude Opus 5 --- rs/moq-audio/Cargo.toml | 2 +- rs/moq-hls/src/import.rs | 23 +++++++++++++++++++---- rs/moq-relay/Cargo.toml | 2 +- rs/moq-rtmp/Cargo.toml | 2 +- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 11e0ab2fff..abaf727a70 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -55,8 +55,8 @@ cpal = { version = "0.18", optional = true } # is the ring buffer and `resampler` its rubato back end; the default # `fft-resampler` stays off, matching the sinc-only stance below. fixed-resample = { version = "0.12", optional = true, default-features = false, features = ["channel", "resampler"] } -kio = { workspace = true, features = ["time"] } hang = { workspace = true } +kio = { workspace = true, features = ["time"] } moq-mux = { workspace = true } moq-net = { workspace = true } # We only use the sinc resampler (Async::new_sinc), so skip the default diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 84ec82adb2..7868190bbe 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -96,6 +96,12 @@ struct StepOutcome { wrote_segments: usize, /// Target segment duration (in seconds) from the playlist, if known. target_duration: Option, + /// Renditions that ingested without error, however many segments each wrote. + /// + /// A live playlist with no new segments yet still counts: that's a healthy rendition with + /// nothing to add this pass, not a failure. Counting segments instead would read a quiet + /// playlist as a dead one. + ok: usize, /// A rendition failure from this step, if any. /// /// [`OnError::Warn`] keeps the other renditions going after one fails, so a step can report @@ -616,13 +622,17 @@ impl Import { let mut backoff = error_backoff(); loop { - // A step that imported nothing while a rendition was failing is a failed pass wearing an - // `Ok`: `step` swallows per-rendition errors so one bad variant doesn't drop the rest. + // A step where *nothing* ingested while a rendition was failing is a failed pass wearing + // an `Ok`: `step` swallows per-rendition errors so one bad variant doesn't drop the rest. // Letting it through would reset the backoff every pass, so a source that returns 404 // forever would spin at the refresh cadence while publishing nothing. + // + // Keyed on renditions that ingested, not on segments written: a healthy live playlist + // with nothing new this pass writes no segments, and reading that as a failure would end + // a multi-rendition import over one dead variant the others were covering for. let stepped = match self.step(OnError::Warn).await { Ok(StepOutcome { - wrote_segments: 0, + ok: 0, failed: Some(err), .. }) => Err(err), @@ -691,10 +701,14 @@ impl Import { let mut wrote_segments = 0; let mut target_duration = None; let mut failed = None; + let mut ok = 0; for track in self.video.iter_mut().chain(self.audio.iter_mut()) { match track.ingest(&self.fetcher, &mut target_duration).await { - Ok(count) => wrote_segments += count, + Ok(count) => { + wrote_segments += count; + ok += 1; + } Err(err) => match on_error { OnError::Fail => return Err(err), // Keep the other renditions going: one bad variant or segment shouldn't @@ -719,6 +733,7 @@ impl Import { wrote_segments, target_duration, failed, + ok, }) } diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 3b41be1960..e1ac9b59b4 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -50,8 +50,8 @@ http-body = "1" http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-moka", "url-standard"], default-features = false } humantime = "2.3" humantime-serde = "1.1" -kio = { workspace = true, features = ["time"] } jsonwebtoken = "11" +kio = { workspace = true, features = ["time"] } moq-native = { workspace = true, default-features = false, features = ["aws-lc-rs", "watch", "tcp"] } moq-net = { workspace = true } moq-stats = { workspace = true } diff --git a/rs/moq-rtmp/Cargo.toml b/rs/moq-rtmp/Cargo.toml index e55f270cf2..6815049dc6 100644 --- a/rs/moq-rtmp/Cargo.toml +++ b/rs/moq-rtmp/Cargo.toml @@ -33,9 +33,9 @@ anyhow = { version = "1", features = ["backtrace"] } byteorder = "1" bytes = "1" futures = "0.3" -kio = { workspace = true, features = ["time"] } hang = { workspace = true } hmac = "0.13" +kio = { workspace = true, features = ["time"] } moq-mux = { workspace = true } moq-net = { workspace = true } rand = "0.10" From 36cdd1d07fd6734d9fb08a4e85adbeae9d47b6ae Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 20:11:25 -0700 Subject: [PATCH 08/13] fix(hls): drop a redundant closure Co-Authored-By: Claude Opus 5 --- rs/moq-hls/src/import.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 7868190bbe..9a24365378 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -720,7 +720,7 @@ impl Import { // permanently gone, so one dead variant must not end an import the rest could // still serve. Keeping whichever error came last instead would make the // outcome depend on rendition order. - let recoverable = err.status().is_none_or(|status| crate::status_retryable(status)); + let recoverable = err.status().is_none_or(crate::status_retryable); if recoverable || failed.is_none() { failed = Some(err); } From 5480f6f9f58053e2f002157589a0d33d9ca0d41c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 20:20:30 -0700 Subject: [PATCH 09/13] docs: don't link a private item from a public doc `Error::status` is public on both `moq_native` and `moq_hls`, but its doc comment linked `status_retryable`, which became private when the backoff moved to kio. `cargo doc -D warnings` rejects that. The rule is spelled out in prose instead. Co-Authored-By: Claude Opus 5 --- rs/moq-hls/src/error.rs | 6 +++--- rs/moq-native/src/error.rs | 7 ++++--- rs/moq-native/src/noq.rs | 2 +- rs/moq-native/src/quiche.rs | 2 +- rs/moq-native/src/quinn.rs | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index 552b397651..50055295e3 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -136,9 +136,9 @@ pub enum Error { impl Error { /// The HTTP status the origin answered with, if it answered with one at all. /// - /// The import loop reads this through [`status_retryable`]: a `503` on a playlist - /// fetch is worth another pass, a `404` is the origin's settled answer. Nothing else here is - /// classified; a failure with no status falls through to the backoff budget. + /// The import loop consults it: a `503` on a playlist fetch is worth another pass, a `404` is + /// the origin's settled answer. Nothing else here is classified; a failure with no status falls + /// through to the backoff budget. pub fn status(&self) -> Option { match self { Self::Reqwest(err) => err.status().map(|status| status.as_u16()), diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index 1b05c51631..da4251168c 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -159,9 +159,10 @@ impl Error { /// /// `None` covers everything else: a dial that never got a response, a QUIC handshake that /// failed, a URL we couldn't parse. Only a status the peer actually sent shows up here, and - /// [`status_retryable`] is what decides whether it invites another attempt. This - /// deliberately does not try to say whether some *other* kind of failure is worth retrying; - /// that's a guess, and the caller's backoff budget bounds it instead. + /// whether it invites another attempt is the caller's call (`408`, `429`, `502`, `503`, and + /// `504` are the ones worth repeating). This deliberately does not try to say whether some + /// *other* kind of failure is worth retrying; that's a guess, and a backoff budget bounds it + /// instead. pub fn status(&self) -> Option { match self { // A race is only settled when both halves were answered, and answered with something not diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index e333a1ff25..96ee57c91c 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -433,7 +433,7 @@ fn classify_client_error(err: &web_transport_noq::ClientError) -> Option Option { match err { diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index 955be8729d..3ed73620d8 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -509,7 +509,7 @@ fn classify_client_error(err: &web_transport_quiche::ClientError) -> Option Option { match err { diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index d57322935d..90c6effed6 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -446,7 +446,7 @@ fn classify_client_error(err: &web_transport_quinn::ClientError) -> Option Option { match err { From 498992c9afbf6f56bc2c8930e361c45386bd074e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 20:57:10 -0700 Subject: [PATCH 10/13] refactor: inline the backoff instead of sharing a type `Backoff` had no natural home. It went into `moq_net` (the wire layer's published surface), then into `kio::time` next to `Deadline` despite having no poll surface, no `Waiter`, and nothing else kio-shaped about it. Both placements were about reach rather than about what the thing is. It isn't much of a thing. The escalation is three lines, and it's now those three lines at each of the six call sites: draw the wait from the top half of the current window, sleep it, double toward a local `const MAX`. The two loops that want a give-up budget (`Reconnect`, the HLS import) track a deadline next to their delay; the other four never wanted one, which is most of what the shared type was carrying. Generality was also the source of its worst bug: the caller-supplied `max` is what let jitter overflow a `u64` and panic. Six sites with fixed 4s/5s/10s/30s/300s ceilings can't reach it. `kio` is byte-identical to main again, `@moq/net` exports no `Retry`, and the four crates that took a `kio` dependency for this drop it. What remains of the PR's public API is one method: `Error::status`, on moq-native and moq-hls. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 2 +- Cargo.lock | 8 +- js/CLAUDE.md | 2 +- js/net/src/connection/reload.ts | 27 ++- js/net/src/index.ts | 2 - js/net/src/retry.test.ts | 59 ------ js/net/src/retry.ts | 120 ------------ js/publish/src/source/retry.ts | 11 +- rs/CLAUDE.md | 2 +- rs/kio/Cargo.toml | 9 +- rs/kio/src/time.rs | 293 +--------------------------- rs/moq-audio/Cargo.toml | 2 +- rs/moq-audio/src/playback/driver.rs | 40 ++-- rs/moq-hls/Cargo.toml | 1 + rs/moq-hls/src/export/mod.rs | 24 ++- rs/moq-hls/src/import.rs | 42 ++-- rs/moq-native/Cargo.toml | 1 - rs/moq-native/src/reconnect.rs | 45 +++-- rs/moq-relay/Cargo.toml | 2 +- rs/moq-relay/src/cluster.rs | 15 +- rs/moq-rtmp/Cargo.toml | 1 - rs/moq-rtmp/src/server.rs | 30 +-- 22 files changed, 148 insertions(+), 590 deletions(-) delete mode 100644 js/net/src/retry.test.ts delete mode 100644 js/net/src/retry.ts diff --git a/CLAUDE.md b/CLAUDE.md index 622cf41c88..48490910bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,7 +101,7 @@ The rename/removal rationale lives in the commit message and PR description, not Retrying is the reflex that hides bugs, so a new retry loop has to answer three questions in the code, not in the reviewer's head. -- **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Use the shared primitive rather than a hand-rolled `sleep`: `kio::time::Backoff` in Rust, `Retry.Backoff` from `@moq/net` in TypeScript. +- **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Three lines at the call site: draw the wait from the top half of the current window (`delay.mul_f64(0.5 + rand::rng().random::() / 2.0)`), sleep it, then `delay = (delay * 2).min(MAX)`. There is deliberately no shared `Backoff` type. Each loop wants a different subset (most want no budget at all), the escalation is smaller than the abstraction over it, and a general one has to accept an arbitrary `max` it then has to defend against. - **When does it stop?** A deadline or an attempt budget, and that budget is what ends the loop. Unlimited retries belong only to a supervisor whose job is to outlive an outage (a reconnecting publisher, a cluster peer, a listener), where the escalating delay is what keeps a permanently-dead target cheap. - **Who owns the budget?** Exactly one layer. An outer supervisor that rebuilds an inner retry loop resets its backoff to the initial delay, so the escalation never happens and a fixed-interval hammer wears an exponential costume. Watch the inner loop's terminal signal instead of restarting it. diff --git a/Cargo.lock b/Cargo.lock index 0b435974a0..cee367d1c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4013,7 +4013,6 @@ name = "kio" version = "0.5.4" dependencies = [ "loom", - "rand 0.10.2", "smallvec", "tokio", "web-async", @@ -4346,7 +4345,6 @@ dependencies = [ "dispatch2", "fixed-resample", "hang", - "kio", "moq-mux", "moq-net", "objc2", @@ -4355,6 +4353,7 @@ dependencies = [ "objc2-core-media", "objc2-foundation", "objc2-screen-capture-kit", + "rand 0.10.2", "rubato", "sonora", "thiserror 2.0.19", @@ -4493,6 +4492,7 @@ dependencies = [ "moq-mux", "moq-net", "percent-encoding", + "rand 0.10.2", "reqwest", "thiserror 2.0.19", "tokio", @@ -4580,7 +4580,6 @@ dependencies = [ "humantime", "humantime-serde", "jni 0.22.4", - "kio", "moq-net", "noq-proto", "notify", @@ -4666,12 +4665,12 @@ dependencies = [ "humantime", "humantime-serde", "jsonwebtoken", - "kio", "moq-native", "moq-net", "moq-stats", "moq-token", "qmux", + "rand 0.10.2", "rcgen", "reqwest", "reqwest-middleware", @@ -4724,7 +4723,6 @@ dependencies = [ "futures", "hang", "hmac 0.13.0", - "kio", "moq-mux", "moq-native", "moq-net", diff --git a/js/CLAUDE.md b/js/CLAUDE.md index d5c86f1587..323ca22c2d 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -80,7 +80,7 @@ Plain custom elements built directly on `@moq/signals`, no framework (except moq ## Conventions -- **Retries go through `@moq/net`'s `Retry`** (root Retries explains the why). `Retry.Backoff` is the schedule (capped exponential, equal jitter, optional give-up budget); `delay()` returns the wait to hand to `effect.timer`, or `undefined` once the budget is spent, and `reset()` starts a fresh sequence. Never hand-roll a fixed delay in a failure path, and don't try to classify which thrown values are worth retrying: the platform hands back `WebTransportError`, `DOMException`, `AggregateError`, and bare `Error`s interchangeably, so the budget is what stops the loop. +- **Retry loops inline their backoff** (root Retries explains the why). Escalate a local delay toward a `max`, jitter each wait (`delay * (0.5 + Math.random() / 2)`), and hand it to `effect.timer`. There is no shared `Backoff` export, and don't try to classify which thrown values are worth retrying: the platform hands back `WebTransportError`, `DOMException`, `AggregateError`, and bare `Error`s interchangeably, so a budget or an attempt count is what stops the loop. - **Avoid callback parameters.** A function taking a `fn`/`create`/`onXxx` to invoke later reads poorly and hides control flow. Prefer returning a value the caller acts on, exposing a method or getter, or splitting into a couple of small calls the caller sequences itself (e.g. a cache `get()` then `insert(value)`, not `getOrCreate(key, () => value)`). Reserve callbacks for genuine event/subscription sinks where there is no alternative (`effect.subscribe`, DOM listeners, `Signal` subscriptions). - ESM only (`"type": "module"`). Relative imports include the `.ts`/`.tsx` extension in the lower-level packages (`net`, `signals`, `hang`); `rewriteRelativeImportExtensions` in `tsconfig.json` rewrites them to `.js` on build. Some higher-level packages (watch/publish) still omit extensions, so match the file you are editing. - Document every exported symbol and add a top-of-file `@module` doc block to each entrypoint (root convention; the published JSR/`.d.ts` docs render these). Use `@public` on the load-bearing classes. diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 0880615498..e1b2f0d71e 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -3,7 +3,6 @@ import * as Announce from "../announced.ts"; import { error } from "../error.ts"; import type * as Path from "../path.ts"; import { empty as emptyPath } from "../path.ts"; -import { Backoff } from "../retry.ts"; import { type ConnectProps, connect, type WebSocketOptions, type WebTransportProps } from "./connect.ts"; import type { Established } from "./established.ts"; import type { Probe, Stats } from "./stats.ts"; @@ -44,6 +43,9 @@ export type ReloadProps = Omit & { delay?: ReloadDelay; }; +/** How long to keep retrying before giving up, when {@link ReloadDelay.timeout} is unset. */ +const DEFAULT_TIMEOUT = 300000; + /** Current state of a {@link Reload} connection. */ export type ReloadStatus = "connecting" | "connected" | "disconnected"; @@ -97,9 +99,10 @@ export class Reload { #closedResolve!: () => void; #closedReject!: (err: Error) => void; - // The current retry sequence's schedule, built from `delay` when the sequence starts. Undefined - // between sequences, so a later edit to `delay` applies to the next one. - #backoff: Backoff | undefined; + // The current wait between attempts, doubling per failure, and when the retry window expires. + // Both are undefined between sequences, so a later edit to `delay` applies to the next one. + #delay: DOMHighResTimeStamp | undefined; + #deadline: DOMHighResTimeStamp | undefined; // Increased by 1 each time to trigger a reload. #tick = new Signal(0); @@ -219,19 +222,27 @@ export class Reload { // shorter is a peer that accepts and immediately severs, which has to keep // escalating or we hammer it forever at the initial delay. if (connected !== undefined && performance.now() - connected >= this.delay.initial) { - this.#backoff = undefined; + this.#delay = undefined; + this.#deadline = undefined; } - this.#backoff ??= new Backoff(this.delay); + const now = performance.now(); + const timeout = this.delay.timeout ?? DEFAULT_TIMEOUT; + this.#delay ??= this.delay.initial; + this.#deadline ??= timeout > 0 ? now + timeout : Number.POSITIVE_INFINITY; - const wait = this.#backoff.delay(); - if (wait === undefined) { + if (now >= this.#deadline) { console.warn("reconnect timed out"); // A graceful close has no error, so report the timeout itself. this.#closedReject(cause === undefined ? new Error("reconnect timed out") : error(cause)); return; } + // Equal jitter, so a fleet of tabs knocked offline together doesn't reconnect on the same + // tick, and never past the deadline the retry window promised. + const wait = Math.min(this.#delay * (0.5 + Math.random() / 2), this.#deadline - now); + this.#delay = Math.min(this.#delay * this.delay.multiplier, this.delay.max); + const tick = this.#tick.peek() + 1; effect.timer(() => this.#tick.update((prev) => Math.max(prev, tick)), wait); } diff --git a/js/net/src/index.ts b/js/net/src/index.ts index 845154dd09..b53d74f168 100644 --- a/js/net/src/index.ts +++ b/js/net/src/index.ts @@ -19,8 +19,6 @@ export { RemoteError } from "./error.ts"; export * as Group from "./group.ts"; /** Broadcast path utilities with delimiter-aware prefix matching. */ export * as Path from "./path.ts"; -/** Retry policy: which failures are worth repeating, and how long to wait before repeating them. */ -export * as Retry from "./retry.ts"; /** Branded time types (nanoseconds, microseconds, milliseconds, seconds) with conversions. */ export * as Time from "./time.ts"; /** Track role handles. */ diff --git a/js/net/src/retry.test.ts b/js/net/src/retry.test.ts deleted file mode 100644 index b944f50f8e..0000000000 --- a/js/net/src/retry.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { expect, test } from "bun:test"; -import { Backoff } from "./retry.ts"; - -test("the window escalates to the cap, each delay inside its jitter band", () => { - const backoff = new Backoff({ initial: 1000, multiplier: 2, max: 8000, timeout: 0 }); - - for (const window of [1000, 2000, 4000, 8000, 8000, 8000]) { - const delay = backoff.delay(); - expect(delay).toBeGreaterThanOrEqual(window / 2); - expect(delay).toBeLessThanOrEqual(window); - } -}); - -test("jitter separates identical schedules", () => { - const props = { initial: 1000, multiplier: 2, max: 8000, timeout: 0 }; - const a = new Backoff(props); - const b = new Backoff(props); - - // One shared draw could collide by chance; a run of them colliding means no jitter at all. - const differs = Array.from({ length: 8 }).some(() => a.delay() !== b.delay()); - expect(differs).toBe(true); -}); - -test("reset returns to the initial window", () => { - const backoff = new Backoff({ initial: 1000, multiplier: 2, max: 8000, timeout: 0 }); - for (let i = 0; i < 4; i++) backoff.delay(); - - backoff.reset(); - expect(backoff.delay()).toBeLessThanOrEqual(1000); -}); - -test("a zero timeout never gives up", () => { - const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 0 }); - for (let i = 0; i < 64; i++) expect(backoff.delay()).toBeDefined(); -}); - -test("a delay never outlives the budget", () => { - // An initial delay longer than the whole budget must not sleep past it: the budget is the - // promise, and one oversized window would blow through it before a single retry lands. - const backoff = new Backoff({ initial: 60000, multiplier: 2, max: 60000, timeout: 50 }); - - const delay = backoff.delay(); - expect(delay).toBeDefined(); - expect(delay).toBeLessThanOrEqual(50); -}); - -test("the budget is a deadline over the whole sequence", async () => { - const backoff = new Backoff({ initial: 1, multiplier: 2, max: 8, timeout: 5 }); - - // The first delay starts the clock; outliving the budget is what stops the sequence. Sleeping - // past it rather than shrinking the timeout keeps the test off `performance.now()`'s resolution. - expect(backoff.delay()).toBeDefined(); - await Bun.sleep(15); - expect(backoff.delay()).toBeUndefined(); - - // A reset says the earlier failures no longer describe reality, so the budget refills. - backoff.reset(); - expect(backoff.delay()).toBeDefined(); -}); diff --git a/js/net/src/retry.ts b/js/net/src/retry.ts deleted file mode 100644 index f4033f40ee..0000000000 --- a/js/net/src/retry.ts +++ /dev/null @@ -1,120 +0,0 @@ -/** - * The retry schedule shared by every loop that re-attempts a failed operation. - * - * {@link Backoff} answers *when* to try again, and its budget is what ends a loop. There is - * deliberately no counterpart answering *whether* a given error is worth repeating: the browser - * hands back whatever the platform threw, and guessing wrong either strands a connection a retry - * would have recovered or hammers a dead one. The budget bounds the damage instead. - * - * @module - */ - -/** Delay before the first retry, in milliseconds. */ -const DEFAULT_INITIAL = 1000; -/** Multiplier applied to the delay after each failure. */ -const DEFAULT_MULTIPLIER = 2; -/** Ceiling on the delay, in milliseconds. */ -const DEFAULT_MAX = 30000; -/** How long to keep retrying before giving up, in milliseconds. */ -const DEFAULT_TIMEOUT = 300000; - -/** - * How long to wait between attempts, and how long to keep making them. - * - * The defaults suit a long-lived connection: a second before the first retry, doubling to a - * half-minute ceiling, giving up after five minutes. - */ -export type BackoffProps = { - /** Delay in milliseconds before the first retry (default: 1000). */ - initial?: DOMHighResTimeStamp; - - /** Multiplier applied to the delay after each failure (default: 2). */ - multiplier?: number; - - /** Ceiling on the delay in milliseconds, however many failures have piled up (default: 30000). */ - max?: DOMHighResTimeStamp; - - /** - * How long to keep trying before giving up, in milliseconds (default: 300000, five minutes). - * Measured from construction or the last {@link Backoff.reset}, and covering the attempts - * themselves rather than just the waits between them. Zero retries forever, which only belongs - * in a supervisor whose job is to outlive an outage. - */ - timeout?: DOMHighResTimeStamp; -}; - -/** - * A capped exponential backoff with jitter and a give-up budget. - * - * Each delay is drawn from the top half of the current window (equal jitter), so a fleet that fails - * together doesn't retry together, while still waiting at least half the escalating delay. The - * window grows by {@link BackoffProps.multiplier} per failure up to {@link BackoffProps.max}, and - * {@link BackoffProps.timeout} bounds the whole sequence. - * - * Call {@link delay} after each failure and {@link reset} after a success worth trusting. Nothing - * else may own a competing schedule for the same operation: an outer supervisor that rebuilds an - * inner loop restarts its backoff at the initial delay and the escalation never happens. - * - * @public - */ -export class Backoff { - readonly #initial: DOMHighResTimeStamp; - readonly #multiplier: number; - readonly #max: DOMHighResTimeStamp; - readonly #timeout: DOMHighResTimeStamp; - - /** The current window's upper bound, grown per failure. */ - #window: DOMHighResTimeStamp; - - /** When the budget runs out, or undefined when there isn't one. */ - #deadline: DOMHighResTimeStamp | undefined; - - constructor(props?: BackoffProps) { - this.#initial = props?.initial ?? DEFAULT_INITIAL; - // Below 1 the window would shrink per failure, turning the escalation into a tight loop. - this.#multiplier = Math.max(props?.multiplier ?? DEFAULT_MULTIPLIER, 1); - this.#max = props?.max ?? DEFAULT_MAX; - this.#timeout = props?.timeout ?? DEFAULT_TIMEOUT; - this.#window = this.#initial; - this.#deadline = this.#budget(); - } - - /** When the budget runs out, or undefined when there isn't one. */ - #budget(): DOMHighResTimeStamp | undefined { - return this.#timeout > 0 ? performance.now() + this.#timeout : undefined; - } - - /** How long to wait before the next attempt, or undefined once the budget is spent. */ - delay(): DOMHighResTimeStamp | undefined { - let remaining = Number.POSITIVE_INFINITY; - - if (this.#deadline !== undefined) { - // Out of budget. The clock runs from construction or the last `reset`, so it covers the - // attempts as well as the waits between them: a caller whose every attempt hangs for most - // of the budget would otherwise outlive it many times over. - remaining = this.#deadline - performance.now(); - if (remaining <= 0) return undefined; - } - - // Equal jitter: at least half the window, never more than all of it. - const delay = this.#window / 2 + Math.random() * (this.#window / 2); - this.#window = Math.min(this.#window * this.#multiplier, this.#max); - - // Never sleep past the deadline: the budget says how long to keep retrying, so overshooting - // it by a whole window would spend more than the caller asked for and skip the attempt that - // still fit. A truncated final delay is the point, not a rounding error. - return Math.min(delay, remaining); - } - - /** - * Start over: the next delay is {@link BackoffProps.initial} again and the budget is full. - * - * Only call this after an outcome that says the earlier failures no longer describe reality: a - * session that stayed up, a request that succeeded, a changed destination. Resetting on an - * attempt that failed immediately turns the escalation into a tight loop. - */ - reset(): void { - this.#window = this.#initial; - this.#deadline = this.#budget(); - } -} diff --git a/js/publish/src/source/retry.ts b/js/publish/src/source/retry.ts index 234c83cd80..9a5a57e6df 100644 --- a/js/publish/src/source/retry.ts +++ b/js/publish/src/source/retry.ts @@ -1,4 +1,3 @@ -import { Retry as NetRetry } from "@moq/net"; import { type Effect, Signal } from "@moq/signals"; /** @@ -26,14 +25,14 @@ export class Retry { * No give-up deadline: {@link LIMIT} is this budget, counted in attempts. Counting time instead * would make the outcome depend on how long the OS takes to say no. */ - static readonly DELAY: NetRetry.BackoffProps = { initial: 250, multiplier: 2, max: 1000, timeout: 0 }; + static readonly DELAY = { initial: 250, multiplier: 2, max: 1000 }; readonly #rerun = new Signal(0); // Deliberately plain fields: effect reruns must not unwind them, or the budget never runs out. #failures = 0; #settings: unknown[] | undefined; - #backoff = new NetRetry.Backoff(Retry.DELAY); + #delay = Retry.DELAY.initial; // How long the next attempt still owes the backoff, set by `failed` and paid by `begin`. #wait: DOMHighResTimeStamp | undefined; @@ -74,7 +73,9 @@ export class Retry { failed(): void { this.#failures += 1; // Unlimited budget, so there is always a next delay. - this.#wait = this.#backoff.delay(); + // Equal jitter, so a page with several captures doesn't reopen them all on the same tick. + this.#wait = this.#delay * (0.5 + Math.random() / 2); + this.#delay = Math.min(this.#delay * Retry.DELAY.multiplier, Retry.DELAY.max); this.#rerun.update((rerun) => rerun + 1); } @@ -99,6 +100,6 @@ export class Retry { #clear(): void { this.#failures = 0; this.#wait = undefined; - this.#backoff.reset(); + this.#delay = Retry.DELAY.initial; } } diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index cdb4232bc5..7e8654627d 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,7 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions -- **Retries go through `kio::time::Backoff`** (root Retries explains the why). It's the schedule (capped exponential, equal jitter, optional give-up budget): `sleep().await` in an async loop, `delay()` when the caller owns the waiting (a blocking thread, a `select!` arm), `reset()` after an outcome worth trusting. `kio::time::Config` is `#[non_exhaustive]`, so build it with `default()` + field set. It lives in kio rather than moq-net because a backoff schedule is a utility, not part of the wire layer's surface. Never hand-roll a `tokio::time::sleep(FIXED)` in a failure arm, and don't add an `is_retryable()` to an error type: the budget is what stops a loop. The one thing worth reading off a failure is a status a peer actually sent, via the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. +- **Retry loops inline their backoff** (root Retries explains the why). Escalate a local `Duration` toward a `const MAX`, jitter each wait, and sleep it. Don't reach for a shared `Backoff` type: one existed briefly and was removed, because four of the six call sites wanted no give-up budget and the general version's caller-supplied `max` was itself the source of an overflow panic. A loop that needs a deadline tracks a `tokio::time::Instant` next to its delay. And don't add an `is_retryable()` to an error type: the budget, or classification-free escalation, is what stops a loop. The one thing worth reading off a failure is a status a peer actually sent, via the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/kio/Cargo.toml b/rs/kio/Cargo.toml index cd704cd403..079f3841ef 100644 --- a/rs/kio/Cargo.toml +++ b/rs/kio/Cargo.toml @@ -13,15 +13,12 @@ keywords = ["async", "producer", "consumer", "state", "sync"] categories = ["asynchronous", "concurrency"] [features] -# Opt-in poll-driven wall-clock deadlines and retry schedules, backed by `web-async` -# (tokio on native, wasmtimer in the browser). Off by default so kio stays runtime-free. -# `rand` is here for `Backoff`'s jitter; a wasm consumer picks its own getrandom -# backend in the leaf binary, same as moq-net. -time = ["dep:web-async", "dep:rand"] +# Opt-in poll-driven wall-clock deadlines, backed by `web-async` (tokio on native, +# wasmtimer in the browser). Off by default so kio stays runtime-free. +time = ["dep:web-async"] tokio = ["dep:tokio"] [dependencies] -rand = { version = "0.10.1", optional = true } smallvec = "1.15" tokio = { workspace = true, features = ["time"], optional = true } web-async = { workspace = true, optional = true } diff --git a/rs/kio/src/time.rs b/rs/kio/src/time.rs index 0874ea2569..eaa8e4ec2e 100644 --- a/rs/kio/src/time.rs +++ b/rs/kio/src/time.rs @@ -1,7 +1,4 @@ -//! Poll-driven wall-clock deadlines and retry schedules. -//! -//! [`Deadline`] is a single instant to poll on. [`Backoff`] is the escalating delay a loop waits -//! between attempts at something that failed, with a budget that eventually stops it. +//! Poll-driven wall-clock deadlines. //! //! Behind the `time` feature. Built on [`web_async::time`], which is `tokio::time` on //! native and `wasmtimer` in the browser, so the rest of kio stays runtime-free. @@ -13,8 +10,6 @@ use std::{pin::Pin, task::Poll}; -use rand::RngExt; - /// Re-exported from `web-async`, so a major bump of that crate is a breaking change /// for these types. pub use web_async::time::{Duration, Instant}; @@ -128,165 +123,6 @@ impl std::fmt::Debug for Deadline { } } -/// How long to wait between attempts, and how long to keep making them. -/// -/// The defaults suit a long-lived connection: a second before the first retry, doubling to a -/// half-minute ceiling, giving up after five minutes. A one-shot request wants a much smaller -/// [`timeout`](Self::timeout); a supervisor that must never stop wants a zero one. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[non_exhaustive] -pub struct Config { - /// Delay before the first retry. - pub initial: Duration, - - /// Multiplier applied to the delay after each failure. - pub multiplier: u32, - - /// Ceiling on the delay, however many failures have piled up. - pub max: Duration, - - /// How long to keep trying before giving up, measured from construction or the last - /// [`reset`](Backoff::reset). This covers the attempts themselves, not just the waits between - /// them, so a caller whose every attempt hangs still gives up on schedule. [`Duration::ZERO`] - /// retries forever, which only belongs in a supervisor whose job is to outlive an outage. - pub timeout: Duration, -} - -impl Default for Config { - fn default() -> Self { - Self { - initial: Duration::from_secs(1), - multiplier: 2, - max: Duration::from_secs(30), - timeout: Duration::from_secs(300), - } - } -} - -/// The escalating delay a loop waits between attempts at something that failed. -/// -/// Capped exponential backoff with jitter and a give-up budget. It answers *when* to try again and, -/// through the budget, when to stop. It deliberately has no opinion on *whether* a given failure is -/// worth repeating: deciding that means guessing, and a wrong guess either strands something a retry -/// would have recovered or hammers something already dead. The budget bounds the damage instead. -/// -/// Each delay is drawn from the top half of the current window (equal jitter), so a fleet that fails -/// together doesn't retry together, while still waiting at least half the escalating delay. The -/// window doubles per failure up to [`Config::max`], and [`Config::timeout`] bounds the whole -/// sequence. -/// -/// Call [`sleep`](Self::sleep) (or [`delay`](Self::delay), if the caller owns the waiting) after each -/// failure and [`reset`](Self::reset) after a success worth trusting. Nothing else may own a competing -/// schedule for the same operation: an outer supervisor that rebuilds an inner loop restarts its -/// backoff at the initial delay and the escalation never happens. -#[derive(Debug)] -pub struct Backoff { - config: Config, - - /// The current window's upper bound, doubled per failure. - window: Duration, - - /// When the budget runs out, or `None` when there isn't one. - deadline: Option, -} - -impl Backoff { - /// A backoff following `config`, with a full budget starting now. - pub fn new(config: Config) -> Self { - Self { - window: config.initial, - deadline: Self::deadline(&config), - config, - } - } - - /// When the budget runs out, or `None` when there isn't one. - /// - /// An unlimited budget never reads the clock, which is what lets a blocking thread with its own - /// [`std::time::Instant`] bookkeeping drive this too. - fn deadline(config: &Config) -> Option { - match config.timeout.is_zero() { - true => None, - // An unrepresentable deadline is treated as no deadline. - false => Instant::now().checked_add(config.timeout), - } - } - - /// How long to wait before the next attempt, or `None` once the budget is spent. - /// - /// For callers that do their own waiting (a blocking thread, a poll loop with other arms). - /// Everything else wants [`sleep`](Self::sleep). - pub fn delay(&mut self) -> Option { - let mut remaining = None; - if let Some(deadline) = self.deadline { - let now = Instant::now(); - - // Out of budget. The clock runs from construction or the last `reset`, so it covers the - // attempts as well as the waits between them: a caller whose every attempt hangs for - // most of the budget would otherwise outlive it many times over. - if now >= deadline { - return None; - } - remaining = deadline.checked_duration_since(now); - } - - let delay = self.jitter(self.window); - self.window = self - .window - .saturating_mul(self.config.multiplier.max(1)) - .min(self.config.max); - - // Never sleep past the deadline: the budget says how long to keep retrying, so overshooting - // it by a whole window would spend more than the caller asked for and skip the attempt that - // still fit. A truncated final delay is the point, not a rounding error. - Some(match remaining { - Some(remaining) => delay.min(remaining), - None => delay, - }) - } - - /// Wait out the next delay, returning `false` once the budget is spent. - /// - /// A `false` means stop retrying: the caller should surface the failure that got it here rather - /// than loop again. - pub async fn sleep(&mut self) -> bool { - let Some(delay) = self.delay() else { return false }; - web_async::time::sleep(delay).await; - true - } - - /// Start over: the next delay is [`Config::initial`] again and the budget is full. - /// - /// Only call this after an outcome that says the earlier failures no longer describe reality: a - /// session that stayed up, a request that succeeded, a changed destination. Resetting on an - /// attempt that failed immediately turns the escalation into a tight loop. - pub fn reset(&mut self) { - self.window = self.config.initial; - self.deadline = Self::deadline(&self.config); - } - - /// Draw the actual delay from the top half of `window`, so peers that failed together spread out. - fn jitter(&self, window: Duration) -> Duration { - let half = window / 2; - - // A window past ~584 years holds more nanoseconds than a `u64`, and truncating one would - // hand `random_range` an empty range to panic on. `max` is caller-configurable (a humantime - // string on the CLI), so saturate rather than trust it to be sane. - let span = u64::try_from(half.as_nanos()).unwrap_or(u64::MAX); - if span == 0 { - return window; - } - - half.saturating_add(Duration::from_nanos(rand::rng().random_range(0..span))) - } -} - -impl Default for Backoff { - fn default() -> Self { - Self::new(Config::default()) - } -} - #[cfg(all(test, not(loom)))] mod tests { use std::task::Waker; @@ -385,131 +221,4 @@ mod tests { assert!(Instant::now() >= at, "returned before the re-armed deadline"); } - - fn config() -> Config { - Config { - initial: Duration::from_secs(1), - multiplier: 2, - max: Duration::from_secs(8), - timeout: Duration::ZERO, - } - } - - /// The window doubles per failure and stops at the cap, and jitter keeps every draw inside the - /// top half of its window. - #[tokio::test(start_paused = true)] - async fn escalates_to_the_cap_within_the_jitter_band() { - let mut backoff = Backoff::new(config()); - - for expected in [1, 2, 4, 8, 8, 8].map(Duration::from_secs) { - let delay = backoff.delay().expect("unlimited budget"); - assert!( - delay >= expected / 2 && delay <= expected, - "{delay:?} outside the jitter band for {expected:?}" - ); - } - } - - /// Two backoffs with the same settings must not step in lockstep, or a fleet that failed - /// together retries together. - #[tokio::test(start_paused = true)] - async fn jitter_separates_identical_schedules() { - let mut a = Backoff::new(config()); - let mut b = Backoff::new(config()); - - // One shared draw could collide by chance; a run of them colliding means no jitter at all. - let differs = (0..8).any(|_| a.delay() != b.delay()); - assert!(differs, "identical backoffs produced identical delays"); - } - - /// `max` comes from a caller-supplied humantime string, so an absurd one has to degrade rather - /// than panic: a window past ~584 years has more nanoseconds than the jitter sample can hold. - #[tokio::test(start_paused = true)] - async fn an_absurd_window_does_not_panic() { - let mut backoff = Backoff::new(Config { - initial: Duration::new(36_893_488_147, 419_103_232), - max: Duration::MAX, - ..config() - }); - - let delay = backoff.delay().expect("unlimited budget"); - assert!( - delay >= Duration::new(18_446_744_073, 709_551_616), - "{delay:?} below half" - ); - } - - #[tokio::test(start_paused = true)] - async fn reset_returns_to_the_initial_window() { - let mut backoff = Backoff::new(config()); - for _ in 0..4 { - backoff.delay(); - } - - backoff.reset(); - let delay = backoff.delay().expect("unlimited budget"); - assert!(delay <= Duration::from_secs(1), "{delay:?} did not return to initial"); - } - - /// An initial delay longer than the whole budget must not sleep past it: the budget is the - /// promise, and one oversized window would blow through it before a single retry lands. - #[tokio::test(start_paused = true)] - async fn a_delay_never_outlives_the_budget() { - let mut backoff = Backoff::new(Config { - initial: Duration::from_secs(60), - timeout: Duration::from_millis(50), - ..config() - }); - - let delay = backoff.delay().expect("budget available"); - assert!(delay <= Duration::from_millis(50), "{delay:?} outlived the budget"); - } - - /// The budget is a wall-clock deadline over the whole sequence, not a per-attempt one, and the - /// sequence lands on it rather than overshooting by a whole window. - #[tokio::test(start_paused = true)] - async fn gives_up_once_the_budget_is_spent() { - let timeout = Duration::from_secs(10); - let mut backoff = Backoff::new(Config { timeout, ..config() }); - - let started = tokio::time::Instant::now(); - while let Some(delay) = backoff.delay() { - tokio::time::sleep(delay).await; - assert!(started.elapsed() < Duration::from_secs(60), "budget never ran out"); - } - - // Tokio's paused clock rounds each sleep to its timer granularity, so the sequence can land a - // hair either side of the deadline it aimed for. - let elapsed = started.elapsed(); - assert!( - elapsed >= timeout - Duration::from_millis(10), - "gave up after only {elapsed:?}" - ); - assert!( - elapsed < timeout + config().max, - "overshot the budget by a whole window: {elapsed:?}" - ); - } - - /// A zero timeout is the supervisor case: keep retrying however long the outage lasts. - #[tokio::test(start_paused = true)] - async fn a_zero_timeout_never_gives_up() { - let mut backoff = Backoff::new(config()); - for _ in 0..64 { - assert!(backoff.sleep().await); - } - } - - /// The budget covers the retry sequence, so a reset after a healthy stretch buys a fresh one. - #[tokio::test(start_paused = true)] - async fn reset_refills_the_budget() { - let mut backoff = Backoff::new(Config { - timeout: Duration::from_secs(10), - ..config() - }); - - while backoff.sleep().await {} - backoff.reset(); - assert!(backoff.sleep().await, "reset did not refill the budget"); - } } diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index abaf727a70..9cd2011e6e 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -56,11 +56,11 @@ cpal = { version = "0.18", optional = true } # `fft-resampler` stays off, matching the sinc-only stance below. fixed-resample = { version = "0.12", optional = true, default-features = false, features = ["channel", "resampler"] } hang = { workspace = true } -kio = { workspace = true, features = ["time"] } moq-mux = { workspace = true } moq-net = { workspace = true } # We only use the sinc resampler (Async::new_sinc), so skip the default # fft_resampler feature and its RustFFT/realfft dependencies. +rand = "0.10" rubato = { version = "4.0", default-features = false } # Pure-Rust port of WebRTC's audio processing module (AEC3, noise suppression, # AGC2). The reason the `aec` feature can exist at all: every other option is a diff --git a/rs/moq-audio/src/playback/driver.rs b/rs/moq-audio/src/playback/driver.rs index 7525d39729..bca51f9d36 100644 --- a/rs/moq-audio/src/playback/driver.rs +++ b/rs/moq-audio/src/playback/driver.rs @@ -13,25 +13,20 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use cpal::traits::{DeviceTrait, StreamTrait}; +use rand::RngExt; use super::mixer::{self, Mixer}; use super::sink::{Registration, Sink}; use crate::Error; -/// Backoff for reopening a device that failed. The first retry is quick because the common case is -/// a device that came right back (a USB re-enumerate, a sample-rate change); the ceiling keeps a -/// permanently gone device from spinning. +/// Backoff bounds for reopening a device that failed. The first retry is quick because the common +/// case is a device that came right back (a USB re-enumerate, a sample-rate change); the ceiling +/// keeps a permanently gone device from spinning. /// -/// No give-up budget: the engine outlives any one device, and the user plugging a headset back in is -/// exactly the external change a retry is waiting for. Unlimited retries are also what keeps this -/// clock-free, so the driver thread can stay on [`std::time::Instant`]. -fn retry_backoff() -> kio::time::Backoff { - let mut config = kio::time::Config::default(); - config.initial = Duration::from_millis(500); - config.max = Duration::from_secs(4); - config.timeout = Duration::ZERO; - kio::time::Backoff::new(config) -} +/// No give-up budget: the engine outlives any one device, and the user plugging a headset back in +/// is exactly the external change a retry is waiting for. +const RETRY_MIN: Duration = Duration::from_millis(500); +const RETRY_MAX: Duration = Duration::from_secs(4); /// Problems tolerated in [`ERROR_WINDOW`] before the stream is rebuilt. /// @@ -363,7 +358,7 @@ pub(super) fn run( stream: None, retired: None, generation: 0, - retry: retry_backoff(), + retry: RETRY_MIN, retry_at: None, underruns: 0, unclassified: 0, @@ -425,8 +420,8 @@ struct Driver { /// Bumped on every stream, so an error from a retired one can be told apart /// from one the live stream raised. generation: u64, - /// Escalating delay before reopening a device that would not start. - retry: kio::time::Backoff, + /// Delay before reopening a device that would not start, doubling per failure. + retry: Duration, /// When a failed start may be retried, and what the command wait times out /// against. `None` while the stream is healthy. retry_at: Option, @@ -484,7 +479,7 @@ impl Driver { // Replaces the previous receiver, dropping anything the old stream // retired and never got drained. self.retired = Some(retired_rx); - self.retry.reset(); + self.retry = RETRY_MIN; tracing::info!(rate, channels, ?format, "opened audio output"); Ok(()) @@ -601,11 +596,14 @@ impl Driver { tracing::warn!("audio output is not keeping up with sink changes"); } - /// When the next restart may be attempted, escalating the backoff. + /// When the next restart may be attempted, doubling the backoff. /// - /// The backoff never gives up, so this always yields a deadline. + /// Jittered so a host running many streams doesn't reopen the device in lockstep after a + /// suspend or a driver reload. fn schedule(&mut self) -> Instant { - Instant::now() + self.retry.delay().expect("unlimited retry budget") + let wait = self.retry.mul_f64(0.5 + rand::rng().random::() / 2.0); + self.retry = (self.retry * 2).min(RETRY_MAX); + Instant::now() + wait } /// Whether a failure reported by stream `generation` should rebuild the @@ -807,7 +805,7 @@ mod tests { stream: None, retired: None, generation: 7, - retry: retry_backoff(), + retry: RETRY_MIN, retry_at: None, underruns: 0, unclassified: 0, diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 836eebf7b1..5ec7e329de 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -30,6 +30,7 @@ m3u8-rs = "6" moq-mux = { workspace = true } moq-net = { workspace = true } percent-encoding = { workspace = true } +rand = "0.10" reqwest = { version = "0.13", default-features = false, features = ["rustls", "gzip"] } thiserror = "2" tokio = { workspace = true, features = ["full"] } diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 6cee23b004..cac5652225 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -27,28 +27,24 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use moq_mux::catalog::{self, CatalogFormat, Stream}; +use rand::RngExt; pub(crate) use playlist::render_media; pub use rendition::{Kind, Rendition}; -/// Backoff for the initial catalog subscription. +/// Backoff bounds for the initial catalog subscription. /// /// The usual failure is a publisher that has announced its broadcast but not yet written its -/// catalog track, so this waits for external state rather than repeating a failed request. Escalating -/// for that reason: a source that stays silent for an hour must not be polled four times a second -/// for an hour. +/// catalog track, so this waits for external state rather than repeating a failed request. +/// Escalating for that reason: a source that stays silent for an hour must not be polled four times +/// a second for an hour. /// /// Deliberately no give-up budget, and no attempt to judge which failures are worth waiting on. The /// broadcast closing is what ends the wait, and a relay-side broadcast outlives its publisher's /// session, so any deadline here is a window in which a publisher outage leaves the broadcaster /// permanently empty with nothing to recover it. -fn catalog_backoff() -> kio::time::Backoff { - let mut config = kio::time::Config::default(); - config.initial = Duration::from_millis(250); - config.max = Duration::from_secs(5); - config.timeout = Duration::ZERO; - kio::time::Backoff::new(config) -} +const CATALOG_RETRY_MIN: Duration = Duration::from_millis(250); +const CATALOG_RETRY_MAX: Duration = Duration::from_secs(5); /// Export tuning shared across renditions. /// @@ -189,15 +185,17 @@ async fn watch_catalog( config: Config, renditions: renditions::Producer, ) { - let mut backoff = catalog_backoff(); + let mut delay = CATALOG_RETRY_MIN; let mut consumer = loop { match catalog::Consumer::<()>::new(&broadcast, CatalogFormat::Hang).await { Ok(consumer) => break consumer, Err(err) => { tracing::warn!(%err, "failed to subscribe to broadcast catalog, retrying"); + let wait = delay.mul_f64(0.5 + rand::rng().random::() / 2.0); + delay = (delay * 2).min(CATALOG_RETRY_MAX); tokio::select! { - _ = backoff.sleep() => {} + _ = tokio::time::sleep(wait) => {} _ = kio::wait(|waiter| broadcast.poll_closed(waiter)) => { renditions.close(); return; diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 9a24365378..ae679facaa 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -19,6 +19,7 @@ use m3u8_rs::{ use moq_mux::catalog::Producer as CatalogProducer; use moq_mux::container::fmp4::Import as Fmp4; use moq_mux::select; +use rand::RngExt; use reqwest::Client; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tracing::{debug, info, warn}; @@ -29,19 +30,16 @@ use crate::{Error, Result, SequenceKind}; /// Per-request timeout for the default HTTP client (playlist + segment fetches). const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); -/// Backoff for retrying a failed import step, so a transient upstream error (a 503, a dropped -/// connection) doesn't tear down the whole import. +/// Backoff bounds for retrying a failed import step, so a transient upstream error (a 503, a +/// dropped connection) doesn't tear down the whole import. /// -/// Bounded by the default give-up budget: an origin that has been unreachable for minutes is an -/// outage the caller should hear about, not one the import should paper over indefinitely while -/// publishing nothing. The ceiling is lower than the default, since a live playlist window is -/// measured in seconds and a longer wait would blow past it anyway. -fn error_backoff() -> kio::time::Backoff { - let mut config = kio::time::Config::default(); - config.initial = Duration::from_secs(1); - config.max = Duration::from_secs(10); - kio::time::Backoff::new(config) -} +/// Bounded by a give-up budget: an origin that has been unreachable for minutes is an outage the +/// caller should hear about, not one the import should paper over indefinitely while publishing +/// nothing. The ceiling is low because a live playlist window is measured in seconds and a longer +/// wait would blow past it anyway. +const STEP_RETRY_MIN: Duration = Duration::from_secs(1); +const STEP_RETRY_MAX: Duration = Duration::from_secs(10); +const STEP_RETRY_BUDGET: Duration = Duration::from_secs(300); /// How far back from the live edge to start when (re-)anchoring to a playlist window. /// @@ -619,7 +617,8 @@ impl Import { /// shortcut is an HTTP status the origin actually sent: a `404` playlist ends the import /// immediately, since no amount of waiting turns it into a `200`. pub async fn run(&mut self) -> Result<()> { - let mut backoff = error_backoff(); + let mut delay = STEP_RETRY_MIN; + let mut deadline = tokio::time::Instant::now() + STEP_RETRY_BUDGET; loop { // A step where *nothing* ingested while a rendition was failing is a failed pass wearing @@ -641,7 +640,9 @@ impl Import { let outcome = match stepped { Ok(outcome) => { - backoff.reset(); + // The source is alive, so the earlier failures no longer describe it. + delay = STEP_RETRY_MIN; + deadline = tokio::time::Instant::now() + STEP_RETRY_BUDGET; outcome } // A status the origin actually sent is its answer: a 404 playlist is not going to @@ -650,10 +651,19 @@ impl Import { return Err(err); } Err(err) => { - warn!(%err, "HLS import step failed, retrying"); - if !backoff.sleep().await { + let now = tokio::time::Instant::now(); + if now >= deadline { return Err(err); } + + warn!(%err, "HLS import step failed, retrying"); + // Jittered so a fleet of importers pointed at one origin doesn't retry in + // lockstep, and never past the deadline the budget promised. + let wait = delay + .mul_f64(0.5 + rand::rng().random::() / 2.0) + .min(deadline - now); + delay = (delay * 2).min(STEP_RETRY_MAX); + tokio::time::sleep(wait).await; continue; } }; diff --git a/rs/moq-native/Cargo.toml b/rs/moq-native/Cargo.toml index 5460607fce..70babdf4c7 100644 --- a/rs/moq-native/Cargo.toml +++ b/rs/moq-native/Cargo.toml @@ -46,7 +46,6 @@ hex = "0.4" humantime = "2.3" humantime-serde = "1.1" -kio = { workspace = true, features = ["time"] } moq-net = { workspace = true } # iroh runs on noq but re-exports only the ControllerFactory trait, not the concrete # congestion configs. Version matched to the copy iroh pulls in. diff --git a/rs/moq-native/src/reconnect.rs b/rs/moq-native/src/reconnect.rs index 6e6bb8d53b..4a7476e598 100644 --- a/rs/moq-native/src/reconnect.rs +++ b/rs/moq-native/src/reconnect.rs @@ -4,6 +4,7 @@ use std::time::Duration; use moq_net::Version; use moq_net::bandwidth::{Consumer as BandwidthConsumer, Producer as BandwidthProducer}; use moq_net::kio; +use rand::RngExt; use url::Url; use crate::{Client, Error}; @@ -73,17 +74,6 @@ impl Default for Backoff { } } -impl From<&Backoff> for kio::time::Config { - fn from(backoff: &Backoff) -> Self { - let mut config = Self::default(); - config.initial = backoff.initial; - config.multiplier = backoff.multiplier; - config.max = backoff.max; - config.timeout = backoff.timeout; - config - } -} - impl Backoff { /// How long broadcasts fed by a reconnecting session should outlive a session /// drop (see [`moq_net::origin::Info::linger`]): slightly past the give-up @@ -98,6 +88,15 @@ impl Backoff { } } +/// When a reconnect sequence gives up, or `None` when [`Backoff::timeout`] is zero and it never +/// does. Measured from now, so it covers the connect attempts as well as the waits between them. +fn deadline_from(backoff: &Backoff) -> Option { + match backoff.timeout.is_zero() { + true => None, + false => Some(tokio::time::Instant::now() + backoff.timeout), + } +} + /// A connection lifecycle transition reported by [`Reconnect::status`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] @@ -204,7 +203,11 @@ impl Reconnect { url: Url, backoff: Backoff, ) -> crate::Result<()> { - let mut retry = kio::time::Backoff::new((&backoff).into()); + // The escalating wait between attempts, and the instant the give-up budget expires. Both + // restart after a session that stayed healthy, so a one-off drop reconnects promptly. A zero + // timeout means no deadline at all: retry for as long as the process lives. + let mut delay = backoff.initial; + let mut deadline = deadline_from(&backoff); let mut last_error: Option = None; loop { @@ -236,7 +239,8 @@ impl Reconnect { // Stayed up past the initial backoff: a healthy session. Reset the backoff // window so a one-off drop reconnects promptly. tracing::warn!(%url, "session closed, reconnecting"); - retry.reset(); + delay = backoff.initial; + deadline = deadline_from(&backoff); last_error = None; } else { // Connected then dropped almost immediately (e.g. the server accepts then @@ -269,8 +273,8 @@ impl Reconnect { } } - tracing::warn!(%url, "reconnecting after backoff"); - if !retry.sleep().await { + let now = tokio::time::Instant::now(); + if deadline.is_some_and(|deadline| now >= deadline) { let timeout = backoff.timeout; let msg = match last_error { Some(err) => format!("reconnect timed out after {timeout:?}: {err}"), @@ -278,6 +282,17 @@ impl Reconnect { }; return Err(Error::Reconnect(msg)); } + + // Jittered so a fleet knocked offline together doesn't reconnect on the same tick, and + // never past the deadline the budget promised. + let mut wait = delay.mul_f64(0.5 + rand::rng().random::() / 2.0); + if let Some(deadline) = deadline { + wait = wait.min(deadline - now); + } + delay = (delay * backoff.multiplier.max(1)).min(backoff.max); + + tracing::warn!(%url, ?wait, "reconnecting after backoff"); + tokio::time::sleep(wait).await; } } diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index e1ac9b59b4..cad8ea69e6 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -51,12 +51,12 @@ http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-moka", "u humantime = "2.3" humantime-serde = "1.1" jsonwebtoken = "11" -kio = { workspace = true, features = ["time"] } moq-native = { workspace = true, default-features = false, features = ["aws-lc-rs", "watch", "tcp"] } moq-net = { workspace = true } moq-stats = { workspace = true } moq-token = { workspace = true, features = ["tokio"] } qmux = { workspace = true, features = ["ws"], optional = true } +rand = "0.10" reqwest = { version = "0.13", default-features = false, features = ["rustls"] } reqwest-middleware = "0.5" rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 1399cfead7..9dcc310309 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -17,6 +17,7 @@ use tracing::Instrument as _; use url::Url; use crate::{AuthToken, nodes::MESH_PREFIX}; +use rand::RngExt; /// How often the discovery loop scans for stale entries. const SWEEP_INTERVAL: Duration = Duration::from_secs(30); @@ -920,10 +921,9 @@ impl Cluster { // A peer is supervised for the life of the relay, so there is no give-up deadline: one that is // unreachable for an hour still has to be redialed when it comes back. Nothing ends this // loop; the escalating delay is what keeps a permanently-dead peer cheap. - let mut config = kio::time::Config::default(); - config.max = tokio::time::Duration::from_secs(300); - config.timeout = tokio::time::Duration::ZERO; - let mut backoff = kio::time::Backoff::new(config); + let base_delay = tokio::time::Duration::from_secs(1); + let max_delay = tokio::time::Duration::from_secs(300); + let mut delay = base_delay; // Sessions shorter than this are treated as churn: we keep backing off // instead of resetting, otherwise a peer that rejects us instantly would @@ -939,7 +939,7 @@ impl Cluster { // stable and reset the backoff on every attempt, so the escalation would never happen. let stable = attempt.connected.is_some_and(|at| at.elapsed() >= stable_threshold); if stable { - backoff.reset(); + delay = base_delay; } if let Err(err) = attempt.result { @@ -949,7 +949,10 @@ impl Cluster { } } - backoff.sleep().await; + // Jittered so a restarting cluster doesn't have every peer redial on the same tick. + let wait = delay.mul_f64(0.5 + rand::rng().random::() / 2.0); + delay = (delay * 2).min(max_delay); + tokio::time::sleep(wait).await; } } diff --git a/rs/moq-rtmp/Cargo.toml b/rs/moq-rtmp/Cargo.toml index 6815049dc6..7d3a88d85b 100644 --- a/rs/moq-rtmp/Cargo.toml +++ b/rs/moq-rtmp/Cargo.toml @@ -35,7 +35,6 @@ bytes = "1" futures = "0.3" hang = { workspace = true } hmac = "0.13" -kio = { workspace = true, features = ["time"] } moq-mux = { workspace = true } moq-net = { workspace = true } rand = "0.10" diff --git a/rs/moq-rtmp/src/server.rs b/rs/moq-rtmp/src/server.rs index 00b7d2f8ce..60d5397e22 100644 --- a/rs/moq-rtmp/src/server.rs +++ b/rs/moq-rtmp/src/server.rs @@ -50,6 +50,7 @@ use tokio::net::{TcpListener, TcpStream}; use crate::Result; use crate::flv; +use rand::RngExt; /// Read buffer size for pulling RTMP chunk-stream bytes off the socket. const READ_BUFFER: usize = 16 * 1024; @@ -216,6 +217,12 @@ impl AsyncWrite for Conn { } } +/// Backoff bounds after a failed `accept`. The listener is supervised for the process's lifetime, +/// so there is no give-up budget: the descriptor pressure or firewall rule behind a failed accept +/// clears on its own, and the next connection resets the escalation. +const ACCEPT_RETRY_MIN: Duration = Duration::from_millis(100); +const ACCEPT_RETRY_MAX: Duration = Duration::from_secs(5); + /// An RTMP server that yields each connection's pending request as a [`Request`]. /// /// Build it with [`bind`](Self::bind), optionally enable RTMPS with @@ -234,10 +241,10 @@ pub struct Server { /// the connection closed or errored before issuing a publish or play. pending: FuturesUnordered>>>, - /// Escalating delay after a failed `accept`. Lives on the server rather than inside - /// [`accept`](Self::accept) so consecutive failures keep escalating across calls, and resets on + /// Delay after a failed `accept`, doubling per consecutive failure. Lives on the server rather + /// than inside [`accept`](Self::accept) so the escalation survives across calls, and resets on /// the next connection that does come in. - accept_backoff: kio::time::Backoff, + accept_delay: Duration, /// While set, `accept` stops asking the listener until this instant. In-flight handshakes keep /// being polled meanwhile: a connection that already got through must not wait out a backoff @@ -250,20 +257,12 @@ impl Server { pub async fn bind(addr: SocketAddr) -> Result { let listener = TcpListener::bind(addr).await?; - // The listener is supervised for the process's lifetime, so there is no give-up budget: the - // descriptor pressure or firewall rule behind a failed accept clears on its own, and the - // next connection resets the escalation. - let mut backoff = kio::time::Config::default(); - backoff.initial = Duration::from_millis(100); - backoff.max = Duration::from_secs(5); - backoff.timeout = Duration::ZERO; - Ok(Self { listener, #[cfg(feature = "tls")] tls: None, pending: FuturesUnordered::new(), - accept_backoff: kio::time::Backoff::new(backoff), + accept_delay: ACCEPT_RETRY_MIN, accept_retry: None, }) } @@ -309,7 +308,7 @@ impl Server { res = self.listener.accept(), if retry.is_none() && self.pending.len() < MAX_PENDING_REQUESTS => match res { Ok((stream, peer)) => { // A connection got through, so whatever the last failure was has cleared. - self.accept_backoff.reset(); + self.accept_delay = ACCEPT_RETRY_MIN; configure_socket(&stream, peer); #[cfg(feature = "tls")] let tls = self.tls.clone(); @@ -357,8 +356,9 @@ impl Server { // rather than slept on here, so the loop keeps serving in-flight handshakes // through the pause. tracing::warn!(%err, "failed to accept RTMP connection; continuing"); - let delay = self.accept_backoff.delay().expect("unlimited accept budget"); - self.accept_retry = Some(tokio::time::Instant::now() + delay); + let wait = self.accept_delay.mul_f64(0.5 + rand::rng().random::() / 2.0); + self.accept_delay = (self.accept_delay * 2).min(ACCEPT_RETRY_MAX); + self.accept_retry = Some(tokio::time::Instant::now() + wait); } }, } From d36a915a56ba0dd1b2567a5b726b9dc080a85b84 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 4 Aug 2026 21:39:27 -0700 Subject: [PATCH 11/13] fix(native): fold a failover's statuses; correct the gst docs `status()` folded a transport race but not an address failover, so a hostname whose addresses all reach a relay that rejects the path with a settled status reported `None` and burned the whole reconnect budget. It now reports one, but only when every raced address answered and none of the answers invites a retry. `doc/bin/gstreamer.md` promised the sink reaches `failed` on "a URL no compiled-in backend can dial". It doesn't: with no classification and `timeout = 0`, only an auth rejection or a settled CONNECT status ends the loop, and a local configuration error retries forever. The page says what actually happens, and what to watch for instead. Co-Authored-By: Claude Opus 5 --- doc/bin/gstreamer.md | 15 ++++++++++----- rs/moq-native/src/noq.rs | 16 +++++++++++++--- rs/moq-native/src/quiche.rs | 16 +++++++++++++--- rs/moq-native/src/quinn.rs | 16 +++++++++++++--- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 6b9b127d38..f3dd53ada4 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -41,11 +41,16 @@ signal when it changes, so you can poll it via `g_object_get` or connect to `not | `estimated-send-bitrate` | uint64 | Estimated send bitrate in bits per second (congestion controller); 0 when unavailable | | `estimated-recv-bitrate` | uint64 | Estimated receive bitrate in bits per second; 0 when unavailable | -`status` distinguishes a transient drop (`disconnected`, the reconnect loop is still retrying) from a -permanent give-up (`failed`), which a bare `connected` bool cannot. The sink retries transport -failures for as long as the pipeline runs, so a relay outage of any length is ridden out; it goes -`failed` only on something a retry cannot clear, such as a rejected token, unusable TLS material, or -a URL no compiled-in backend can dial. +`status` distinguishes a drop the reconnect loop is still retrying (`disconnected`) from a permanent +give-up (`failed`), which a bare `connected` bool cannot. The sink retries for as long as the +pipeline runs, so a relay outage of any length is ridden out. It goes `failed` only on an answer the +relay actually gave that redialing cannot change: a rejected token, or a CONNECT answered with a +status that isn't an invitation to retry. + +Everything else keeps retrying, including local configuration mistakes such as a URL scheme no +compiled-in backend can dial or TLS material that won't load. Those never reach `failed`, so watch +the logs when a sink stays `disconnected` from the very first attempt: a pipeline that has never +connected once is far more likely misconfigured than waiting out an outage. ## Prerequisites diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index 96ee57c91c..01f4e44e0e 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -408,9 +408,19 @@ impl Error { err.status().map(|status| status.as_u16()) } Self::Client(err) => client_status(err), - // One address answering is not the set answering, so a raced dial reports nothing - // rather than letting a single response speak for the rest. - Self::Failover(_) => None, + // Every raced address has to have answered, and answered with something not worth + // repeating, before the set counts as settled: one address refusing says nothing about + // the others, which may simply have been unroutable. + Self::Failover(failures) => { + let mut settled = None; + for failure in failures { + match failure.error.status() { + Some(status) if !crate::error::status_retryable(status) => settled = Some(status), + _ => return None, + } + } + settled + } _ => None, } } diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index 3ed73620d8..6cc28a5793 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -484,9 +484,19 @@ impl Error { err.status().map(|status| status.as_u16()) } Self::ClientConnect(err) => client_status(err), - // One address answering is not the set answering, so a raced dial reports nothing - // rather than letting a single response speak for the rest. - Self::Failover(_) => None, + // Every raced address has to have answered, and answered with something not worth + // repeating, before the set counts as settled: one address refusing says nothing about + // the others, which may simply have been unroutable. + Self::Failover(failures) => { + let mut settled = None; + for failure in failures { + match failure.error.status() { + Some(status) if !crate::error::status_retryable(status) => settled = Some(status), + _ => return None, + } + } + settled + } _ => None, } } diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 90c6effed6..f0a536c3c4 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -421,9 +421,19 @@ impl Error { err.status().map(|status| status.as_u16()) } Self::Client(err) => client_status(err), - // One address answering is not the set answering, so a raced dial reports nothing - // rather than letting a single response speak for the rest. - Self::Failover(_) => None, + // Every raced address has to have answered, and answered with something not worth + // repeating, before the set counts as settled: one address refusing says nothing about + // the others, which may simply have been unroutable. + Self::Failover(failures) => { + let mut settled = None; + for failure in failures { + match failure.error.status() { + Some(status) if !crate::error::status_retryable(status) => settled = Some(status), + _ => return None, + } + } + settled + } _ => None, } } From 580b1cb77e2c60c4cd3243b628f28d0e7e5b50c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 5 Aug 2026 15:40:36 -0700 Subject: [PATCH 12/13] refactor: bound retries by time, not by error type Drop the Error::status()/status_retryable classification from moq-native and moq-hls, along with all the status plumbing through the QUIC backends. Every failure now rides the same jittered backoff, and a short budget is what stops the loop: an ephemeral failure is one that clears within it, so the budget classifies without anyone maintaining a status list. The only remaining short-circuit is the pre-existing is_auth guard. Shorten the budgets to match. Reconnect gives up after 10s (was 5m) with a 5s delay ceiling (was 30s), in Rust and JS. The HLS import budget drops from 300s to 10s. Supervisor loops that have nobody to return an error to (cluster peers, device reopen, RTMP accept) still retry forever, but the cluster ceiling drops from 300s to 10s so a dead peer is loudly broken and a returning one is picked up within seconds. Trim the CLAUDE.md retry guidance to one paragraph stating the policy. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 14 +----- doc/bin/gstreamer.md | 12 ++--- doc/bin/relay/cluster.md | 5 +- js/CLAUDE.md | 2 +- js/net/src/connection/reload.ts | 16 +++--- rs/CLAUDE.md | 2 +- rs/moq-gst/src/sink/session.rs | 13 +++-- rs/moq-hls/src/error.rs | 37 -------------- rs/moq-hls/src/import.rs | 39 ++++----------- rs/moq-hls/src/lib.rs | 1 - rs/moq-native/src/error.rs | 40 --------------- rs/moq-native/src/noq.rs | 52 +++---------------- rs/moq-native/src/quiche.rs | 52 +++---------------- rs/moq-native/src/quinn.rs | 89 +++------------------------------ rs/moq-native/src/reconnect.rs | 38 +++++++------- rs/moq-native/src/websocket.rs | 11 ---- rs/moq-relay/src/cluster.rs | 7 +-- 17 files changed, 81 insertions(+), 349 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 48490910bc..c385a2b9a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,19 +99,7 @@ The rename/removal rationale lives in the commit message and PR description, not ## Retries -Retrying is the reflex that hides bugs, so a new retry loop has to answer three questions in the code, not in the reviewer's head. - -- **How long between attempts?** Capped exponential backoff with jitter, never a fixed delay. Three lines at the call site: draw the wait from the top half of the current window (`delay.mul_f64(0.5 + rand::rng().random::() / 2.0)`), sleep it, then `delay = (delay * 2).min(MAX)`. There is deliberately no shared `Backoff` type. Each loop wants a different subset (most want no budget at all), the escalation is smaller than the abstraction over it, and a general one has to accept an arbitrary `max` it then has to defend against. -- **When does it stop?** A deadline or an attempt budget, and that budget is what ends the loop. Unlimited retries belong only to a supervisor whose job is to outlive an outage (a reconnecting publisher, a cluster peer, a listener), where the escalating delay is what keeps a permanently-dead target cheap. -- **Who owns the budget?** Exactly one layer. An outer supervisor that rebuilds an inner retry loop resets its backoff to the initial delay, so the escalation never happens and a fixed-interval hammer wears an exponential costume. Watch the inner loop's terminal signal instead of restarting it. - -**Don't classify errors as retryable.** It is tempting to give an error type an `is_retryable()` and skip the wait when the answer is no. Resist it: deciding whether a failure is permanent means guessing, the guess has to stay correct as every wrapped error type evolves, and getting it wrong either strands a connection a retry would have recovered or hammers a dead one. The budget already bounds the damage; the only thing classification buys is surfacing a config error sooner. - -The exception is an answer a peer actually gave, where the protocol defines the meaning. An HTTP status is the one we have: `moq_native::Error::status` and `moq_hls::Error::status` report the status a server sent, and each crate decides what to do with it (`408`, `429`, `502`, `503`, and `504` are worth another try). That is reading a response, not inferring intent from a failure. - -Resetting a backoff is its own claim: only after an outcome that says the earlier failures no longer describe reality (a session that stayed healthy, a request that succeeded, a changed destination). Resetting on an attempt that failed immediately turns escalation into a tight loop. - -Not every wait is a retry. Periodic refreshes, readiness probes, stream reads, alternate-address races, and test synchronization don't repeat a failed operation, so none of this applies to them. +Fail fast; retry only what a few seconds can fix. Every retry loop uses capped exponential backoff with jitter, inlined at the call site (no shared `Backoff` type), and is bounded by *time*, not by error type: a short budget (~10s), then surface the last real error. Never classify errors as retryable (`is_retryable()`, HTTP status lists); an ephemeral failure is one that clears within the budget, so the budget is the classifier. The only unbounded loops are process-lifetime supervisors with nobody to return an error to (cluster peers, device reopen, accept loops); they retry forever but cap the delay at seconds and warn per attempt, loudly broken rather than silently parked. Exactly one layer owns a retry: an outer loop that rebuilds an inner one resets its escalation, so watch the inner loop's terminal signal instead. ## Root Cause First diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index f3dd53ada4..af18550795 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -43,14 +43,10 @@ signal when it changes, so you can poll it via `g_object_get` or connect to `not `status` distinguishes a drop the reconnect loop is still retrying (`disconnected`) from a permanent give-up (`failed`), which a bare `connected` bool cannot. The sink retries for as long as the -pipeline runs, so a relay outage of any length is ridden out. It goes `failed` only on an answer the -relay actually gave that redialing cannot change: a rejected token, or a CONNECT answered with a -status that isn't an invitation to retry. - -Everything else keeps retrying, including local configuration mistakes such as a URL scheme no -compiled-in backend can dial or TLS material that won't load. Those never reach `failed`, so watch -the logs when a sink stays `disconnected` from the very first attempt: a pipeline that has never -connected once is far more likely misconfigured than waiting out an outage. +pipeline runs, so a relay outage of any length is ridden out; it goes `failed` only on a rejected +token, the one answer redialing cannot change. Everything else keeps retrying, so watch the logs +when a sink stays `disconnected` from the very first attempt: a pipeline that has never connected +once is far more likely misconfigured than waiting out an outage. ## Prerequisites diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 4c9db5079b..7c5916c828 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -120,9 +120,8 @@ See [Authentication](/bin/relay/auth) for the full setup. Peers are redialed indefinitely, with exponential backoff and jitter so a restarting cluster doesn't reconnect in lockstep. That includes a peer that rejects us: a bad token logs `cluster peer error; will retry` on every attempt rather than giving up, so watch for a peer that never reaches -`cluster peer session closed`. The delay escalates to five minutes, which is what keeps a -permanently-rejected peer cheap rather than noisy. A session that stays up for ten seconds is -treated as healthy and clears the escalation, so a peer that comes back redials promptly. +`cluster peer session closed`. The delay escalates to ten seconds at most, so a dead or rejecting +peer stays loudly visible in the logs and a returning one is picked up within seconds. ## Migration from older configs diff --git a/js/CLAUDE.md b/js/CLAUDE.md index 323ca22c2d..37f6898f33 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -80,7 +80,7 @@ Plain custom elements built directly on `@moq/signals`, no framework (except moq ## Conventions -- **Retry loops inline their backoff** (root Retries explains the why). Escalate a local delay toward a `max`, jitter each wait (`delay * (0.5 + Math.random() / 2)`), and hand it to `effect.timer`. There is no shared `Backoff` export, and don't try to classify which thrown values are worth retrying: the platform hands back `WebTransportError`, `DOMException`, `AggregateError`, and bare `Error`s interchangeably, so a budget or an attempt count is what stops the loop. +- **Retry loops inline their backoff** (root Retries has the policy). Escalate a local delay toward a `max`, jitter each wait (`delay * (0.5 + Math.random() / 2)`), and hand it to `effect.timer`; a short time or attempt budget stops the loop, never a classification of the thrown value. - **Avoid callback parameters.** A function taking a `fn`/`create`/`onXxx` to invoke later reads poorly and hides control flow. Prefer returning a value the caller acts on, exposing a method or getter, or splitting into a couple of small calls the caller sequences itself (e.g. a cache `get()` then `insert(value)`, not `getOrCreate(key, () => value)`). Reserve callbacks for genuine event/subscription sinks where there is no alternative (`effect.subscribe`, DOM listeners, `Signal` subscriptions). - ESM only (`"type": "module"`). Relative imports include the `.ts`/`.tsx` extension in the lower-level packages (`net`, `signals`, `hang`); `rewriteRelativeImportExtensions` in `tsconfig.json` rewrites them to `.js` on build. Some higher-level packages (watch/publish) still omit extensions, so match the file you are editing. - Document every exported symbol and add a top-of-file `@module` doc block to each entrypoint (root convention; the published JSR/`.d.ts` docs render these). Use `@public` on the load-bearing classes. diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index e1b2f0d71e..e8b0bd9969 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -20,13 +20,12 @@ export type ReloadDelay = { /** The multiplier for the delay (default: 2). */ multiplier: number; - /** The maximum delay in milliseconds (default: 30000). */ + /** The maximum delay in milliseconds (default: 5000). */ max: DOMHighResTimeStamp; /** * Maximum total time in milliseconds to spend retrying before giving up (default: - * 300000, 5 minutes). Resets after each successful connection. Set to 0 for - * unlimited retries. + * 10000). Resets after each successful connection. Set to 0 for unlimited retries. */ timeout?: DOMHighResTimeStamp; }; @@ -43,8 +42,13 @@ export type ReloadProps = Omit & { delay?: ReloadDelay; }; -/** How long to keep retrying before giving up, when {@link ReloadDelay.timeout} is unset. */ -const DEFAULT_TIMEOUT = 300000; +/** + * How long to keep retrying before giving up, when {@link ReloadDelay.timeout} is unset. + * + * Short on purpose: a failure that clears within it was transient, and one that doesn't should + * surface as an error rather than leave the page silently reconnecting for minutes. + */ +const DEFAULT_TIMEOUT = 10000; /** Current state of a {@link Reload} connection. */ export type ReloadStatus = "connecting" | "connected" | "disconnected"; @@ -116,7 +120,7 @@ export class Reload { constructor(props?: ReloadProps) { this.url = Signal.from(props?.url); this.enabled = Signal.from(props?.enabled ?? false); - this.delay = props?.delay ?? { initial: 1000, multiplier: 2, max: 30000 }; + this.delay = props?.delay ?? { initial: 1000, multiplier: 2, max: 5000 }; this.webtransport = props?.webtransport; this.websocket = props?.websocket; this.discovery = props?.discovery; diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 7e8654627d..633248edb3 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,7 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions -- **Retry loops inline their backoff** (root Retries explains the why). Escalate a local `Duration` toward a `const MAX`, jitter each wait, and sleep it. Don't reach for a shared `Backoff` type: one existed briefly and was removed, because four of the six call sites wanted no give-up budget and the general version's caller-supplied `max` was itself the source of an overflow panic. A loop that needs a deadline tracks a `tokio::time::Instant` next to its delay. And don't add an `is_retryable()` to an error type: the budget, or classification-free escalation, is what stops a loop. The one thing worth reading off a failure is a status a peer actually sent, via the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. +- **Retry loops inline their backoff** (root Retries has the policy). Escalate a local `Duration` toward a `const MAX` (seconds, not minutes), jitter each wait (`delay.mul_f64(0.5 + rand::rng().random::() / 2.0)`), and track the give-up deadline as a `tokio::time::Instant` next to the delay. No shared backoff helper, and no `is_retryable()` on error types. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index 89289bfec6..4fd9aa96dd 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -42,8 +42,7 @@ pub enum ConnectionStatus { /// A session is connected and publishing. #[enum_value(name = "Connected: session established", nick = "connected")] Connected, - /// The reconnect loop gave up permanently (an auth rejection, or a CONNECT status that isn't an - /// invitation to retry). Terminal. + /// The reconnect loop gave up permanently (an auth rejection). Terminal. #[enum_value(name = "Failed: connection rejected, gave up", nick = "failed")] Failed, } @@ -139,11 +138,11 @@ impl Session { // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC // outages of any length, which is the trade this element wants: a pipeline nobody is watching - // should still be publishing when the relay comes back. The loop still ends on the two answers - // a server states outright (an auth rejection, or a CONNECT status that isn't an invitation to - // retry), posting the bus error below. During an outage the pad threads keep writing (bounded - // by moq-net's per-group eviction) and the relay catches up from a group boundary on - // reconnect. A bounded policy is available via `ClientConfig::backoff`. + // should still be publishing when the relay comes back. The loop still ends on an auth + // rejection (the one answer redialing cannot change), posting the bus error below. During an + // outage the pad threads keep writing (bounded by moq-net's per-group eviction) and the relay + // catches up from a group boundary on reconnect. A bounded policy is available via + // `ClientConfig::backoff`. let mut config = moq_native::ClientConfig::default(); config.tls.disable_verify = Some(settings.tls_disable_verify); config.backoff.timeout = std::time::Duration::ZERO; diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index 50055295e3..efa4b09532 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -18,15 +18,6 @@ impl std::fmt::Display for SequenceKind { } } -/// Whether an HTTP response status means "ask again later". -/// -/// A response that arrived is the server's answer, and only this narrow set invites another -/// attempt: request timeout, rate limit, and the gateway/overload statuses. Every other status, -/// `404` and `403` included, is settled. -pub(crate) fn status_retryable(status: u16) -> bool { - matches!(status, 408 | 429 | 502 | 503 | 504) -} - /// Errors produced by the HLS <-> MoQ gateway (import and export). #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] @@ -133,20 +124,6 @@ pub enum Error { Other(std::sync::Arc), } -impl Error { - /// The HTTP status the origin answered with, if it answered with one at all. - /// - /// The import loop consults it: a `503` on a playlist fetch is worth another pass, a `404` is - /// the origin's settled answer. Nothing else here is classified; a failure with no status falls - /// through to the backoff budget. - pub fn status(&self) -> Option { - match self { - Self::Reqwest(err) => err.status().map(|status| status.as_u16()), - _ => None, - } - } -} - impl From for Error { fn from(err: reqwest::Error) -> Self { Error::Reqwest(std::sync::Arc::new(err)) @@ -167,17 +144,3 @@ impl From for Error { /// Convenience alias for results from the HLS gateway. pub type Result = std::result::Result; - -#[cfg(test)] -mod tests { - use super::*; - - /// The import loop consults this, so an origin's settled answer has to reach it intact. - #[test] - fn an_http_failure_reports_its_status() { - // A failure the origin never answered carries no status, so the budget decides instead. - assert_eq!(Error::NoVariants.status(), None); - assert_eq!(Error::ParsePlaylist("not a playlist".to_string()).status(), None); - assert_eq!(Error::Moq(moq_net::Error::Transport("lost".to_string())).status(), None); - } -} diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index ae679facaa..a592d0607f 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -33,13 +33,12 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// Backoff bounds for retrying a failed import step, so a transient upstream error (a 503, a /// dropped connection) doesn't tear down the whole import. /// -/// Bounded by a give-up budget: an origin that has been unreachable for minutes is an outage the -/// caller should hear about, not one the import should paper over indefinitely while publishing -/// nothing. The ceiling is low because a live playlist window is measured in seconds and a longer -/// wait would blow past it anyway. +/// The budget is short on purpose: a failure that clears within it was transient, and one that +/// doesn't is an outage the caller should hear about rather than one the import papers over while +/// publishing nothing. No error is classified; the budget is the filter. const STEP_RETRY_MIN: Duration = Duration::from_secs(1); -const STEP_RETRY_MAX: Duration = Duration::from_secs(10); -const STEP_RETRY_BUDGET: Duration = Duration::from_secs(300); +const STEP_RETRY_MAX: Duration = Duration::from_secs(5); +const STEP_RETRY_BUDGET: Duration = Duration::from_secs(10); /// How far back from the live edge to start when (re-)anchoring to a playlist window. /// @@ -100,14 +99,11 @@ struct StepOutcome { /// nothing to add this pass, not a failure. Counting segments instead would read a quiet /// playlist as a dead one. ok: usize, - /// A rendition failure from this step, if any. + /// The first rendition failure from this step, if any. /// /// [`OnError::Warn`] keeps the other renditions going after one fails, so a step can report /// `Ok` having imported nothing at all. The loop needs to tell that apart from a quiet playlist /// with no new segments, or it treats a permanently broken source as steady progress. - /// - /// When several renditions fail, this is the one whose failure another pass could still clear, - /// so a single permanently-dead variant doesn't end an import the others could still serve. failed: Option, } @@ -612,10 +608,9 @@ impl Import { /// Run the import loop until cancelled. /// - /// A failed step is logged and retried with escalating backoff, and the import ends once the - /// backoff budget is spent, so a broken source surfaces instead of looping forever. The one - /// shortcut is an HTTP status the origin actually sent: a `404` playlist ends the import - /// immediately, since no amount of waiting turns it into a `200`. + /// A failed step is logged and retried with escalating backoff, and the import ends with the + /// last error once the backoff budget is spent, so a broken source surfaces within seconds + /// instead of looping forever. pub async fn run(&mut self) -> Result<()> { let mut delay = STEP_RETRY_MIN; let mut deadline = tokio::time::Instant::now() + STEP_RETRY_BUDGET; @@ -645,11 +640,6 @@ impl Import { deadline = tokio::time::Instant::now() + STEP_RETRY_BUDGET; outcome } - // A status the origin actually sent is its answer: a 404 playlist is not going to - // become a 200 on the next pass. Everything else rides the backoff budget. - Err(err) if err.status().is_some_and(|status| !crate::status_retryable(status)) => { - return Err(err); - } Err(err) => { let now = tokio::time::Instant::now(); if now >= deadline { @@ -725,15 +715,8 @@ impl Import { // drop the rest or abort the whole step. OnError::Warn => { warn!(label = %track.label, %err, "rendition import step failed, will retry"); - // Prefer a failure another pass could still clear. The import is worth - // continuing as long as *any* rendition might come back, even when another is - // permanently gone, so one dead variant must not end an import the rest could - // still serve. Keeping whichever error came last instead would make the - // outcome depend on rendition order. - let recoverable = err.status().is_none_or(crate::status_retryable); - if recoverable || failed.is_none() { - failed = Some(err); - } + // Keep the first failure so the outcome doesn't depend on rendition order. + failed.get_or_insert(err); } }, } diff --git a/rs/moq-hls/src/lib.rs b/rs/moq-hls/src/lib.rs index 1708142543..5b97dd6cda 100644 --- a/rs/moq-hls/src/lib.rs +++ b/rs/moq-hls/src/lib.rs @@ -25,7 +25,6 @@ pub mod import; #[cfg(feature = "server")] pub mod server; -pub(crate) use error::status_retryable; pub use error::*; #[cfg(feature = "server")] pub use server::Server; diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index da4251168c..434a85ae7b 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -1,14 +1,5 @@ use std::sync::Arc; -/// Whether an HTTP response status means "ask again later". -/// -/// A response that arrived is the server's answer, and only this narrow set invites another -/// attempt: request timeout, rate limit, and the gateway/overload statuses. Every other status, -/// `404` and `403` included, is settled. -pub(crate) fn status_retryable(status: u16) -> bool { - matches!(status, 408 | 429 | 502 | 503 | 504) -} - /// Errors produced while configuring or establishing native MoQ connections. /// /// Backend-specific failures live in per-backend error types ([`crate::tls::Error`], @@ -154,37 +145,6 @@ impl Error { pub fn is_auth(&self) -> bool { self.connect_error().is_some_and(|err| err.is_auth()) } - - /// The HTTP status a server answered a connection attempt with, if it answered with one at all. - /// - /// `None` covers everything else: a dial that never got a response, a QUIC handshake that - /// failed, a URL we couldn't parse. Only a status the peer actually sent shows up here, and - /// whether it invites another attempt is the caller's call (`408`, `429`, `502`, `503`, and - /// `504` are the ones worth repeating). This deliberately does not try to say whether some - /// *other* kind of failure is worth retrying; that's a guess, and a backoff budget bounds it - /// instead. - pub fn status(&self) -> Option { - match self { - // A race is only settled when both halves were answered, and answered with something not - // worth repeating: one transport being refused says nothing about the other, so a `404` - // over QUIC alongside a dead WebSocket is still just a failed dial. - #[cfg(feature = "websocket")] - Self::TransportRace { quic, websocket } => match (quic.status(), websocket.status()) { - (Some(quic), Some(websocket)) if !status_retryable(quic) && !status_retryable(websocket) => Some(quic), - _ => None, - }, - - #[cfg(feature = "quinn")] - Self::Quinn(err) => err.status(), - #[cfg(feature = "noq")] - Self::Noq(err) => err.status(), - #[cfg(feature = "quiche")] - Self::Quiche(err) => err.status(), - #[cfg(feature = "websocket")] - Self::WebSocket(err) => err.status(), - _ => None, - } - } } // The wrapped sources aren't `Clone`, so `#[from]` can't store them behind `Arc` diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index 01f4e44e0e..8ca8e26786 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -397,33 +397,6 @@ impl Error { _ => None, } } - - /// The HTTP status a server answered with, if it answered with one at all. - /// - /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the - /// WebTransport CONNECT response. See [`crate::Error::status`]. - pub(crate) fn status(&self) -> Option { - match self { - Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { - err.status().map(|status| status.as_u16()) - } - Self::Client(err) => client_status(err), - // Every raced address has to have answered, and answered with something not worth - // repeating, before the set counts as settled: one address refusing says nothing about - // the others, which may simply have been unroutable. - Self::Failover(failures) => { - let mut settled = None; - for failure in failures { - match failure.error.status() { - Some(status) if !crate::error::status_retryable(status) => settled = Some(status), - _ => return None, - } - } - settled - } - _ => None, - } - } } fn map_client_error(err: web_transport_noq::ClientError) -> Error { @@ -435,35 +408,26 @@ fn map_client_error(err: web_transport_noq::ClientError) -> Error { } fn classify_client_error(err: &web_transport_noq::ClientError) -> Option { - client_status(err).and_then(crate::ConnectError::from_status_u16) -} - -/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at -/// all (as opposed to the connection failing underneath the request). -/// -/// Both classifications read this: [`classify_client_error`] turns an auth status into a -/// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults -/// the status. A `404` or `405` is the server's settled answer, so retrying -/// it just burns the reconnect budget on a URL that will never work. -fn client_status(err: &web_transport_noq::ClientError) -> Option { match err { - web_transport_noq::ClientError::HttpError(err) => connect_status(err), + web_transport_noq::ClientError::HttpError(err) => classify_connect_error(err), _ => None, } } -fn connect_status(err: &web_transport_noq::ConnectError) -> Option { +fn classify_connect_error(err: &web_transport_noq::ConnectError) -> Option { match err { - web_transport_noq::ConnectError::ErrorStatus(status) => Some(status.as_u16()), - web_transport_noq::ConnectError::ProtoError(err) => proto_status(err), + web_transport_noq::ConnectError::ErrorStatus(status) => crate::ConnectError::from_status_u16(status.as_u16()), + web_transport_noq::ConnectError::ProtoError(err) => classify_proto_error(err), _ => None, } } -fn proto_status(err: &web_transport_noq::proto::ConnectError) -> Option { +fn classify_proto_error(err: &web_transport_noq::proto::ConnectError) -> Option { match err { web_transport_noq::proto::ConnectError::ErrorStatus(status) - | web_transport_noq::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), + | web_transport_noq::proto::ConnectError::WrongStatus(Some(status)) => { + crate::ConnectError::from_status_u16(status.as_u16()) + } _ => None, } } diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index 6cc28a5793..f93e65f45f 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -473,33 +473,6 @@ impl Error { _ => None, } } - - /// The HTTP status a server answered with, if it answered with one at all. - /// - /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the - /// WebTransport CONNECT response. See [`crate::Error::status`]. - pub(crate) fn status(&self) -> Option { - match self { - Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { - err.status().map(|status| status.as_u16()) - } - Self::ClientConnect(err) => client_status(err), - // Every raced address has to have answered, and answered with something not worth - // repeating, before the set counts as settled: one address refusing says nothing about - // the others, which may simply have been unroutable. - Self::Failover(failures) => { - let mut settled = None; - for failure in failures { - match failure.error.status() { - Some(status) if !crate::error::status_retryable(status) => settled = Some(status), - _ => return None, - } - } - settled - } - _ => None, - } - } } fn map_client_error(err: web_transport_quiche::ClientError) -> Error { @@ -511,35 +484,26 @@ fn map_client_error(err: web_transport_quiche::ClientError) -> Error { } fn classify_client_error(err: &web_transport_quiche::ClientError) -> Option { - client_status(err).and_then(crate::ConnectError::from_status_u16) -} - -/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at -/// all (as opposed to the connection failing underneath the request). -/// -/// Both classifications read this: [`classify_client_error`] turns an auth status into a -/// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults -/// the status. A `404` or `405` is the server's settled answer, so retrying -/// it just burns the reconnect budget on a URL that will never work. -fn client_status(err: &web_transport_quiche::ClientError) -> Option { match err { - web_transport_quiche::ClientError::Connect(err) => connect_status(err), + web_transport_quiche::ClientError::Connect(err) => classify_connect_error(err), _ => None, } } -fn connect_status(err: &web_transport_quiche::h3::ConnectError) -> Option { +fn classify_connect_error(err: &web_transport_quiche::h3::ConnectError) -> Option { match err { - web_transport_quiche::h3::ConnectError::Status(status) => Some(status.as_u16()), - web_transport_quiche::h3::ConnectError::Proto(err) => proto_status(err), + web_transport_quiche::h3::ConnectError::Status(status) => crate::ConnectError::from_status_u16(status.as_u16()), + web_transport_quiche::h3::ConnectError::Proto(err) => classify_proto_error(err), _ => None, } } -fn proto_status(err: &web_transport_quiche::proto::ConnectError) -> Option { +fn classify_proto_error(err: &web_transport_quiche::proto::ConnectError) -> Option { match err { web_transport_quiche::proto::ConnectError::ErrorStatus(status) - | web_transport_quiche::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), + | web_transport_quiche::proto::ConnectError::WrongStatus(Some(status)) => { + crate::ConnectError::from_status_u16(status.as_u16()) + } _ => None, } } diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index f0a536c3c4..1ba24ba6e3 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -410,33 +410,6 @@ impl Error { _ => None, } } - - /// The HTTP status a server answered with, if it answered with one at all. - /// - /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the - /// WebTransport CONNECT response. See [`crate::Error::status`]. - pub(crate) fn status(&self) -> Option { - match self { - Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { - err.status().map(|status| status.as_u16()) - } - Self::Client(err) => client_status(err), - // Every raced address has to have answered, and answered with something not worth - // repeating, before the set counts as settled: one address refusing says nothing about - // the others, which may simply have been unroutable. - Self::Failover(failures) => { - let mut settled = None; - for failure in failures { - match failure.error.status() { - Some(status) if !crate::error::status_retryable(status) => settled = Some(status), - _ => return None, - } - } - settled - } - _ => None, - } - } } fn map_client_error(err: web_transport_quinn::ClientError) -> Error { @@ -448,35 +421,26 @@ fn map_client_error(err: web_transport_quinn::ClientError) -> Error { } fn classify_client_error(err: &web_transport_quinn::ClientError) -> Option { - client_status(err).and_then(crate::ConnectError::from_status_u16) -} - -/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at -/// all (as opposed to the connection failing underneath the request). -/// -/// Both classifications read this: [`classify_client_error`] turns an auth status into a -/// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults -/// the status. A `404` or `405` is the server's settled answer, so retrying -/// it just burns the reconnect budget on a URL that will never work. -fn client_status(err: &web_transport_quinn::ClientError) -> Option { match err { - web_transport_quinn::ClientError::HttpError(err) => connect_status(err), + web_transport_quinn::ClientError::HttpError(err) => classify_connect_error(err), _ => None, } } -fn connect_status(err: &web_transport_quinn::ConnectError) -> Option { +fn classify_connect_error(err: &web_transport_quinn::ConnectError) -> Option { match err { - web_transport_quinn::ConnectError::ErrorStatus(status) => Some(status.as_u16()), - web_transport_quinn::ConnectError::ProtoError(err) => proto_status(err), + web_transport_quinn::ConnectError::ErrorStatus(status) => crate::ConnectError::from_status_u16(status.as_u16()), + web_transport_quinn::ConnectError::ProtoError(err) => classify_proto_error(err), _ => None, } } -fn proto_status(err: &web_transport_quinn::proto::ConnectError) -> Option { +fn classify_proto_error(err: &web_transport_quinn::proto::ConnectError) -> Option { match err { web_transport_quinn::proto::ConnectError::ErrorStatus(status) - | web_transport_quinn::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), + | web_transport_quinn::proto::ConnectError::WrongStatus(Some(status)) => { + crate::ConnectError::from_status_u16(status.as_u16()) + } _ => None, } } @@ -722,43 +686,6 @@ impl quinn::ConnectionIdGenerator for ServerIdGenerator { mod tests { use super::*; - fn connect_rejected(status: u16) -> Error { - Error::Client(web_transport_quinn::ClientError::HttpError( - web_transport_quinn::ConnectError::ErrorStatus( - web_transport_quinn::http::StatusCode::from_u16(status).unwrap(), - ), - )) - } - - /// A CONNECT the relay answered carries its status through to the caller, so a wrong path or an - /// endpoint that doesn't speak WebTransport can surface immediately rather than after the whole - /// reconnect budget. - #[test] - fn a_rejected_connect_reports_its_status() { - for status in [400, 404, 405, 410, 501] { - assert_eq!(connect_rejected(status).status(), Some(status)); - assert!( - !crate::error::status_retryable(status), - "{status} should stop the reconnect loop" - ); - } - - for status in [408, 429, 502, 503, 504] { - assert_eq!(connect_rejected(status).status(), Some(status)); - assert!(crate::error::status_retryable(status), "{status} should be retried"); - } - - // Auth is peeled off into its own variant before reaching the generic client arm. - assert_eq!( - connect_rejected(401).connect_error(), - Some(crate::ConnectError::Unauthorized) - ); - assert_eq!( - connect_rejected(403).connect_error(), - Some(crate::ConnectError::Forbidden) - ); - } - /// Build a controller from each family's factory and downcast it to the /// concrete quinn implementation it must map to. #[test] diff --git a/rs/moq-native/src/reconnect.rs b/rs/moq-native/src/reconnect.rs index 4a7476e598..300fb4585a 100644 --- a/rs/moq-native/src/reconnect.rs +++ b/rs/moq-native/src/reconnect.rs @@ -14,10 +14,11 @@ use crate::{Client, Error}; /// This decides how long to wait between reconnect attempts and when to give up. The delays carry /// jitter, so a fleet knocked offline together doesn't reconnect in lockstep. /// -/// [`timeout`](Self::timeout) is what ends a hopeless loop, not a judgment about the error: the only -/// failures short-circuited are the ones a server states outright (an auth rejection, or a CONNECT -/// status that isn't an invitation to retry). A zero timeout removes that backstop, so it belongs -/// only where an unattended process must outlive an outage of any length. +/// [`timeout`](Self::timeout) is what ends a hopeless loop, not a judgment about the error: every +/// failure is retried the same way (except an auth rejection, which a redial cannot change), and +/// the short default budget is what surfaces a broken target instead of hiding it. A zero timeout +/// removes that backstop, so it belongs only where an unattended process must outlive an outage of +/// any length. #[derive(Clone, Debug, clap::Args, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] @@ -41,7 +42,7 @@ pub struct Backoff { #[arg( id = "backoff-max", long, - default_value = "30s", + default_value = "5s", env = "MOQ_BACKOFF_MAX", value_parser = humantime::parse_duration, )] @@ -55,7 +56,7 @@ pub struct Backoff { #[arg( id = "backoff-timeout", long, - default_value = "5m", + default_value = "10s", env = "MOQ_BACKOFF_TIMEOUT", value_parser = humantime::parse_duration, )] @@ -68,8 +69,8 @@ impl Default for Backoff { Self { initial: Duration::from_secs(1), multiplier: 2, - max: Duration::from_secs(30), - timeout: Duration::from_secs(300), + max: Duration::from_secs(5), + timeout: Duration::from_secs(10), } } } @@ -117,8 +118,8 @@ struct State { status: Option, /// The negotiated MoQ version of the live session, or `None` when disconnected. version: Option, - /// Set when the reconnect loop permanently gives up: the backoff timeout expiring, or a server - /// answer that redialing cannot change. + /// Set when the reconnect loop permanently gives up: the backoff timeout expiring, or an auth + /// rejection that redialing cannot change. error: Option, /// The currently-connected session, or `None` while reconnecting. Read by /// [`ConnectionStatsReader`] to snapshot live connection stats. @@ -257,18 +258,13 @@ impl Reconnect { } } Err(err) => { - // The two answers a server can give that redialing cannot change: it rejected our - // credentials, or it answered the CONNECT with a status that isn't an invitation - // to come back. Everything else falls through to the backoff, whose budget is - // what eventually stops the loop. + // An auth rejection is the one answer redialing cannot change. Everything else + // falls through to the backoff, whose budget is what stops the loop: a failure + // that clears within the budget was transient, and one that doesn't surfaces as + // the last real error rather than being guessed at up front. if err.is_auth() { return Err(err); } - if let Some(status) = err.status() - && !crate::error::status_retryable(status) - { - return Err(err); - } last_error = Some(err); } } @@ -459,8 +455,8 @@ mod tests { let backoff = Backoff::default(); assert_eq!(backoff.initial, Duration::from_secs(1)); assert_eq!(backoff.multiplier, 2); - assert_eq!(backoff.max, Duration::from_secs(30)); - assert_eq!(backoff.timeout, Duration::from_secs(300)); + assert_eq!(backoff.max, Duration::from_secs(5)); + assert_eq!(backoff.timeout, Duration::from_secs(10)); } /// The linger outlives the give-up timeout (so the reconnect error surfaces diff --git a/rs/moq-native/src/websocket.rs b/rs/moq-native/src/websocket.rs index f80a262077..c1fb1d1433 100644 --- a/rs/moq-native/src/websocket.rs +++ b/rs/moq-native/src/websocket.rs @@ -223,17 +223,6 @@ impl Error { _ => None, } } - - /// The HTTP status the server answered the upgrade with, if it answered with one at all. - /// - /// qmux surfaces a non-101 WebSocket upgrade response as `Http(status)`. See - /// [`crate::Error::status`]. - pub(crate) fn status(&self) -> Option { - match self { - Self::Connect(qmux::Error::Http(status)) => Some(*status), - _ => None, - } - } } /// Listens for incoming WebSocket connections on a TCP port. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 9dcc310309..355aaccb6e 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -919,10 +919,11 @@ impl Cluster { } // A peer is supervised for the life of the relay, so there is no give-up deadline: one that is - // unreachable for an hour still has to be redialed when it comes back. Nothing ends this - // loop; the escalating delay is what keeps a permanently-dead peer cheap. + // unreachable for an hour still has to be redialed when it comes back. The ceiling stays low + // so a returning peer is picked up within seconds, and the warn below fires at least that + // often, so a dead peer is loudly broken rather than silently parked. let base_delay = tokio::time::Duration::from_secs(1); - let max_delay = tokio::time::Duration::from_secs(300); + let max_delay = tokio::time::Duration::from_secs(10); let mut delay = base_delay; // Sessions shorter than this are treated as churn: we keep backing off From 7e1ead18e420bea44b167e1ea50a22cf62f1fb2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 5 Aug 2026 16:19:06 -0700 Subject: [PATCH 13/13] refactor: restore the HTTP-status shortcut on the short budgets An HTTP status is an answer the origin actually sent, so it short-circuits the backoff: a 404 fails immediately instead of burning the budget, while 408/429 and the gateway statuses ride it. This restores Error::status() and status_retryable on moq-native and moq-hls (and the status plumbing through the QUIC backends) on top of the fail-fast defaults. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 2 +- doc/bin/gstreamer.md | 9 ++-- rs/CLAUDE.md | 2 +- rs/moq-gst/src/sink/session.rs | 13 ++--- rs/moq-hls/src/error.rs | 37 ++++++++++++++ rs/moq-hls/src/import.rs | 23 +++++++-- rs/moq-hls/src/lib.rs | 1 + rs/moq-native/src/error.rs | 40 +++++++++++++++ rs/moq-native/src/noq.rs | 52 +++++++++++++++++--- rs/moq-native/src/quiche.rs | 52 +++++++++++++++++--- rs/moq-native/src/quinn.rs | 89 +++++++++++++++++++++++++++++++--- rs/moq-native/src/reconnect.rs | 27 ++++++----- rs/moq-native/src/websocket.rs | 22 +++++---- 13 files changed, 308 insertions(+), 61 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c385a2b9a9..842cc25a40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,7 @@ The rename/removal rationale lives in the commit message and PR description, not ## Retries -Fail fast; retry only what a few seconds can fix. Every retry loop uses capped exponential backoff with jitter, inlined at the call site (no shared `Backoff` type), and is bounded by *time*, not by error type: a short budget (~10s), then surface the last real error. Never classify errors as retryable (`is_retryable()`, HTTP status lists); an ephemeral failure is one that clears within the budget, so the budget is the classifier. The only unbounded loops are process-lifetime supervisors with nobody to return an error to (cluster peers, device reopen, accept loops); they retry forever but cap the delay at seconds and warn per attempt, loudly broken rather than silently parked. Exactly one layer owns a retry: an outer loop that rebuilds an inner one resets its escalation, so watch the inner loop's terminal signal instead. +Fail fast; retry only what a few seconds can fix. Every retry loop uses capped exponential backoff with jitter, inlined at the call site (no shared `Backoff` type), and is bounded by *time*, not by error type: a short budget (~10s), then surface the last real error. Don't classify errors as retryable (`is_retryable()`); an ephemeral failure is one that clears within the budget, so the budget is the classifier. The one exception is an answer a peer actually sent, where the protocol defines the meaning: an HTTP status short-circuits (a `404` fails immediately; `408`/`429`/gateway statuses ride the backoff) via the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. The only unbounded loops are process-lifetime supervisors with nobody to return an error to (cluster peers, device reopen, accept loops); they retry forever but cap the delay at seconds and warn per attempt, loudly broken rather than silently parked. Exactly one layer owns a retry: an outer loop that rebuilds an inner one resets its escalation, so watch the inner loop's terminal signal instead. ## Root Cause First diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index af18550795..028ab17971 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -43,10 +43,11 @@ signal when it changes, so you can poll it via `g_object_get` or connect to `not `status` distinguishes a drop the reconnect loop is still retrying (`disconnected`) from a permanent give-up (`failed`), which a bare `connected` bool cannot. The sink retries for as long as the -pipeline runs, so a relay outage of any length is ridden out; it goes `failed` only on a rejected -token, the one answer redialing cannot change. Everything else keeps retrying, so watch the logs -when a sink stays `disconnected` from the very first attempt: a pipeline that has never connected -once is far more likely misconfigured than waiting out an outage. +pipeline runs, so a relay outage of any length is ridden out; it goes `failed` only on an answer the +relay actually gave that redialing cannot change: a rejected token, or a CONNECT answered with a +status that isn't an invitation to retry. Everything else keeps retrying, so watch the logs when a +sink stays `disconnected` from the very first attempt: a pipeline that has never connected once is +far more likely misconfigured than waiting out an outage. ## Prerequisites diff --git a/rs/CLAUDE.md b/rs/CLAUDE.md index 633248edb3..44ff36d3f0 100644 --- a/rs/CLAUDE.md +++ b/rs/CLAUDE.md @@ -109,7 +109,7 @@ Negotiation: `version::NEGOTIATED` lists SETUP-negotiated versions in preference ## Rust conventions -- **Retry loops inline their backoff** (root Retries has the policy). Escalate a local `Duration` toward a `const MAX` (seconds, not minutes), jitter each wait (`delay.mul_f64(0.5 + rand::rng().random::() / 2.0)`), and track the give-up deadline as a `tokio::time::Instant` next to the delay. No shared backoff helper, and no `is_retryable()` on error types. +- **Retry loops inline their backoff** (root Retries has the policy). Escalate a local `Duration` toward a `const MAX` (seconds, not minutes), jitter each wait (`delay.mul_f64(0.5 + rand::rng().random::() / 2.0)`), and track the give-up deadline as a `tokio::time::Instant` next to the delay. No shared backoff helper, and no `is_retryable()` on error types; the one thing worth reading off a failure is a status a peer actually sent, via the `status()` accessors on `moq_native::Error` / `moq_hls::Error`. - **Prefer `kio` over tokio sync primitives**: reach for `kio::Producer`/`Consumer` (and the `poll_*` plumbing) instead of `tokio::sync` channels or `watch`. A `tokio::sync::watch` (or a channel) carrying a single value is a code smell. `kio` ties into the runtime-free `poll_*` model and avoids a hard runtime dependency. - **Errors**: `thiserror` with `#[from]` for libraries, `anyhow` (with `.context("...")`, not `.map_err(|_| anyhow!())`) for binaries. Always `#[non_exhaustive]` on public error enums (e.g. `moq-net/src/error.rs`, `moq-ffi/src/error.rs`, `moq-loc/src/lib.rs`). Use `#[error(transparent)]` + `#[from]` for wrapped foreign errors (see `moq-token/src/error.rs`). - **Config + TOML merge**: any `#[arg]` field on a TOML-loadable config must be `Option`, never a bare `bool`/`String`/etc. The TOML->CLI merge re-applies clap defaults and silently clobbers TOML values for bare fields. See `moq-relay/src/config.rs` and its regression tests (`cli_does_not_clobber_toml_*`); add such a test for any new flag. diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index 4fd9aa96dd..89289bfec6 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -42,7 +42,8 @@ pub enum ConnectionStatus { /// A session is connected and publishing. #[enum_value(name = "Connected: session established", nick = "connected")] Connected, - /// The reconnect loop gave up permanently (an auth rejection). Terminal. + /// The reconnect loop gave up permanently (an auth rejection, or a CONNECT status that isn't an + /// invitation to retry). Terminal. #[enum_value(name = "Failed: connection rejected, gave up", nick = "failed")] Failed, } @@ -138,11 +139,11 @@ impl Session { // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC // outages of any length, which is the trade this element wants: a pipeline nobody is watching - // should still be publishing when the relay comes back. The loop still ends on an auth - // rejection (the one answer redialing cannot change), posting the bus error below. During an - // outage the pad threads keep writing (bounded by moq-net's per-group eviction) and the relay - // catches up from a group boundary on reconnect. A bounded policy is available via - // `ClientConfig::backoff`. + // should still be publishing when the relay comes back. The loop still ends on the two answers + // a server states outright (an auth rejection, or a CONNECT status that isn't an invitation to + // retry), posting the bus error below. During an outage the pad threads keep writing (bounded + // by moq-net's per-group eviction) and the relay catches up from a group boundary on + // reconnect. A bounded policy is available via `ClientConfig::backoff`. let mut config = moq_native::ClientConfig::default(); config.tls.disable_verify = Some(settings.tls_disable_verify); config.backoff.timeout = std::time::Duration::ZERO; diff --git a/rs/moq-hls/src/error.rs b/rs/moq-hls/src/error.rs index efa4b09532..50055295e3 100644 --- a/rs/moq-hls/src/error.rs +++ b/rs/moq-hls/src/error.rs @@ -18,6 +18,15 @@ impl std::fmt::Display for SequenceKind { } } +/// Whether an HTTP response status means "ask again later". +/// +/// A response that arrived is the server's answer, and only this narrow set invites another +/// attempt: request timeout, rate limit, and the gateway/overload statuses. Every other status, +/// `404` and `403` included, is settled. +pub(crate) fn status_retryable(status: u16) -> bool { + matches!(status, 408 | 429 | 502 | 503 | 504) +} + /// Errors produced by the HLS <-> MoQ gateway (import and export). #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] @@ -124,6 +133,20 @@ pub enum Error { Other(std::sync::Arc), } +impl Error { + /// The HTTP status the origin answered with, if it answered with one at all. + /// + /// The import loop consults it: a `503` on a playlist fetch is worth another pass, a `404` is + /// the origin's settled answer. Nothing else here is classified; a failure with no status falls + /// through to the backoff budget. + pub fn status(&self) -> Option { + match self { + Self::Reqwest(err) => err.status().map(|status| status.as_u16()), + _ => None, + } + } +} + impl From for Error { fn from(err: reqwest::Error) -> Self { Error::Reqwest(std::sync::Arc::new(err)) @@ -144,3 +167,17 @@ impl From for Error { /// Convenience alias for results from the HLS gateway. pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + /// The import loop consults this, so an origin's settled answer has to reach it intact. + #[test] + fn an_http_failure_reports_its_status() { + // A failure the origin never answered carries no status, so the budget decides instead. + assert_eq!(Error::NoVariants.status(), None); + assert_eq!(Error::ParsePlaylist("not a playlist".to_string()).status(), None); + assert_eq!(Error::Moq(moq_net::Error::Transport("lost".to_string())).status(), None); + } +} diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index a592d0607f..abdb9376eb 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -99,11 +99,14 @@ struct StepOutcome { /// nothing to add this pass, not a failure. Counting segments instead would read a quiet /// playlist as a dead one. ok: usize, - /// The first rendition failure from this step, if any. + /// A rendition failure from this step, if any. /// /// [`OnError::Warn`] keeps the other renditions going after one fails, so a step can report /// `Ok` having imported nothing at all. The loop needs to tell that apart from a quiet playlist /// with no new segments, or it treats a permanently broken source as steady progress. + /// + /// When several renditions fail, this is the one whose failure another pass could still clear, + /// so a single permanently-dead variant doesn't end an import the others could still serve. failed: Option, } @@ -610,7 +613,8 @@ impl Import { /// /// A failed step is logged and retried with escalating backoff, and the import ends with the /// last error once the backoff budget is spent, so a broken source surfaces within seconds - /// instead of looping forever. + /// instead of looping forever. The one shortcut is an HTTP status the origin actually sent: a + /// `404` playlist ends the import immediately, since no amount of waiting turns it into a `200`. pub async fn run(&mut self) -> Result<()> { let mut delay = STEP_RETRY_MIN; let mut deadline = tokio::time::Instant::now() + STEP_RETRY_BUDGET; @@ -640,6 +644,11 @@ impl Import { deadline = tokio::time::Instant::now() + STEP_RETRY_BUDGET; outcome } + // A status the origin actually sent is its answer: a 404 playlist is not going to + // become a 200 on the next pass. Everything else rides the backoff budget. + Err(err) if err.status().is_some_and(|status| !crate::status_retryable(status)) => { + return Err(err); + } Err(err) => { let now = tokio::time::Instant::now(); if now >= deadline { @@ -715,8 +724,14 @@ impl Import { // drop the rest or abort the whole step. OnError::Warn => { warn!(label = %track.label, %err, "rendition import step failed, will retry"); - // Keep the first failure so the outcome doesn't depend on rendition order. - failed.get_or_insert(err); + // Prefer a failure another pass could still clear, so a single + // permanently-dead variant doesn't end an import the others could still + // serve. Keeping whichever error came last instead would make the outcome + // depend on rendition order. + let recoverable = err.status().is_none_or(crate::status_retryable); + if recoverable || failed.is_none() { + failed = Some(err); + } } }, } diff --git a/rs/moq-hls/src/lib.rs b/rs/moq-hls/src/lib.rs index 5b97dd6cda..1708142543 100644 --- a/rs/moq-hls/src/lib.rs +++ b/rs/moq-hls/src/lib.rs @@ -25,6 +25,7 @@ pub mod import; #[cfg(feature = "server")] pub mod server; +pub(crate) use error::status_retryable; pub use error::*; #[cfg(feature = "server")] pub use server::Server; diff --git a/rs/moq-native/src/error.rs b/rs/moq-native/src/error.rs index 434a85ae7b..da4251168c 100644 --- a/rs/moq-native/src/error.rs +++ b/rs/moq-native/src/error.rs @@ -1,5 +1,14 @@ use std::sync::Arc; +/// Whether an HTTP response status means "ask again later". +/// +/// A response that arrived is the server's answer, and only this narrow set invites another +/// attempt: request timeout, rate limit, and the gateway/overload statuses. Every other status, +/// `404` and `403` included, is settled. +pub(crate) fn status_retryable(status: u16) -> bool { + matches!(status, 408 | 429 | 502 | 503 | 504) +} + /// Errors produced while configuring or establishing native MoQ connections. /// /// Backend-specific failures live in per-backend error types ([`crate::tls::Error`], @@ -145,6 +154,37 @@ impl Error { pub fn is_auth(&self) -> bool { self.connect_error().is_some_and(|err| err.is_auth()) } + + /// The HTTP status a server answered a connection attempt with, if it answered with one at all. + /// + /// `None` covers everything else: a dial that never got a response, a QUIC handshake that + /// failed, a URL we couldn't parse. Only a status the peer actually sent shows up here, and + /// whether it invites another attempt is the caller's call (`408`, `429`, `502`, `503`, and + /// `504` are the ones worth repeating). This deliberately does not try to say whether some + /// *other* kind of failure is worth retrying; that's a guess, and a backoff budget bounds it + /// instead. + pub fn status(&self) -> Option { + match self { + // A race is only settled when both halves were answered, and answered with something not + // worth repeating: one transport being refused says nothing about the other, so a `404` + // over QUIC alongside a dead WebSocket is still just a failed dial. + #[cfg(feature = "websocket")] + Self::TransportRace { quic, websocket } => match (quic.status(), websocket.status()) { + (Some(quic), Some(websocket)) if !status_retryable(quic) && !status_retryable(websocket) => Some(quic), + _ => None, + }, + + #[cfg(feature = "quinn")] + Self::Quinn(err) => err.status(), + #[cfg(feature = "noq")] + Self::Noq(err) => err.status(), + #[cfg(feature = "quiche")] + Self::Quiche(err) => err.status(), + #[cfg(feature = "websocket")] + Self::WebSocket(err) => err.status(), + _ => None, + } + } } // The wrapped sources aren't `Clone`, so `#[from]` can't store them behind `Arc` diff --git a/rs/moq-native/src/noq.rs b/rs/moq-native/src/noq.rs index 8ca8e26786..01f4e44e0e 100644 --- a/rs/moq-native/src/noq.rs +++ b/rs/moq-native/src/noq.rs @@ -397,6 +397,33 @@ impl Error { _ => None, } } + + /// The HTTP status a server answered with, if it answered with one at all. + /// + /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the + /// WebTransport CONNECT response. See [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { + match self { + Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { + err.status().map(|status| status.as_u16()) + } + Self::Client(err) => client_status(err), + // Every raced address has to have answered, and answered with something not worth + // repeating, before the set counts as settled: one address refusing says nothing about + // the others, which may simply have been unroutable. + Self::Failover(failures) => { + let mut settled = None; + for failure in failures { + match failure.error.status() { + Some(status) if !crate::error::status_retryable(status) => settled = Some(status), + _ => return None, + } + } + settled + } + _ => None, + } + } } fn map_client_error(err: web_transport_noq::ClientError) -> Error { @@ -408,26 +435,35 @@ fn map_client_error(err: web_transport_noq::ClientError) -> Error { } fn classify_client_error(err: &web_transport_noq::ClientError) -> Option { + client_status(err).and_then(crate::ConnectError::from_status_u16) +} + +/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at +/// all (as opposed to the connection failing underneath the request). +/// +/// Both classifications read this: [`classify_client_error`] turns an auth status into a +/// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults +/// the status. A `404` or `405` is the server's settled answer, so retrying +/// it just burns the reconnect budget on a URL that will never work. +fn client_status(err: &web_transport_noq::ClientError) -> Option { match err { - web_transport_noq::ClientError::HttpError(err) => classify_connect_error(err), + web_transport_noq::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn classify_connect_error(err: &web_transport_noq::ConnectError) -> Option { +fn connect_status(err: &web_transport_noq::ConnectError) -> Option { match err { - web_transport_noq::ConnectError::ErrorStatus(status) => crate::ConnectError::from_status_u16(status.as_u16()), - web_transport_noq::ConnectError::ProtoError(err) => classify_proto_error(err), + web_transport_noq::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web_transport_noq::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn classify_proto_error(err: &web_transport_noq::proto::ConnectError) -> Option { +fn proto_status(err: &web_transport_noq::proto::ConnectError) -> Option { match err { web_transport_noq::proto::ConnectError::ErrorStatus(status) - | web_transport_noq::proto::ConnectError::WrongStatus(Some(status)) => { - crate::ConnectError::from_status_u16(status.as_u16()) - } + | web_transport_noq::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), _ => None, } } diff --git a/rs/moq-native/src/quiche.rs b/rs/moq-native/src/quiche.rs index f93e65f45f..6cc28a5793 100644 --- a/rs/moq-native/src/quiche.rs +++ b/rs/moq-native/src/quiche.rs @@ -473,6 +473,33 @@ impl Error { _ => None, } } + + /// The HTTP status a server answered with, if it answered with one at all. + /// + /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the + /// WebTransport CONNECT response. See [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { + match self { + Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { + err.status().map(|status| status.as_u16()) + } + Self::ClientConnect(err) => client_status(err), + // Every raced address has to have answered, and answered with something not worth + // repeating, before the set counts as settled: one address refusing says nothing about + // the others, which may simply have been unroutable. + Self::Failover(failures) => { + let mut settled = None; + for failure in failures { + match failure.error.status() { + Some(status) if !crate::error::status_retryable(status) => settled = Some(status), + _ => return None, + } + } + settled + } + _ => None, + } + } } fn map_client_error(err: web_transport_quiche::ClientError) -> Error { @@ -484,26 +511,35 @@ fn map_client_error(err: web_transport_quiche::ClientError) -> Error { } fn classify_client_error(err: &web_transport_quiche::ClientError) -> Option { + client_status(err).and_then(crate::ConnectError::from_status_u16) +} + +/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at +/// all (as opposed to the connection failing underneath the request). +/// +/// Both classifications read this: [`classify_client_error`] turns an auth status into a +/// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults +/// the status. A `404` or `405` is the server's settled answer, so retrying +/// it just burns the reconnect budget on a URL that will never work. +fn client_status(err: &web_transport_quiche::ClientError) -> Option { match err { - web_transport_quiche::ClientError::Connect(err) => classify_connect_error(err), + web_transport_quiche::ClientError::Connect(err) => connect_status(err), _ => None, } } -fn classify_connect_error(err: &web_transport_quiche::h3::ConnectError) -> Option { +fn connect_status(err: &web_transport_quiche::h3::ConnectError) -> Option { match err { - web_transport_quiche::h3::ConnectError::Status(status) => crate::ConnectError::from_status_u16(status.as_u16()), - web_transport_quiche::h3::ConnectError::Proto(err) => classify_proto_error(err), + web_transport_quiche::h3::ConnectError::Status(status) => Some(status.as_u16()), + web_transport_quiche::h3::ConnectError::Proto(err) => proto_status(err), _ => None, } } -fn classify_proto_error(err: &web_transport_quiche::proto::ConnectError) -> Option { +fn proto_status(err: &web_transport_quiche::proto::ConnectError) -> Option { match err { web_transport_quiche::proto::ConnectError::ErrorStatus(status) - | web_transport_quiche::proto::ConnectError::WrongStatus(Some(status)) => { - crate::ConnectError::from_status_u16(status.as_u16()) - } + | web_transport_quiche::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), _ => None, } } diff --git a/rs/moq-native/src/quinn.rs b/rs/moq-native/src/quinn.rs index 1ba24ba6e3..f0a536c3c4 100644 --- a/rs/moq-native/src/quinn.rs +++ b/rs/moq-native/src/quinn.rs @@ -410,6 +410,33 @@ impl Error { _ => None, } } + + /// The HTTP status a server answered with, if it answered with one at all. + /// + /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the + /// WebTransport CONNECT response. See [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { + match self { + Self::FetchFingerprint(err) | Self::FingerprintStatus(err) | Self::ReadFingerprint(err) => { + err.status().map(|status| status.as_u16()) + } + Self::Client(err) => client_status(err), + // Every raced address has to have answered, and answered with something not worth + // repeating, before the set counts as settled: one address refusing says nothing about + // the others, which may simply have been unroutable. + Self::Failover(failures) => { + let mut settled = None; + for failure in failures { + match failure.error.status() { + Some(status) if !crate::error::status_retryable(status) => settled = Some(status), + _ => return None, + } + } + settled + } + _ => None, + } + } } fn map_client_error(err: web_transport_quinn::ClientError) -> Error { @@ -421,26 +448,35 @@ fn map_client_error(err: web_transport_quinn::ClientError) -> Error { } fn classify_client_error(err: &web_transport_quinn::ClientError) -> Option { + client_status(err).and_then(crate::ConnectError::from_status_u16) +} + +/// The HTTP status the server answered the WebTransport CONNECT with, when it answered with one at +/// all (as opposed to the connection failing underneath the request). +/// +/// Both classifications read this: [`classify_client_error`] turns an auth status into a +/// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults +/// the status. A `404` or `405` is the server's settled answer, so retrying +/// it just burns the reconnect budget on a URL that will never work. +fn client_status(err: &web_transport_quinn::ClientError) -> Option { match err { - web_transport_quinn::ClientError::HttpError(err) => classify_connect_error(err), + web_transport_quinn::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn classify_connect_error(err: &web_transport_quinn::ConnectError) -> Option { +fn connect_status(err: &web_transport_quinn::ConnectError) -> Option { match err { - web_transport_quinn::ConnectError::ErrorStatus(status) => crate::ConnectError::from_status_u16(status.as_u16()), - web_transport_quinn::ConnectError::ProtoError(err) => classify_proto_error(err), + web_transport_quinn::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web_transport_quinn::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn classify_proto_error(err: &web_transport_quinn::proto::ConnectError) -> Option { +fn proto_status(err: &web_transport_quinn::proto::ConnectError) -> Option { match err { web_transport_quinn::proto::ConnectError::ErrorStatus(status) - | web_transport_quinn::proto::ConnectError::WrongStatus(Some(status)) => { - crate::ConnectError::from_status_u16(status.as_u16()) - } + | web_transport_quinn::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), _ => None, } } @@ -686,6 +722,43 @@ impl quinn::ConnectionIdGenerator for ServerIdGenerator { mod tests { use super::*; + fn connect_rejected(status: u16) -> Error { + Error::Client(web_transport_quinn::ClientError::HttpError( + web_transport_quinn::ConnectError::ErrorStatus( + web_transport_quinn::http::StatusCode::from_u16(status).unwrap(), + ), + )) + } + + /// A CONNECT the relay answered carries its status through to the caller, so a wrong path or an + /// endpoint that doesn't speak WebTransport can surface immediately rather than after the whole + /// reconnect budget. + #[test] + fn a_rejected_connect_reports_its_status() { + for status in [400, 404, 405, 410, 501] { + assert_eq!(connect_rejected(status).status(), Some(status)); + assert!( + !crate::error::status_retryable(status), + "{status} should stop the reconnect loop" + ); + } + + for status in [408, 429, 502, 503, 504] { + assert_eq!(connect_rejected(status).status(), Some(status)); + assert!(crate::error::status_retryable(status), "{status} should be retried"); + } + + // Auth is peeled off into its own variant before reaching the generic client arm. + assert_eq!( + connect_rejected(401).connect_error(), + Some(crate::ConnectError::Unauthorized) + ); + assert_eq!( + connect_rejected(403).connect_error(), + Some(crate::ConnectError::Forbidden) + ); + } + /// Build a controller from each family's factory and downcast it to the /// concrete quinn implementation it must map to. #[test] diff --git a/rs/moq-native/src/reconnect.rs b/rs/moq-native/src/reconnect.rs index 300fb4585a..7041e66495 100644 --- a/rs/moq-native/src/reconnect.rs +++ b/rs/moq-native/src/reconnect.rs @@ -14,11 +14,11 @@ use crate::{Client, Error}; /// This decides how long to wait between reconnect attempts and when to give up. The delays carry /// jitter, so a fleet knocked offline together doesn't reconnect in lockstep. /// -/// [`timeout`](Self::timeout) is what ends a hopeless loop, not a judgment about the error: every -/// failure is retried the same way (except an auth rejection, which a redial cannot change), and -/// the short default budget is what surfaces a broken target instead of hiding it. A zero timeout -/// removes that backstop, so it belongs only where an unattended process must outlive an outage of -/// any length. +/// [`timeout`](Self::timeout) is what ends a hopeless loop: every failure rides the same backoff, +/// and the short default budget is what surfaces a broken target instead of hiding it. The only +/// failures short-circuited are answers a server actually gave (an auth rejection, or a CONNECT +/// status that isn't an invitation to retry). A zero timeout removes the backstop, so it belongs +/// only where an unattended process must outlive an outage of any length. #[derive(Clone, Debug, clap::Args, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] @@ -118,8 +118,8 @@ struct State { status: Option, /// The negotiated MoQ version of the live session, or `None` when disconnected. version: Option, - /// Set when the reconnect loop permanently gives up: the backoff timeout expiring, or an auth - /// rejection that redialing cannot change. + /// Set when the reconnect loop permanently gives up: the backoff timeout expiring, or a server + /// answer that redialing cannot change. error: Option, /// The currently-connected session, or `None` while reconnecting. Read by /// [`ConnectionStatsReader`] to snapshot live connection stats. @@ -258,13 +258,18 @@ impl Reconnect { } } Err(err) => { - // An auth rejection is the one answer redialing cannot change. Everything else - // falls through to the backoff, whose budget is what stops the loop: a failure - // that clears within the budget was transient, and one that doesn't surfaces as - // the last real error rather than being guessed at up front. + // The two answers a server can give that redialing cannot change: it rejected our + // credentials, or it answered the CONNECT with a status that isn't an invitation + // to come back. Everything else falls through to the backoff, whose budget is + // what stops the loop. if err.is_auth() { return Err(err); } + if let Some(status) = err.status() + && !crate::error::status_retryable(status) + { + return Err(err); + } last_error = Some(err); } } diff --git a/rs/moq-native/src/websocket.rs b/rs/moq-native/src/websocket.rs index c1fb1d1433..f99e0b66bc 100644 --- a/rs/moq-native/src/websocket.rs +++ b/rs/moq-native/src/websocket.rs @@ -223,6 +223,17 @@ impl Error { _ => None, } } + + /// The HTTP status the server answered the upgrade with, if it answered with one at all. + /// + /// qmux surfaces a non-101 WebSocket upgrade response as `Http(status)`. See + /// [`crate::Error::status`]. + pub(crate) fn status(&self) -> Option { + match self { + Self::Connect(qmux::Error::Http(status)) => Some(*status), + _ => None, + } + } } /// Listens for incoming WebSocket connections on a TCP port. @@ -246,16 +257,7 @@ impl Listener { // `qmux_versions_for` returns `&[]` (every QMux draft) for ALPNs the spec // doesn't restrict; qmux by default also accepts legacy clients that // only offer a bare wire-format ALPN (today's moq-net clients still do). - // - // Keep-alive matches the dial side (5s ping / 30s deadline, parity with QUIC's - // idle timeout) and is on by default because the accept side is where its - // absence costs something: a peer whose host crashed sends no FIN, and a - // WebSocket has no idle timeout of its own, so the session -- and every - // broadcast announced through it -- would survive until the OS probes the - // socket hours later. `qmux::Server` defaults it OFF, so this must be explicit. - let server = qmux::Server::new() - .with_protocols(alpns.iter().map(|&a| (a, qmux_versions_for(a)))) - .with_keep_alive(qmux::KeepAlive::default()); + let server = qmux::Server::new().with_protocols(alpns.iter().map(|&a| (a, qmux_versions_for(a)))); Ok(Self { listener, server }) }