From 87b18b6acbde1238346d223b0809758a163d087a Mon Sep 17 00:00:00 2001 From: Eilon Moalem Date: Tue, 28 Jul 2026 14:41:25 +0300 Subject: [PATCH] feat(realtime): scale uplink maxBitrate with publish fps below 15 fps The encoder spends its per-second bitrate budget regardless of cadence, so a 10 fps publisher packs ~3x the bytes into every frame; the oversized frames' wire-arrival span inflates the server's ingest jitter buffer by ~35-100 ms. Scaling the cap to ~200 kbit/frame below 15 fps restores 30 fps-parity ingest latency at equal-or-better QP (measured: 10 fps ingest hold 129.7 -> 28.4 ms, e2e 660 -> 557 ms from us-east4; full causal A/B in the api repo, experiments/2026-07-28-livekit-size-vs-interval-jb). No-op for every registry model today (all declare 30 fps) and for >=15 fps publishers; react-native path unchanged (no publishFps -> natural rate). --- .../realtime/browser/prepare-connection.ts | 1 + packages/sdk/src/realtime/config-realtime.ts | 16 ++++++++ packages/sdk/src/realtime/media-channel.ts | 20 +++++++++- .../sdk/tests/publish-options.unit.test.ts | 39 +++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 packages/sdk/tests/publish-options.unit.test.ts diff --git a/packages/sdk/src/realtime/browser/prepare-connection.ts b/packages/sdk/src/realtime/browser/prepare-connection.ts index 66517f3..3a74e12 100644 --- a/packages/sdk/src/realtime/browser/prepare-connection.ts +++ b/packages/sdk/src/realtime/browser/prepare-connection.ts @@ -79,6 +79,7 @@ export const prepareBrowserConnection: PrepareConnection = ({ createMediaChannel: (config) => createLiveKitMediaChannel({ ...config, + publishFps: fps, createFrameMetadataWorker: frameTiming ? takeFrameMetadataWorker : undefined, }), dispose: () => { diff --git a/packages/sdk/src/realtime/config-realtime.ts b/packages/sdk/src/realtime/config-realtime.ts index f17f4d3..063cfa7 100644 --- a/packages/sdk/src/realtime/config-realtime.ts +++ b/packages/sdk/src/realtime/config-realtime.ts @@ -35,6 +35,22 @@ export const REALTIME_CONFIG = { defaultMaxVideoBitrateBps: 3_500_000, vp9MaxVideoBitrateBps: 3_000_000, defaultPublishFps: 30, + /** + * Low-fps uplink bitrate scaling (measured 2026-07-28, api repo + * experiments/2026-07-28-livekit-size-vs-interval-jb). The encoder spends + * its per-SECOND budget regardless of cadence, so a 10 fps publisher packs + * ~3x the bytes into every frame; the frames' wire-arrival span then + * inflates the server's ingest jitter buffer by ~35-100 ms. Scaling the + * cap to ~200 kbit per frame restores 30 fps-parity ingest latency at + * equal-or-better QP (measured: 10 fps hold 129.7 -> 28.4 ms, e2e + * 660 -> 557 ms from us-east4). At >=15 fps the natural rate is already + * optimal and a binding cap measurably hurts (rate-controller churn), so + * scaling only engages BELOW the threshold; the floor keeps the encoder + * far from the simulcast layer-starvation cliff. + */ + lowFpsBitrateScaleBelowFps: 15, + lowFpsBitsPerFrame: 200_000, + lowFpsMinBitrateBps: 1_000_000, }, observability: { stallFpsThreshold: 0.5, diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 36232c2..e21db68 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -19,12 +19,26 @@ export function getDefaultVideoPublishOptions( source: TrackPublishOptions["source"], videoCodec?: VideoCodec, frameMetadata = false, + publishFps?: number, ): TrackPublishOptions { const resolvedCodec = videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec; - const maxBitrate = + const naturalBitrate = resolvedCodec === "vp9" ? REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps : REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps; + // The encoder spends its per-second budget regardless of cadence, so at low + // publish rates a fixed cap packs into oversized frames whose wire-arrival + // span inflates the server's ingest jitter buffer. Scale the cap to + // ~constant bits-per-frame below the threshold; at >=15 fps the natural + // rate is optimal and a binding cap hurts, so leave it untouched. + const fps = publishFps ?? REALTIME_CONFIG.livekit.defaultPublishFps; + const maxBitrate = + fps < REALTIME_CONFIG.livekit.lowFpsBitrateScaleBelowFps + ? Math.max( + REALTIME_CONFIG.livekit.lowFpsMinBitrateBps, + Math.min(naturalBitrate, Math.round(fps * REALTIME_CONFIG.livekit.lowFpsBitsPerFrame)), + ) + : naturalBitrate; return { source, @@ -48,6 +62,8 @@ export interface MediaChannelConfig { localStream: MediaStream | null; logger?: Logger; videoCodec?: VideoCodec; + /** Intended publish cadence (fps) — scales the uplink maxBitrate below 15 fps. */ + publishFps?: number; createFrameMetadataWorker?: () => Worker; } @@ -192,7 +208,7 @@ export class LiveKitMediaChannel implements MediaChannel { if (!this.cameraTrackSource) throw new Error("Cannot publish video track: media channel is not connected"); await this.room.localParticipant.publishTrack( track, - getDefaultVideoPublishOptions(this.cameraTrackSource, this.config.videoCodec, this.frameMetadataEnabled), + getDefaultVideoPublishOptions(this.cameraTrackSource, this.config.videoCodec, this.frameMetadataEnabled, this.config.publishFps), ); } else { await this.room.localParticipant.publishTrack(track); diff --git a/packages/sdk/tests/publish-options.unit.test.ts b/packages/sdk/tests/publish-options.unit.test.ts new file mode 100644 index 0000000..761c751 --- /dev/null +++ b/packages/sdk/tests/publish-options.unit.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; + +import { REALTIME_CONFIG } from "../src/realtime/config-realtime"; +import { getDefaultVideoPublishOptions } from "../src/realtime/media-channel"; + +const NATURAL_H264 = REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps; +const NATURAL_VP9 = REALTIME_CONFIG.livekit.vp9MaxVideoBitrateBps; + +describe("getDefaultVideoPublishOptions low-fps bitrate scaling", () => { + it("keeps the natural bitrate at the default 30 fps", () => { + const opts = getDefaultVideoPublishOptions("camera", "h264", false, 30); + expect(opts.videoEncoding?.maxBitrate).toBe(NATURAL_H264); + }); + + it("keeps the natural bitrate when fps is omitted (registry models)", () => { + const opts = getDefaultVideoPublishOptions("camera", "h264", false); + expect(opts.videoEncoding?.maxBitrate).toBe(NATURAL_H264); + }); + + it("does NOT engage at 15 fps (binding caps measurably hurt at >=15 fps)", () => { + const opts = getDefaultVideoPublishOptions("camera", "h264", false, 15); + expect(opts.videoEncoding?.maxBitrate).toBe(NATURAL_H264); + }); + + it("scales to ~200 kbit/frame at 10 fps", () => { + const opts = getDefaultVideoPublishOptions("camera", "h264", false, 10); + expect(opts.videoEncoding?.maxBitrate).toBe(10 * REALTIME_CONFIG.livekit.lowFpsBitsPerFrame); + }); + + it("never drops below the viability floor at very low fps", () => { + const opts = getDefaultVideoPublishOptions("camera", "h264", false, 2); + expect(opts.videoEncoding?.maxBitrate).toBe(REALTIME_CONFIG.livekit.lowFpsMinBitrateBps); + }); + + it("never exceeds the codec's natural bitrate", () => { + const opts = getDefaultVideoPublishOptions("camera", "vp9", false, 14.9); + expect(opts.videoEncoding?.maxBitrate).toBeLessThanOrEqual(NATURAL_VP9); + }); +});