From ede90334fceecb073edfe339700a781722654e03 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 1 May 2026 22:12:12 +0000 Subject: [PATCH] Resource hardening for small (4 GB) hosts colocated with TS6 Live measurement on a 4 GB host running the bot next to its TS6 server: one viewer at 720p30 / 4608 kbps pushed the bot's RSS past 750 MB and was still climbing. The TS6 server, sharing the same RAM, started dropping packets - including viewer join requests - and eventually crashed under memory pressure, taking the whole stack with it. Operators reported "I can't reliably get into the stream and the server crashes". Three changes that together keep the colocated case stable: * Encoder defaults dropped to 480p24 / 2500 kbps (from 720p30 / 4608 kbps). Cuts libvpx's reference frame pool roughly in half; expected ~250 MB per viewer instead of ~750 MB. Operators with a beefier host bump SCREEN_WIDTH/HEIGHT + STREAM_BITRATE together in .env. * `STREAM_VIEWER_LIMIT` default 2 (from 4). With the lower defaults that's still 600-700 MB peak for the bot, leaving room for the TS6 server + OS on a 4 GB box. * New `ICE_DROP_NETWORKS` setting (default `172.16.0.0/12`). In host-network mode aiortc gathers a host candidate per docker bridge gateway plus matching srflx, blowing the offer SDP to 5 KB / 11 UDP fragments that some TS6 server builds choke on. We strip those before send_join_response - aiortc keeps them internally, we just don't burden the server with what no external viewer can route to anyway. --- .env.example | 43 ++++++--- src/ts6_stream_bot/config.py | 58 +++++++++--- .../pipeline/stream_publisher.py | 71 ++++++++++++++ tests/test_sdp_filter.py | 93 +++++++++++++++++++ 4 files changed, 239 insertions(+), 26 deletions(-) create mode 100644 tests/test_sdp_filter.py diff --git a/.env.example b/.env.example index d6d0cad..28a5099 100644 --- a/.env.example +++ b/.env.example @@ -6,11 +6,16 @@ BOT_API_KEY= # Logging LOG_LEVEL=INFO -# Display / capture (defaults are sane, only change if you know why) +# Display / capture +# Defaults are tuned for a 4 GB host that runs the TS6 server next to +# the bot. Each viewer adds ~250 MB RSS at 480p24 (the default), and +# the encoder cost scales roughly with pixels * fps. Bump these only +# if your host has the budget AND your viewers can decode 720p+ in +# their TS6 client. DISPLAY=:99 -SCREEN_WIDTH=1920 -SCREEN_HEIGHT=1080 -SCREEN_FPS=30 +SCREEN_WIDTH=854 +SCREEN_HEIGHT=480 +SCREEN_FPS=24 # PulseAudio PULSE_SINK=bot_sink @@ -53,17 +58,20 @@ TS6_CHANNEL_PASSWORD= # for each viewer and the bot replies via respondjoinstreamrequest); # 0 = auto-accept. # - STREAM_VIEWER_LIMIT: each viewer spins up its own VP8 encoder -# (~280 MB RSS at 720p30) so unlimited can OOM a small host. The default -# of 4 is conservative; bump it once you've measured memory under load. -# -1 = unlimited (TS3 convention). 0 is interpreted by some TS6 builds -# as "no viewers allowed", which silently drops every join attempt - -# don't use 0. -# - STREAM_BITRATE: hint to the server (kbps); 4608 mirrors the default the -# TS6 client uses for "high quality" screen share. -STREAM_BITRATE=4608 +# (roughly 250 MB RSS at the 480p24 default; more at 720p / 1080p). +# On a 4 GB host that also runs the TS6 server, two viewers is the +# safe ceiling - going higher squeezed the TS6 server out of RAM +# in our live tests, which then crashed the whole stack. Bump once +# you've measured. -1 = unlimited (TS3 convention). 0 is interpreted +# by some TS6 builds as "no viewers allowed" - don't use 0. +# - STREAM_BITRATE: hint to the server (kbps). 2500 pairs with 480p24; +# ~4608 with 720p; ~6000+ with 1080p. Both the bitrate and the +# resolution feed into libvpx's encoder buffer, so drop both +# together if you hit OOM, raise both together if you have headroom. +STREAM_BITRATE=2500 STREAM_ACCESSIBILITY=0 STREAM_MODE=1 -STREAM_VIEWER_LIMIT=4 +STREAM_VIEWER_LIMIT=2 # --- WebRTC ICE servers ---------------------------------------------------- # STUN exposes the bot's public-NAT'd IP as a server-reflexive candidate; @@ -74,3 +82,12 @@ STUN_URL=stun:stun.l.google.com:19302 # TURN_URL=turn:openrelay.metered.ca:80 # TURN_USERNAME=openrelayproject # TURN_PASSWORD=openrelayproject + +# CIDR blocks whose ICE candidates get stripped from the offer SDP we +# hand to the TS6 server. In host-network mode aiortc gathers a host +# candidate per docker bridge interface (172.x.x.x); each such useless +# candidate also spawns a srflx pair, blowing the offer to ~5 KB / 11 +# UDP fragments which some TS6 server builds choke on. Default drops +# the entire RFC1918 docker range. Set to an empty value (or a +# different CIDR) to override; multiple values are comma-separated. +# ICE_DROP_NETWORKS=172.16.0.0/12 diff --git a/src/ts6_stream_bot/config.py b/src/ts6_stream_bot/config.py index a719872..42f3f01 100644 --- a/src/ts6_stream_bot/config.py +++ b/src/ts6_stream_bot/config.py @@ -34,15 +34,19 @@ class Settings(BaseSettings): LOG_LEVEL: str = "INFO" # --- Display / capture ------------------------------------------------- - # 720p30 keeps the per-viewer VP8 encode out of OOM territory on - # small hosts. aiortc + PyAV at 1080p30 with even one peer can run a - # 4 GB host out of memory (live deploy hit SIGKILL after ~6 s of - # streaming). Bump SCREEN_WIDTH/SCREEN_HEIGHT in .env if your host - # has the budget; you'll also want to bump STREAM_BITRATE. + # Defaults are tuned for a 4 GB host running both the bot and the TS6 + # server side by side. Each viewer adds its own VP8 encoder (PyAV + + # libvpx through aiortc); live measurement at 720p30 / 4608 kbps had + # one viewer push the bot's RSS past 750 MB and still climbing - on + # a 4 GB box, two viewers + the TS6 server is enough to OOM the + # whole stack and crash the server with it. 480p24 / 2500 kbps cuts + # the encoder's reference-frame pool roughly in half. Bump every + # value if your host has the budget; you'll usually want to scale + # SCREEN_WIDTH/HEIGHT and STREAM_BITRATE together. DISPLAY: str = ":99" - SCREEN_WIDTH: int = 1280 - SCREEN_HEIGHT: int = 720 - SCREEN_FPS: int = 30 + SCREEN_WIDTH: int = 854 + SCREEN_HEIGHT: int = 480 + SCREEN_FPS: int = 24 # --- Audio ------------------------------------------------------------- PULSE_SINK: str = "bot_sink" @@ -91,7 +95,15 @@ class Settings(BaseSettings): # --- Stream parameters ------------------------------------------------- # These shape the `setupstream` request the bot sends on connect. # Defaults match what the TS6 client UI uses for a normal screen-share. - STREAM_BITRATE: int = Field(default=4608, description="Stream bitrate hint (kbps).") + STREAM_BITRATE: int = Field( + default=2500, + description=( + "Stream bitrate hint (kbps). 2500 pairs well with the 480p24 " + "default and keeps libvpx's encoder buffer modest. Bump to " + "~4608 for 720p, ~6000+ for 1080p (you'll also need to lift " + "SCREEN_WIDTH/HEIGHT)." + ), + ) STREAM_ACCESSIBILITY: int = Field( default=0, description=( @@ -108,12 +120,14 @@ class Settings(BaseSettings): ), ) STREAM_VIEWER_LIMIT: int = Field( - default=4, + default=2, description=( "Max concurrent viewers. Each viewer spins up its own VP8 encoder " - "(~280 MB RSS at 720p30); without a cap the bot can OOM the host " - "before the operator notices. -1 = unlimited (TS3 convention) - " - "only set this if you've measured memory under load." + "(roughly 250 MB RSS at 480p24, more at higher resolutions); " + "without a cap the bot can OOM the host before the operator " + "notices, and on hosts where the TS6 server lives next to the " + "bot that takes the server out too. -1 = unlimited (TS3 " + "convention) - only set this if you've measured memory under load." ), ) @@ -136,6 +150,24 @@ class Settings(BaseSettings): TURN_USERNAME: str = Field(default="", description="TURN auth username.") TURN_PASSWORD: str = Field(default="", description="TURN auth password.") + # When the bot runs in `network_mode: host`, aiortc gathers a host + # candidate for *every* interface in the host namespace - that + # includes Docker's bridge gateways (172.16.0.0/12) which no + # external viewer can route to. Each useless candidate also + # spawns a srflx pair, blowing the offer SDP up to ~5 KB / 11 + # UDP fragments. Some TS6 server builds cope poorly with that + # volume and drop or crash. Filtering them out before send_join_response + # keeps the SDP small and the server stable. + ICE_DROP_NETWORKS: list[str] = Field( + default=["172.16.0.0/12"], + description=( + "CIDR blocks whose host / srflx ICE candidates are stripped " + "from the offer SDP before we hand it to the TS6 server. " + "Default targets Docker's bridge gateway range. Set to an " + "empty list to disable." + ), + ) + @field_validator("BOT_API_KEY") @classmethod def _reject_insecure_api_key(cls, v: str) -> str: diff --git a/src/ts6_stream_bot/pipeline/stream_publisher.py b/src/ts6_stream_bot/pipeline/stream_publisher.py index 558b2bf..393b827 100644 --- a/src/ts6_stream_bot/pipeline/stream_publisher.py +++ b/src/ts6_stream_bot/pipeline/stream_publisher.py @@ -24,6 +24,7 @@ import asyncio import contextlib +import ipaddress from dataclasses import dataclass, field import structlog @@ -334,6 +335,23 @@ async def _on_ice_state_change() -> None: offer_sdp = pc.localDescription.sdp + # Strip ICE candidates whose IP falls in a configured drop network + # (default: Docker's bridge gateway range, useless for any external + # viewer in host-network mode and bloats the offer past the point + # the TS6 server reliably forwards). aiortc keeps the full set + # internally - we only trim what we hand to the server. + if settings.ICE_DROP_NETWORKS: + offer_sdp, dropped = _filter_sdp_candidates( + offer_sdp, settings.ICE_DROP_NETWORKS + ) + if dropped: + log.info( + "stream_publisher.ice_candidates_filtered", + clid=viewer_clid, + dropped=dropped, + networks=settings.ICE_DROP_NETWORKS, + ) + # Surface the local ICE candidate set so we can tell at a glance # whether STUN/TURN actually contributed anything. The trace we # need to debug "ICE failed" almost always boils down to "the bot @@ -469,6 +487,58 @@ async def _wait_for_ice_gathering( await asyncio.sleep(poll) +def _filter_sdp_candidates( + sdp: str, drop_networks: list[str] +) -> tuple[str, list[dict[str, str]]]: + """Strip a=candidate: lines whose IP sits inside one of ``drop_networks``. + + Returns the rewritten SDP plus the structured list of candidates we + threw away (so callers can log them). Invalid CIDR strings are + silently ignored - we'd rather miss a filter than crash the join + flow over a typo in .env. + """ + parsed_nets: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + for raw in drop_networks: + try: + parsed_nets.append(ipaddress.ip_network(raw, strict=False)) + except ValueError: + log.warning("stream_publisher.bad_drop_network", value=raw) + if not parsed_nets: + return sdp, [] + + # Preserve the original line endings - WebRTC SDPs are CRLF-terminated + # and aiortc / browsers parse strictly. + lines = sdp.splitlines(keepends=True) + kept: list[str] = [] + dropped: list[dict[str, str]] = [] + for line in lines: + stripped = line.rstrip("\r\n") + if not stripped.startswith("a=candidate:"): + kept.append(line) + continue + parts = stripped[len("a=candidate:") :].split() + if len(parts) < 5: + kept.append(line) + continue + ip_str = parts[4] + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + kept.append(line) + continue + if any(ip in net for net in parsed_nets): + dropped.append( + { + "ip": ip_str, + "port": parts[5] if len(parts) > 5 else "?", + "typ": parts[7] if len(parts) > 7 and parts[6] == "typ" else "?", + } + ) + continue + kept.append(line) + return "".join(kept), dropped + + def _parse_local_candidates(sdp: str) -> list[dict[str, str]]: """Pull the c=...candidate lines out of an SDP into a structured form so the diagnostic log shows "what did STUN actually give us".""" @@ -494,4 +564,5 @@ def _parse_local_candidates(sdp: str) -> list[dict[str, str]]: __all__ = [ "StreamPublisher", "StreamPublisherStatus", + "_filter_sdp_candidates", ] diff --git a/tests/test_sdp_filter.py b/tests/test_sdp_filter.py new file mode 100644 index 0000000..5e5981e --- /dev/null +++ b/tests/test_sdp_filter.py @@ -0,0 +1,93 @@ +"""Tests for the ICE-candidate SDP filter. + +The filter strips a=candidate: lines whose IP sits inside any of the +configured drop networks. Default usage drops Docker bridge gateways +(172.16.0.0/12) so the offer SDP doesn't bloat past what the TS6 +server can reliably forward. +""" + +from __future__ import annotations + +from ts6_stream_bot.pipeline.stream_publisher import _filter_sdp_candidates + +_SDP = ( + "v=0\r\n" + "o=- 1 1 IN IP4 0.0.0.0\r\n" + "s=-\r\n" + "m=video 9 UDP/TLS/RTP/SAVPF 97\r\n" + "c=IN IP4 0.0.0.0\r\n" + "a=candidate:1 1 udp 2122260223 193.34.69.21 47453 typ host\r\n" + "a=candidate:2 1 udp 2122260223 172.18.0.1 51882 typ host\r\n" + "a=candidate:3 1 udp 2122260223 172.17.0.1 35938 typ host\r\n" + "a=candidate:4 1 udp 1686052607 193.34.69.21 35938 typ srflx raddr 172.17.0.1 rport 35938\r\n" + "a=ice-ufrag:abcd\r\n" + "a=fingerprint:sha-256 AA:BB\r\n" +) + + +def test_drops_candidates_inside_network() -> None: + """All host candidates in 172.16/12 should be removed.""" + rewritten, dropped = _filter_sdp_candidates(_SDP, ["172.16.0.0/12"]) + + # The dropped candidate lines themselves are gone. + assert "172.18.0.1 51882" not in rewritten + assert "172.17.0.1 35938 typ host" not in rewritten + + # Public host + srflx (which advertises the public IP as its + # connect address; the raddr=172.17.0.1 metadata is preserved + # because the filter keys on the candidate IP, not the raddr). + assert rewritten.count("a=candidate:") == 2 + assert "193.34.69.21 47453" in rewritten + assert "193.34.69.21 35938 typ srflx" in rewritten + + dropped_ips = sorted(d["ip"] for d in dropped) + assert dropped_ips == ["172.17.0.1", "172.18.0.1"] + + +def test_preserves_non_candidate_lines() -> None: + """Filtering candidate lines must not touch the rest of the SDP.""" + rewritten, _ = _filter_sdp_candidates(_SDP, ["172.16.0.0/12"]) + + for line in ("v=0", "o=- 1 1", "m=video", "a=ice-ufrag:abcd", "a=fingerprint:sha-256"): + assert line in rewritten + + +def test_keeps_crlf_line_endings() -> None: + """WebRTC SDPs are CRLF-terminated; aiortc / browsers parse strictly.""" + rewritten, _ = _filter_sdp_candidates(_SDP, ["172.16.0.0/12"]) + # Every retained line ends with CRLF. + assert rewritten.endswith("\r\n") + assert "\n\n" not in rewritten # no orphaned LF-only line endings + + +def test_empty_network_list_passes_sdp_through() -> None: + rewritten, dropped = _filter_sdp_candidates(_SDP, []) + assert rewritten == _SDP + assert dropped == [] + + +def test_invalid_cidr_is_logged_and_skipped() -> None: + """A typo in .env shouldn't crash the join flow.""" + rewritten, dropped = _filter_sdp_candidates(_SDP, ["not-a-cidr", "172.16.0.0/12"]) + # Valid network still applied. + assert "172.18.0.1" not in rewritten + assert {d["ip"] for d in dropped} == {"172.17.0.1", "172.18.0.1"} + + +def test_no_match_returns_full_sdp() -> None: + rewritten, dropped = _filter_sdp_candidates(_SDP, ["10.0.0.0/8"]) + assert rewritten == _SDP + assert dropped == [] + + +def test_ipv6_drop_network() -> None: + """The filter should also handle IPv6 networks correctly.""" + sdp = ( + "v=0\r\n" + "a=candidate:1 1 udp 2122131711 fd75:cbb5:9606::1 50226 typ host\r\n" + "a=candidate:2 1 udp 2121998079 192.168.178.29 50223 typ host\r\n" + ) + rewritten, dropped = _filter_sdp_candidates(sdp, ["fd00::/8"]) + assert "fd75:cbb5:9606::1" not in rewritten + assert "192.168.178.29" in rewritten + assert [d["ip"] for d in dropped] == ["fd75:cbb5:9606::1"]