Resource hardening for small hosts colocated with TS6

.env.example

@@ -6,11 +6,16 @@ BOT_API_KEY=
 # Logging
 LOG_LEVEL=INFO
 
-# Display / capture (defaults are sane, only change if you know why)
+# Display / capture
+# Defaults are tuned for a 4 GB host that runs the TS6 server next to
+# the bot. Each viewer adds ~250 MB RSS at 480p24 (the default), and
+# the encoder cost scales roughly with pixels * fps. Bump these only
+# if your host has the budget AND your viewers can decode 720p+ in
+# their TS6 client.
 DISPLAY=:99
-SCREEN_WIDTH=1920
-SCREEN_HEIGHT=1080
-SCREEN_FPS=30
+SCREEN_WIDTH=854
+SCREEN_HEIGHT=480
+SCREEN_FPS=24
 
 # PulseAudio
 PULSE_SINK=bot_sink
@@ -53,17 +58,20 @@ TS6_CHANNEL_PASSWORD=
 #   for each viewer and the bot replies via respondjoinstreamrequest);
 #   0 = auto-accept.
 # - STREAM_VIEWER_LIMIT: each viewer spins up its own VP8 encoder
-#   (~280 MB RSS at 720p30) so unlimited can OOM a small host. The default
-#   of 4 is conservative; bump it once you've measured memory under load.
-#   -1 = unlimited (TS3 convention). 0 is interpreted by some TS6 builds
-#   as "no viewers allowed", which silently drops every join attempt -
-#   don't use 0.
-# - STREAM_BITRATE: hint to the server (kbps); 4608 mirrors the default the
-#   TS6 client uses for "high quality" screen share.
-STREAM_BITRATE=4608
+#   (roughly 250 MB RSS at the 480p24 default; more at 720p / 1080p).
+#   On a 4 GB host that also runs the TS6 server, two viewers is the
+#   safe ceiling - going higher squeezed the TS6 server out of RAM
+#   in our live tests, which then crashed the whole stack. Bump once
+#   you've measured. -1 = unlimited (TS3 convention). 0 is interpreted
+#   by some TS6 builds as "no viewers allowed" - don't use 0.
+# - STREAM_BITRATE: hint to the server (kbps). 2500 pairs with 480p24;
+#   ~4608 with 720p; ~6000+ with 1080p. Both the bitrate and the
+#   resolution feed into libvpx's encoder buffer, so drop both
+#   together if you hit OOM, raise both together if you have headroom.
+STREAM_BITRATE=2500
 STREAM_ACCESSIBILITY=0
 STREAM_MODE=1
-STREAM_VIEWER_LIMIT=4
+STREAM_VIEWER_LIMIT=2
 
 # --- WebRTC ICE servers ----------------------------------------------------
 # STUN exposes the bot's public-NAT'd IP as a server-reflexive candidate;
@@ -74,3 +82,12 @@ STUN_URL=stun:stun.l.google.com:19302
 # TURN_URL=turn:openrelay.metered.ca:80
 # TURN_USERNAME=openrelayproject
 # TURN_PASSWORD=openrelayproject
+
+# CIDR blocks whose ICE candidates get stripped from the offer SDP we
+# hand to the TS6 server. In host-network mode aiortc gathers a host
+# candidate per docker bridge interface (172.x.x.x); each such useless
+# candidate also spawns a srflx pair, blowing the offer to ~5 KB / 11
+# UDP fragments which some TS6 server builds choke on. Default drops
+# the entire RFC1918 docker range. Set to an empty value (or a
+# different CIDR) to override; multiple values are comma-separated.
+# ICE_DROP_NETWORKS=172.16.0.0/12

src/ts6_stream_bot/config.py

@@ -34,15 +34,19 @@ class Settings(BaseSettings):
     LOG_LEVEL: str = "INFO"
 
     # --- Display / capture -------------------------------------------------
-    # 720p30 keeps the per-viewer VP8 encode out of OOM territory on
-    # small hosts. aiortc + PyAV at 1080p30 with even one peer can run a
-    # 4 GB host out of memory (live deploy hit SIGKILL after ~6 s of
-    # streaming). Bump SCREEN_WIDTH/SCREEN_HEIGHT in .env if your host
-    # has the budget; you'll also want to bump STREAM_BITRATE.
+    # Defaults are tuned for a 4 GB host running both the bot and the TS6
+    # server side by side. Each viewer adds its own VP8 encoder (PyAV +
+    # libvpx through aiortc); live measurement at 720p30 / 4608 kbps had
+    # one viewer push the bot's RSS past 750 MB and still climbing - on
+    # a 4 GB box, two viewers + the TS6 server is enough to OOM the
+    # whole stack and crash the server with it. 480p24 / 2500 kbps cuts
+    # the encoder's reference-frame pool roughly in half. Bump every
+    # value if your host has the budget; you'll usually want to scale
+    # SCREEN_WIDTH/HEIGHT and STREAM_BITRATE together.
     DISPLAY: str = ":99"
-    SCREEN_WIDTH: int = 1280
-    SCREEN_HEIGHT: int = 720
-    SCREEN_FPS: int = 30
+    SCREEN_WIDTH: int = 854
+    SCREEN_HEIGHT: int = 480
+    SCREEN_FPS: int = 24
 
     # --- Audio -------------------------------------------------------------
     PULSE_SINK: str = "bot_sink"
@@ -91,7 +95,15 @@ class Settings(BaseSettings):
     # --- Stream parameters -------------------------------------------------
     # These shape the `setupstream` request the bot sends on connect.
     # Defaults match what the TS6 client UI uses for a normal screen-share.
-    STREAM_BITRATE: int = Field(default=4608, description="Stream bitrate hint (kbps).")
+    STREAM_BITRATE: int = Field(
+        default=2500,
+        description=(
+            "Stream bitrate hint (kbps). 2500 pairs well with the 480p24 "
+            "default and keeps libvpx's encoder buffer modest. Bump to "
+            "~4608 for 720p, ~6000+ for 1080p (you'll also need to lift "
+            "SCREEN_WIDTH/HEIGHT)."
+        ),
+    )
     STREAM_ACCESSIBILITY: int = Field(
         default=0,
         description=(
@@ -108,12 +120,14 @@ class Settings(BaseSettings):
         ),
     )
     STREAM_VIEWER_LIMIT: int = Field(
-        default=4,
+        default=2,
         description=(
             "Max concurrent viewers. Each viewer spins up its own VP8 encoder "
-            "(~280 MB RSS at 720p30); without a cap the bot can OOM the host "
-            "before the operator notices. -1 = unlimited (TS3 convention) - "
-            "only set this if you've measured memory under load."
+            "(roughly 250 MB RSS at 480p24, more at higher resolutions); "
+            "without a cap the bot can OOM the host before the operator "
+            "notices, and on hosts where the TS6 server lives next to the "
+            "bot that takes the server out too. -1 = unlimited (TS3 "
+            "convention) - only set this if you've measured memory under load."
         ),
     )
 
@@ -136,6 +150,24 @@ class Settings(BaseSettings):
     TURN_USERNAME: str = Field(default="", description="TURN auth username.")
     TURN_PASSWORD: str = Field(default="", description="TURN auth password.")
 
+    # When the bot runs in `network_mode: host`, aiortc gathers a host
+    # candidate for *every* interface in the host namespace - that
+    # includes Docker's bridge gateways (172.16.0.0/12) which no
+    # external viewer can route to. Each useless candidate also
+    # spawns a srflx pair, blowing the offer SDP up to ~5 KB / 11
+    # UDP fragments. Some TS6 server builds cope poorly with that
+    # volume and drop or crash. Filtering them out before send_join_response
+    # keeps the SDP small and the server stable.
+    ICE_DROP_NETWORKS: list[str] = Field(
+        default=["172.16.0.0/12"],
+        description=(
+            "CIDR blocks whose host / srflx ICE candidates are stripped "
+            "from the offer SDP before we hand it to the TS6 server. "
+            "Default targets Docker's bridge gateway range. Set to an "
+            "empty list to disable."
+        ),
+    )
+
     @field_validator("BOT_API_KEY")
     @classmethod
     def _reject_insecure_api_key(cls, v: str) -> str:

src/ts6_stream_bot/pipeline/stream_publisher.py

@@ -24,6 +24,7 @@
 
 import asyncio
 import contextlib
+import ipaddress
 from dataclasses import dataclass, field
 
 import structlog
@@ -334,6 +335,23 @@ async def _on_ice_state_change() -> None:
 
         offer_sdp = pc.localDescription.sdp
 
+        # Strip ICE candidates whose IP falls in a configured drop network
+        # (default: Docker's bridge gateway range, useless for any external
+        # viewer in host-network mode and bloats the offer past the point
+        # the TS6 server reliably forwards). aiortc keeps the full set
+        # internally - we only trim what we hand to the server.
+        if settings.ICE_DROP_NETWORKS:
+            offer_sdp, dropped = _filter_sdp_candidates(
+                offer_sdp, settings.ICE_DROP_NETWORKS
+            )
+            if dropped:
+                log.info(
+                    "stream_publisher.ice_candidates_filtered",
+                    clid=viewer_clid,
+                    dropped=dropped,
+                    networks=settings.ICE_DROP_NETWORKS,
+                )
+
         # Surface the local ICE candidate set so we can tell at a glance
         # whether STUN/TURN actually contributed anything. The trace we
         # need to debug "ICE failed" almost always boils down to "the bot
@@ -469,6 +487,58 @@ async def _wait_for_ice_gathering(
             await asyncio.sleep(poll)
 
 
+def _filter_sdp_candidates(
+    sdp: str, drop_networks: list[str]
+) -> tuple[str, list[dict[str, str]]]:
+    """Strip a=candidate: lines whose IP sits inside one of ``drop_networks``.
+
+    Returns the rewritten SDP plus the structured list of candidates we
+    threw away (so callers can log them). Invalid CIDR strings are
+    silently ignored - we'd rather miss a filter than crash the join
+    flow over a typo in .env.
+    """
+    parsed_nets: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
+    for raw in drop_networks:
+        try:
+            parsed_nets.append(ipaddress.ip_network(raw, strict=False))
+        except ValueError:
+            log.warning("stream_publisher.bad_drop_network", value=raw)
+    if not parsed_nets:
+        return sdp, []
+
+    # Preserve the original line endings - WebRTC SDPs are CRLF-terminated
+    # and aiortc / browsers parse strictly.
+    lines = sdp.splitlines(keepends=True)
+    kept: list[str] = []
+    dropped: list[dict[str, str]] = []
+    for line in lines:
+        stripped = line.rstrip("\r\n")
+        if not stripped.startswith("a=candidate:"):
+            kept.append(line)
+            continue
+        parts = stripped[len("a=candidate:") :].split()
+        if len(parts) < 5:
+            kept.append(line)
+            continue
+        ip_str = parts[4]
+        try:
+            ip = ipaddress.ip_address(ip_str)
+        except ValueError:
+            kept.append(line)
+            continue
+        if any(ip in net for net in parsed_nets):
+            dropped.append(
+                {
+                    "ip": ip_str,
+                    "port": parts[5] if len(parts) > 5 else "?",
+                    "typ": parts[7] if len(parts) > 7 and parts[6] == "typ" else "?",
+                }
+            )
+            continue
+        kept.append(line)
+    return "".join(kept), dropped
+
+
 def _parse_local_candidates(sdp: str) -> list[dict[str, str]]:
     """Pull the c=...candidate lines out of an SDP into a structured form
     so the diagnostic log shows "what did STUN actually give us"."""
@@ -494,4 +564,5 @@ def _parse_local_candidates(sdp: str) -> list[dict[str, str]]:
 __all__ = [
     "StreamPublisher",
     "StreamPublisherStatus",
+    "_filter_sdp_candidates",
 ]

tests/test_sdp_filter.py

@@ -0,0 +1,93 @@
+"""Tests for the ICE-candidate SDP filter.
+
+The filter strips a=candidate: lines whose IP sits inside any of the
+configured drop networks. Default usage drops Docker bridge gateways
+(172.16.0.0/12) so the offer SDP doesn't bloat past what the TS6
+server can reliably forward.
+"""
+
+from __future__ import annotations
+
+from ts6_stream_bot.pipeline.stream_publisher import _filter_sdp_candidates
+
+_SDP = (
+    "v=0\r\n"
+    "o=- 1 1 IN IP4 0.0.0.0\r\n"
+    "s=-\r\n"
+    "m=video 9 UDP/TLS/RTP/SAVPF 97\r\n"
+    "c=IN IP4 0.0.0.0\r\n"
+    "a=candidate:1 1 udp 2122260223 193.34.69.21 47453 typ host\r\n"
+    "a=candidate:2 1 udp 2122260223 172.18.0.1 51882 typ host\r\n"
+    "a=candidate:3 1 udp 2122260223 172.17.0.1 35938 typ host\r\n"
+    "a=candidate:4 1 udp 1686052607 193.34.69.21 35938 typ srflx raddr 172.17.0.1 rport 35938\r\n"
+    "a=ice-ufrag:abcd\r\n"
+    "a=fingerprint:sha-256 AA:BB\r\n"
+)
+
+
+def test_drops_candidates_inside_network() -> None:
+    """All host candidates in 172.16/12 should be removed."""
+    rewritten, dropped = _filter_sdp_candidates(_SDP, ["172.16.0.0/12"])
+
+    # The dropped candidate lines themselves are gone.
+    assert "172.18.0.1 51882" not in rewritten
+    assert "172.17.0.1 35938 typ host" not in rewritten
+
+    # Public host + srflx (which advertises the public IP as its
+    # connect address; the raddr=172.17.0.1 metadata is preserved
+    # because the filter keys on the candidate IP, not the raddr).
+    assert rewritten.count("a=candidate:") == 2
+    assert "193.34.69.21 47453" in rewritten
+    assert "193.34.69.21 35938 typ srflx" in rewritten
+
+    dropped_ips = sorted(d["ip"] for d in dropped)
+    assert dropped_ips == ["172.17.0.1", "172.18.0.1"]
+
+
+def test_preserves_non_candidate_lines() -> None:
+    """Filtering candidate lines must not touch the rest of the SDP."""
+    rewritten, _ = _filter_sdp_candidates(_SDP, ["172.16.0.0/12"])
+
+    for line in ("v=0", "o=- 1 1", "m=video", "a=ice-ufrag:abcd", "a=fingerprint:sha-256"):
+        assert line in rewritten
+
+
+def test_keeps_crlf_line_endings() -> None:
+    """WebRTC SDPs are CRLF-terminated; aiortc / browsers parse strictly."""
+    rewritten, _ = _filter_sdp_candidates(_SDP, ["172.16.0.0/12"])
+    # Every retained line ends with CRLF.
+    assert rewritten.endswith("\r\n")
+    assert "\n\n" not in rewritten  # no orphaned LF-only line endings
+
+
+def test_empty_network_list_passes_sdp_through() -> None:
+    rewritten, dropped = _filter_sdp_candidates(_SDP, [])
+    assert rewritten == _SDP
+    assert dropped == []
+
+
+def test_invalid_cidr_is_logged_and_skipped() -> None:
+    """A typo in .env shouldn't crash the join flow."""
+    rewritten, dropped = _filter_sdp_candidates(_SDP, ["not-a-cidr", "172.16.0.0/12"])
+    # Valid network still applied.
+    assert "172.18.0.1" not in rewritten
+    assert {d["ip"] for d in dropped} == {"172.17.0.1", "172.18.0.1"}
+
+
+def test_no_match_returns_full_sdp() -> None:
+    rewritten, dropped = _filter_sdp_candidates(_SDP, ["10.0.0.0/8"])
+    assert rewritten == _SDP
+    assert dropped == []
+
+
+def test_ipv6_drop_network() -> None:
+    """The filter should also handle IPv6 networks correctly."""
+    sdp = (
+        "v=0\r\n"
+        "a=candidate:1 1 udp 2122131711 fd75:cbb5:9606::1 50226 typ host\r\n"
+        "a=candidate:2 1 udp 2121998079 192.168.178.29 50223 typ host\r\n"
+    )
+    rewritten, dropped = _filter_sdp_candidates(sdp, ["fd00::/8"])
+    assert "fd75:cbb5:9606::1" not in rewritten
+    assert "192.168.178.29" in rewritten
+    assert [d["ip"] for d in dropped] == ["fd75:cbb5:9606::1"]