feat(container-runner): jitter idle timeout to avoid teardown waves

container-runner/src/actor.rs

@@ -12,10 +12,11 @@
 use rivetkit::{Actor, ActorKeySegment, Ctx, Request, Response, WebSocket, action};
 use tokio::sync::Mutex as TokioMutex;
 
 use crate::child::{ChildProcess, SpawnSpec, log_prefix};
 use crate::input::{ActorInput, ActorState};
 use crate::{
-	children, drain_grace, effective_stop_grace, exit_token, idle_timeout, reject_second_start,
+	children, drain_grace, effective_stop_grace, exit_token, idle_timeout, idle_timeout_with_jitter,
+	reject_second_start,
 	release_child_port, request_exit, reserve_child_port, runner_config,
 };
 
@@ -94,21 +95,23 @@
 	/// window, if no request has arrived, ask the actor to sleep (`stop_child` then
 	/// exits the container). Cancelled early if the actor starts shutting down.
 	fn arm_idle_timeout(self: &Arc<Self>, ctx: &Ctx<Self>, actor_id: String) {
-		let Some(timeout) = idle_timeout() else {
+		let Some(base) = idle_timeout() else {
 			return;
 		};
+		// Jitter the window so instances started together do not sleep in lockstep.
+		let delay = idle_timeout_with_jitter(base);
 		let this = self.clone();
 		let ctx = ctx.clone();
 		tokio::spawn(async move {
 			let abort = ctx.abort_signal();
 			tokio::select! {
-				_ = tokio::time::sleep(timeout) => {}
+				_ = tokio::time::sleep(delay) => {}
 				_ = abort.cancelled() => return,
 			}
 			if this.idle_state.load(Ordering::Relaxed) == IDLE_REQUESTED {
 				return;
 			}
-			tracing::info!(actor_id = %actor_id, ?timeout, "no request within idle timeout, sleeping");
+			tracing::info!(actor_id = %actor_id, ?delay, "no request within idle timeout, sleeping");
 			if let Err(err) = ctx.sleep() {
 				tracing::debug!(error = ?err, actor_id = %actor_id, "idle sleep request failed");
 			}
@@ -362,7 +365,7 @@
 	/// Engine-initiated sleep. `no_sleep` blocks only idle sleep; the engine can
 	/// still sleep an actor (dashboard, crash policy, eviction), so we stop the child.
 	/// In idle-timeout mode the sleep is (treated as) an idle sleep, so it skips the
 	/// drain and stops promptly; otherwise it drains for in-flight work first.
 	async fn on_sleep(self: Arc<Self>, ctx: Ctx<Self>) -> Result<()> {
 		if idle_timeout().is_some() {
 			self.stop_child(ctx.actor_id(), "actor sleeping (idle)").await;

container-runner/src/main.rs

@@ -133,9 +133,31 @@
 	*IDLE_TIMEOUT
 }
 
+/// The idle window plus up to 20% jitter (capped at 60s), so instances armed at the
+/// same time do not all sleep in the same instant and tear down in a wave. Jitter is
+/// only ever added, never subtracted, so an actor never sleeps before its window.
+pub fn idle_timeout_with_jitter(base: Duration) -> Duration {
+	let max_jitter = base.mul_f64(0.2).min(Duration::from_secs(60));
+	base + random_duration_up_to(max_jitter)
+}
+
+/// A `Duration` uniformly in `[0, max]`, drawn from the OS CSPRNG. Falls back to no
+/// jitter when the CSPRNG is unavailable.
+fn random_duration_up_to(max: Duration) -> Duration {
+	let max_ms = max.as_millis() as u64;
+	if max_ms == 0 {
+		return Duration::ZERO;
+	}
+	let mut buf = [0u8; 8];
+	match std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut buf)) {
+		Ok(()) => Duration::from_millis(u64::from_le_bytes(buf) % (max_ms + 1)),
+		Err(_) => Duration::ZERO,
+	}
+}
+
 /// When set, an actor that starts a second time self-sleeps instead of running
 /// again; its persisted `started_once` records the first real start. Configured via
 /// RIVET_REJECT_SECOND_START (truthy `1`/`true`/`yes`/`on`). Off by default.
 static REJECT_SECOND_START: LazyLock<bool> = LazyLock::new(|| {
 	std::env::var("RIVET_REJECT_SECOND_START")
 		.map(|value| matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))