diff --git a/container-runner/src/actor.rs b/container-runner/src/actor.rs index c13ea7e732..aa30f5ebab 100644 --- a/container-runner/src/actor.rs +++ b/container-runner/src/actor.rs @@ -15,7 +15,8 @@ use tokio::sync::Mutex as TokioMutex; use crate::child::{ChildProcess, SpawnSpec, log_prefix}; use crate::input::{ActorInput, ActorState}; use crate::{ - children, drain_grace, effective_stop_grace, exit_token, idle_timeout, reject_second_start, + children, drain_grace, effective_stop_grace, exit_token, idle_timeout, idle_timeout_with_jitter, + reject_second_start, release_child_port, request_exit, reserve_child_port, runner_config, }; @@ -94,21 +95,23 @@ impl GameServer { /// window, if no request has arrived, ask the actor to sleep (`stop_child` then /// exits the container). Cancelled early if the actor starts shutting down. fn arm_idle_timeout(self: &Arc, ctx: &Ctx, actor_id: String) { - let Some(timeout) = idle_timeout() else { + let Some(base) = idle_timeout() else { return; }; + // Jitter the window so instances started together do not sleep in lockstep. + let delay = idle_timeout_with_jitter(base); let this = self.clone(); let ctx = ctx.clone(); tokio::spawn(async move { let abort = ctx.abort_signal(); tokio::select! { - _ = tokio::time::sleep(timeout) => {} + _ = tokio::time::sleep(delay) => {} _ = abort.cancelled() => return, } if this.idle_state.load(Ordering::Relaxed) == IDLE_REQUESTED { return; } - tracing::info!(actor_id = %actor_id, ?timeout, "no request within idle timeout, sleeping"); + tracing::info!(actor_id = %actor_id, ?delay, "no request within idle timeout, sleeping"); if let Err(err) = ctx.sleep() { tracing::debug!(error = ?err, actor_id = %actor_id, "idle sleep request failed"); } diff --git a/container-runner/src/main.rs b/container-runner/src/main.rs index 9d7d7b6fbd..8486d49898 100644 --- a/container-runner/src/main.rs +++ b/container-runner/src/main.rs @@ -133,6 +133,28 @@ pub fn idle_timeout() -> Option { *IDLE_TIMEOUT } +/// The idle window plus up to 20% jitter (capped at 60s), so instances armed at the +/// same time do not all sleep in the same instant and tear down in a wave. Jitter is +/// only ever added, never subtracted, so an actor never sleeps before its window. +pub fn idle_timeout_with_jitter(base: Duration) -> Duration { + let max_jitter = base.mul_f64(0.2).min(Duration::from_secs(60)); + base + random_duration_up_to(max_jitter) +} + +/// A `Duration` uniformly in `[0, max]`, drawn from the OS CSPRNG. Falls back to no +/// jitter when the CSPRNG is unavailable. +fn random_duration_up_to(max: Duration) -> Duration { + let max_ms = max.as_millis() as u64; + if max_ms == 0 { + return Duration::ZERO; + } + let mut buf = [0u8; 8]; + match std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut buf)) { + Ok(()) => Duration::from_millis(u64::from_le_bytes(buf) % (max_ms + 1)), + Err(_) => Duration::ZERO, + } +} + /// When set, an actor that starts a second time self-sleeps instead of running /// again; its persisted `started_once` records the first real start. Configured via /// RIVET_REJECT_SECOND_START (truthy `1`/`true`/`yes`/`on`). Off by default.