From d51dbf4eecd620b547f3ab211d4b6db23668c51a Mon Sep 17 00:00:00 2001 From: meanaverage <18336095+meanaverage@users.noreply.github.com> Date: Sat, 26 Sep 2026 11:26:22 -0700 Subject: [PATCH] feat(web): audio playback (RDPSND) through an `audioPlayback` callback The web client had no audio output. An RDPSND client backend (16-bit PCM at 48, 44.1 and 22.05 kHz) now forwards what the server sends to the session loop, which hands it to the JavaScript callback given with the new `audioPlayback(callback)` extension: `{ type: 'wave', sampleRate, channels, data }`, `{ type: 'volume', left, right }` and `{ type: 'close' }`. With a callback, audio playback is enabled in the client info; without one, nothing changes. Windows servers only start playback once RDPDR is up as well (FreeRDP enables device redirection along with audio for the same reason), so a device-less RDPDR is attached when no printer is. --- crates/ironrdp-web/src/audio.rs | 203 ++++++++++++++++++ crates/ironrdp-web/src/lib.rs | 1 + crates/ironrdp-web/src/session.rs | 47 +++- .../iron-remote-desktop-rdp/src/main.ts | 8 + 4 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 crates/ironrdp-web/src/audio.rs diff --git a/crates/ironrdp-web/src/audio.rs b/crates/ironrdp-web/src/audio.rs new file mode 100644 index 0000000000..5db0753068 --- /dev/null +++ b/crates/ironrdp-web/src/audio.rs @@ -0,0 +1,203 @@ +//! Audio playback (MS-RDPEA over the RDPSND static channel). +//! +//! The backend lives inside the SVC processor, which must be `Send`, so it only forwards what the server +//! sends to the session loop; the loop hands it to the JS callback registered with the `audio_playback` +//! extension, which plays it. + +use std::borrow::Cow; + +use futures_channel::mpsc; +use ironrdp::rdpsnd::client::RdpsndClientHandler; +use ironrdp::rdpsnd::pdu::{AudioFormat, PitchPdu, VolumePdu, WaveFormat}; +use tracing::debug; +use wasm_bindgen::JsValue; + +use crate::session::RdpInputEvent; + +/// What the RDPSND backend forwards to the session loop. +#[derive(Debug)] +pub(crate) enum AudioMessage { + /// A block of interleaved 16-bit little-endian PCM. + Wave { + sample_rate: u32, + channels: u16, + data: Vec, + }, + /// Server volume, 0..=0xFFFF per channel. + Volume { left: u16, right: u16 }, + /// The server stopped playback. + Close, +} + +#[derive(Debug)] +pub(crate) struct WasmAudioBackend { + tx: mpsc::UnboundedSender, + formats: Vec, +} + +impl WasmAudioBackend { + pub(crate) fn new(tx: mpsc::UnboundedSender) -> Self { + // Uncompressed PCM only: Web Audio plays it directly, and it keeps the client simple. + let pcm = |rate: u32, channels: u16| AudioFormat { + format: WaveFormat::PCM, + n_channels: channels, + n_samples_per_sec: rate, + n_avg_bytes_per_sec: rate * u32::from(channels) * 2, + n_block_align: channels * 2, + bits_per_sample: 16, + data: None, + }; + Self { + tx, + formats: vec![pcm(48_000, 2), pcm(44_100, 2), pcm(22_050, 2)], + } + } + + fn send(&self, message: AudioMessage) { + if self.tx.unbounded_send(RdpInputEvent::Audio(message)).is_err() { + debug!("Audio message dropped: the session loop is gone"); + } + } +} + +impl RdpsndClientHandler for WasmAudioBackend { + fn get_formats(&self) -> &[AudioFormat] { + &self.formats + } + + fn wave(&mut self, format: &AudioFormat, _ts: u32, data: Cow<'_, [u8]>) { + if format.format != WaveFormat::PCM || format.bits_per_sample != 16 { + debug!(?format, "Unsupported audio format; block dropped"); + return; + } + self.send(AudioMessage::Wave { + sample_rate: format.n_samples_per_sec, + channels: format.n_channels, + data: data.into_owned(), + }); + } + + fn set_volume(&mut self, volume: VolumePdu) { + self.send(AudioMessage::Volume { + left: volume.volume_left, + right: volume.volume_right, + }); + } + + fn set_pitch(&mut self, _pitch: PitchPdu) {} + + fn close(&mut self) { + self.send(AudioMessage::Close); + } +} + +/// The message as the object the JS callback receives: +/// `{ type: 'wave', sampleRate, channels, data: Uint8Array }`, `{ type: 'volume', left, right }` (0..1), +/// or `{ type: 'close' }`. +pub(crate) fn to_js(message: AudioMessage) -> JsValue { + let object = js_sys::Object::new(); + let set = |key: &str, value: JsValue| { + let _ = js_sys::Reflect::set(&object, &JsValue::from_str(key), &value); + }; + match message { + AudioMessage::Wave { + sample_rate, + channels, + data, + } => { + set("type", JsValue::from_str("wave")); + set("sampleRate", JsValue::from(sample_rate)); + set("channels", JsValue::from(channels)); + set("data", js_sys::Uint8Array::from(data.as_slice()).into()); + } + AudioMessage::Volume { left, right } => { + set("type", JsValue::from_str("volume")); + set("left", JsValue::from(f64::from(left) / f64::from(u16::MAX))); + set("right", JsValue::from(f64::from(right) / f64::from(u16::MAX))); + } + AudioMessage::Close => set("type", JsValue::from_str("close")), + } + object.into() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn backend() -> (WasmAudioBackend, mpsc::UnboundedReceiver) { + let (tx, rx) = mpsc::unbounded(); + (WasmAudioBackend::new(tx), rx) + } + + fn next_audio(rx: &mut mpsc::UnboundedReceiver) -> Option { + match rx.try_recv() { + Ok(RdpInputEvent::Audio(message)) => Some(message), + _ => None, + } + } + + #[test] + fn advertises_16_bit_pcm_only() { + let (backend, _rx) = backend(); + let formats = backend.get_formats(); + assert!(!formats.is_empty()); + for format in formats { + assert_eq!(format.format, WaveFormat::PCM); + assert_eq!(format.bits_per_sample, 16); + assert_eq!(format.n_block_align, format.n_channels * 2); + assert_eq!( + format.n_avg_bytes_per_sec, + format.n_samples_per_sec * u32::from(format.n_block_align) + ); + } + } + + #[test] + fn forwards_pcm_blocks_with_their_format() { + let (mut backend, mut rx) = backend(); + let format = backend.get_formats()[1].clone(); + backend.wave(&format, 0, Cow::Borrowed(&[1, 2, 3, 4])); + match next_audio(&mut rx) { + Some(AudioMessage::Wave { + sample_rate, + channels, + data, + }) => { + assert_eq!(sample_rate, format.n_samples_per_sec); + assert_eq!(channels, format.n_channels); + assert_eq!(data, [1, 2, 3, 4]); + } + other => panic!("expected a wave block, got {other:?}"), + } + } + + #[test] + fn drops_blocks_it_cannot_play() { + let (mut backend, mut rx) = backend(); + let mut eight_bit = backend.get_formats()[0].clone(); + eight_bit.bits_per_sample = 8; + backend.wave(&eight_bit, 0, Cow::Borrowed(&[0; 4])); + let mut compressed = backend.get_formats()[0].clone(); + compressed.format = WaveFormat::ADPCM; + backend.wave(&compressed, 0, Cow::Borrowed(&[0; 4])); + assert!(next_audio(&mut rx).is_none()); + } + + #[test] + fn forwards_volume_and_close() { + let (mut backend, mut rx) = backend(); + backend.set_volume(VolumePdu { + volume_left: 0x8000, + volume_right: 0xFFFF, + }); + backend.close(); + assert!(matches!( + next_audio(&mut rx), + Some(AudioMessage::Volume { + left: 0x8000, + right: 0xFFFF + }) + )); + assert!(matches!(next_audio(&mut rx), Some(AudioMessage::Close))); + } +} diff --git a/crates/ironrdp-web/src/lib.rs b/crates/ironrdp-web/src/lib.rs index 664e42e698..9002dbb72f 100644 --- a/crates/ironrdp-web/src/lib.rs +++ b/crates/ironrdp-web/src/lib.rs @@ -15,6 +15,7 @@ extern crate getrandom2 as _; extern crate getrandom4 as _; extern crate time as _; +mod audio; mod canvas; mod clipboard; mod error; diff --git a/crates/ironrdp-web/src/session.rs b/crates/ironrdp-web/src/session.rs index a47bd3d0c0..ee21f6b362 100644 --- a/crates/ironrdp-web/src/session.rs +++ b/crates/ironrdp-web/src/session.rs @@ -90,6 +90,7 @@ struct SessionBuilderInner { printer_driver_name: Option, use_display_control: bool, + audio_playback_callback: Option, enable_credssp: bool, enable_server_pointer: bool, legacy_graphics: bool, @@ -134,6 +135,7 @@ impl Default for SessionBuilderInner { printer_driver_name: None, use_display_control: false, + audio_playback_callback: None, enable_credssp: true, enable_server_pointer: true, legacy_graphics: false, @@ -254,6 +256,10 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder { |vmconnect: String| { self.0.borrow_mut().vmconnect = Some(vmconnect) }; |kdc_proxy_url: String| { self.0.borrow_mut().kdc_proxy_url = Some(kdc_proxy_url) }; |display_control: bool| { self.0.borrow_mut().use_display_control = display_control }; + |audio_playback: JsValue| { + // A function receiving the audio (see audio::to_js); anything else turns playback off. + self.0.borrow_mut().audio_playback_callback = audio_playback.dyn_into::().ok(); + }; |enable_credssp: bool| { self.0.borrow_mut().enable_credssp = enable_credssp }; |enable_server_pointer: bool| { self.0.borrow_mut().enable_server_pointer = enable_server_pointer }; |legacy_graphics: bool| { self.0.borrow_mut().legacy_graphics = legacy_graphics }; @@ -426,6 +432,9 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder { let enable_credssp = self.0.borrow().enable_credssp; config.enable_credssp = enable_credssp; + // Audio playback (RDPSND) when a callback takes the audio. + let audio_playback_callback = self.0.borrow().audio_playback_callback.clone(); + config.enable_audio_playback = audio_playback_callback.is_some(); let enable_server_pointer = self.0.borrow().enable_server_pointer; config.enable_server_pointer = enable_server_pointer; @@ -516,6 +525,9 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder { printer_driver_name, computer_name: client_name.clone(), use_display_control, + audio_backend: audio_playback_callback + .as_ref() + .map(|_| crate::audio::WasmAudioBackend::new(input_events_tx.clone())), }) .await?; @@ -536,6 +548,7 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder { render_canvas, set_cursor_style_callback, set_cursor_style_callback_context, + audio_playback_callback, input_events_rx: RefCell::new(Some(input_events_rx)), rdp_reader: RefCell::new(Some(rdp_reader)), @@ -555,6 +568,8 @@ pub(crate) enum RdpInputEvent { /// Printer backend → event loop: a print job finished and its bytes are /// ready for delivery to JS. See [`crate::printer::PrinterBackendMessage`]. Printer(crate::printer::PrinterBackendMessage), + /// RDPSND backend → event loop: audio for the JS callback. See [`crate::audio::AudioMessage`]. + Audio(crate::audio::AudioMessage), FastPath(FastPathInputEvents), Resize { width: u32, @@ -584,6 +599,7 @@ pub(crate) struct Session { render_canvas: HtmlCanvasElement, set_cursor_style_callback: js_sys::Function, set_cursor_style_callback_context: JsValue, + audio_playback_callback: Option, // Consumed when `run` is called input_events_rx: RefCell>>, @@ -886,6 +902,14 @@ impl iron_remote_desktop::Session for Session { } Vec::new() } + RdpInputEvent::Audio(message) => { + if let Some(callback) = &self.audio_playback_callback { + if let Err(error) = callback.call1(&JsValue::NULL, &crate::audio::to_js(message)) { + warn!(?error, "audio playback callback failed"); + } + } + Vec::new() + } RdpInputEvent::TerminateSession => { active_stage.graceful_shutdown() .context("graceful shutdown")? @@ -1584,6 +1608,7 @@ struct ConnectParams { /// `computer_name` when constructing the `Rdpdr` processor. computer_name: String, use_display_control: bool, + audio_backend: Option, } fn default_printer_driver_name() -> String { @@ -1634,6 +1659,7 @@ async fn connect( printer_driver_name, computer_name, use_display_control, + audio_backend, }: ConnectParams, ) -> Result<(connector::ConnectionResult, WebSocket), IronError> { let mut framed = ironrdp_futures::LocalFuturesFramed::new(ws); @@ -1647,11 +1673,26 @@ async fn connect( connector.attach_static_channel(CliprdrClient::new(Box::new(clipboard_backend))); } + let audio = audio_backend.is_some(); + if let Some(audio_backend) = audio_backend { + connector.attach_static_channel(Rdpsnd::new(Box::new(audio_backend))); + // Windows servers only start audio playback once RDPDR is up too (FreeRDP enables device redirection + // along with audio for the same reason); without a printer, a device-less RDPDR does it. + if printer_backend.is_none() { + connector.attach_static_channel(Rdpdr::new( + Box::new(ironrdp::rdpdr::NoopRdpdrBackend), + computer_name.clone(), + )); + } + } + if let Some(printer_backend) = printer_backend { // Windows servers only speak on RDPDR when RDPSND is advertised too - // (MS-RDPEFS Appendix A<1>). We do not play audio in the web client, - // but the no-op RDPSND processor satisfies that channel dependency. - connector.attach_static_channel(Rdpsnd::new(Box::new(NoopRdpsndBackend))); + // (MS-RDPEFS Appendix A<1>). Without audio playback, the no-op RDPSND + // processor satisfies that channel dependency. + if !audio { + connector.attach_static_channel(Rdpsnd::new(Box::new(NoopRdpsndBackend))); + } connector.attach_static_channel( Rdpdr::new(Box::new(printer_backend), computer_name).with_printer_driver( printer_device_id, diff --git a/web-client/iron-remote-desktop-rdp/src/main.ts b/web-client/iron-remote-desktop-rdp/src/main.ts index acd28b2e96..f7564fc7fc 100644 --- a/web-client/iron-remote-desktop-rdp/src/main.ts +++ b/web-client/iron-remote-desktop-rdp/src/main.ts @@ -45,6 +45,14 @@ export function displayControl(enable: boolean): Extension { return new Extension('display_control', enable); } +/** + * Plays the remote desktop's audio: `callback` receives `{ type: 'wave', sampleRate, channels, data }` + * (interleaved 16-bit little-endian PCM), `{ type: 'volume', left, right }` (0..1) and `{ type: 'close' }`. + */ +export function audioPlayback(callback: (message: unknown) => void): Extension { + return new Extension('audio_playback', callback); +} + export function kdcProxyUrl(url: string): Extension { return new Extension('kdc_proxy_url', url); }