feat(web): audio playback (RDPSND) through an audioPlayback callback

crates/ironrdp-web/src/audio.rs

@@ -0,0 +1,203 @@
+//! Audio playback (MS-RDPEA over the RDPSND static channel).
+//!
+//! The backend lives inside the SVC processor, which must be `Send`, so it only forwards what the server
+//! sends to the session loop; the loop hands it to the JS callback registered with the `audio_playback`
+//! extension, which plays it.
+
+use std::borrow::Cow;
+
+use futures_channel::mpsc;
+use ironrdp::rdpsnd::client::RdpsndClientHandler;
+use ironrdp::rdpsnd::pdu::{AudioFormat, PitchPdu, VolumePdu, WaveFormat};
+use tracing::debug;
+use wasm_bindgen::JsValue;
+
+use crate::session::RdpInputEvent;
+
+/// What the RDPSND backend forwards to the session loop.
+#[derive(Debug)]
+pub(crate) enum AudioMessage {
+    /// A block of interleaved 16-bit little-endian PCM.
+    Wave {
+        sample_rate: u32,
+        channels: u16,
+        data: Vec<u8>,
+    },
+    /// Server volume, 0..=0xFFFF per channel.
+    Volume { left: u16, right: u16 },
+    /// The server stopped playback.
+    Close,
+}
+
+#[derive(Debug)]
+pub(crate) struct WasmAudioBackend {
+    tx: mpsc::UnboundedSender<RdpInputEvent>,
+    formats: Vec<AudioFormat>,
+}
+
+impl WasmAudioBackend {
+    pub(crate) fn new(tx: mpsc::UnboundedSender<RdpInputEvent>) -> Self {
+        // Uncompressed PCM only: Web Audio plays it directly, and it keeps the client simple.
+        let pcm = |rate: u32, channels: u16| AudioFormat {
+            format: WaveFormat::PCM,
+            n_channels: channels,
+            n_samples_per_sec: rate,
+            n_avg_bytes_per_sec: rate * u32::from(channels) * 2,
+            n_block_align: channels * 2,
+            bits_per_sample: 16,
+            data: None,
+        };
+        Self {
+            tx,
+            formats: vec![pcm(48_000, 2), pcm(44_100, 2), pcm(22_050, 2)],
+        }
+    }
+
+    fn send(&self, message: AudioMessage) {
+        if self.tx.unbounded_send(RdpInputEvent::Audio(message)).is_err() {
+            debug!("Audio message dropped: the session loop is gone");
+        }
+    }
+}
+
+impl RdpsndClientHandler for WasmAudioBackend {
+    fn get_formats(&self) -> &[AudioFormat] {
+        &self.formats
+    }
+
+    fn wave(&mut self, format: &AudioFormat, _ts: u32, data: Cow<'_, [u8]>) {
+        if format.format != WaveFormat::PCM || format.bits_per_sample != 16 {
+            debug!(?format, "Unsupported audio format; block dropped");
+            return;
+        }
+        self.send(AudioMessage::Wave {
+            sample_rate: format.n_samples_per_sec,
+            channels: format.n_channels,
+            data: data.into_owned(),
+        });
+    }
+
+    fn set_volume(&mut self, volume: VolumePdu) {
+        self.send(AudioMessage::Volume {
+            left: volume.volume_left,
+            right: volume.volume_right,
+        });
+    }
+
+    fn set_pitch(&mut self, _pitch: PitchPdu) {}
+
+    fn close(&mut self) {
+        self.send(AudioMessage::Close);
+    }
+}
+
+/// The message as the object the JS callback receives:
+/// `{ type: 'wave', sampleRate, channels, data: Uint8Array }`, `{ type: 'volume', left, right }` (0..1),
+/// or `{ type: 'close' }`.
+pub(crate) fn to_js(message: AudioMessage) -> JsValue {
+    let object = js_sys::Object::new();
+    let set = |key: &str, value: JsValue| {
+        let _ = js_sys::Reflect::set(&object, &JsValue::from_str(key), &value);
+    };
+    match message {
+        AudioMessage::Wave {
+            sample_rate,
+            channels,
+            data,
+        } => {
+            set("type", JsValue::from_str("wave"));
+            set("sampleRate", JsValue::from(sample_rate));
+            set("channels", JsValue::from(channels));
+            set("data", js_sys::Uint8Array::from(data.as_slice()).into());
+        }
+        AudioMessage::Volume { left, right } => {
+            set("type", JsValue::from_str("volume"));
+            set("left", JsValue::from(f64::from(left) / f64::from(u16::MAX)));
+            set("right", JsValue::from(f64::from(right) / f64::from(u16::MAX)));
+        }
+        AudioMessage::Close => set("type", JsValue::from_str("close")),
+    }
+    object.into()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn backend() -> (WasmAudioBackend, mpsc::UnboundedReceiver<RdpInputEvent>) {
+        let (tx, rx) = mpsc::unbounded();
+        (WasmAudioBackend::new(tx), rx)
+    }
+
+    fn next_audio(rx: &mut mpsc::UnboundedReceiver<RdpInputEvent>) -> Option<AudioMessage> {
+        match rx.try_recv() {
+            Ok(RdpInputEvent::Audio(message)) => Some(message),
+            _ => None,
+        }
+    }
+
+    #[test]
+    fn advertises_16_bit_pcm_only() {
+        let (backend, _rx) = backend();
+        let formats = backend.get_formats();
+        assert!(!formats.is_empty());
+        for format in formats {
+            assert_eq!(format.format, WaveFormat::PCM);
+            assert_eq!(format.bits_per_sample, 16);
+            assert_eq!(format.n_block_align, format.n_channels * 2);
+            assert_eq!(
+                format.n_avg_bytes_per_sec,
+                format.n_samples_per_sec * u32::from(format.n_block_align)
+            );
+        }
+    }
+
+    #[test]
+    fn forwards_pcm_blocks_with_their_format() {
+        let (mut backend, mut rx) = backend();
+        let format = backend.get_formats()[1].clone();
+        backend.wave(&format, 0, Cow::Borrowed(&[1, 2, 3, 4]));
+        match next_audio(&mut rx) {
+            Some(AudioMessage::Wave {
+                sample_rate,
+                channels,
+                data,
+            }) => {
+                assert_eq!(sample_rate, format.n_samples_per_sec);
+                assert_eq!(channels, format.n_channels);
+                assert_eq!(data, [1, 2, 3, 4]);
+            }
+            other => panic!("expected a wave block, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn drops_blocks_it_cannot_play() {
+        let (mut backend, mut rx) = backend();
+        let mut eight_bit = backend.get_formats()[0].clone();
+        eight_bit.bits_per_sample = 8;
+        backend.wave(&eight_bit, 0, Cow::Borrowed(&[0; 4]));
+        let mut compressed = backend.get_formats()[0].clone();
+        compressed.format = WaveFormat::ADPCM;
+        backend.wave(&compressed, 0, Cow::Borrowed(&[0; 4]));
+        assert!(next_audio(&mut rx).is_none());
+    }
+
+    #[test]
+    fn forwards_volume_and_close() {
+        let (mut backend, mut rx) = backend();
+        backend.set_volume(VolumePdu {
+            volume_left: 0x8000,
+            volume_right: 0xFFFF,
+        });
+        backend.close();
+        assert!(matches!(
+            next_audio(&mut rx),
+            Some(AudioMessage::Volume {
+                left: 0x8000,
+                right: 0xFFFF
+            })
+        ));
+        assert!(matches!(next_audio(&mut rx), Some(AudioMessage::Close)));
+    }
+}

crates/ironrdp-web/src/lib.rs

@@ -15,6 +15,7 @@ extern crate getrandom2 as _;
 extern crate getrandom4 as _;
 extern crate time as _;
 
+mod audio;
 mod canvas;
 mod clipboard;
 mod error;

crates/ironrdp-web/src/session.rs

@@ -90,6 +90,7 @@ struct SessionBuilderInner {
     printer_driver_name: Option<String>,
 
     use_display_control: bool,
+    audio_playback_callback: Option<js_sys::Function>,
     enable_credssp: bool,
     enable_server_pointer: bool,
     legacy_graphics: bool,
@@ -134,6 +135,7 @@ impl Default for SessionBuilderInner {
             printer_driver_name: None,
 
             use_display_control: false,
+            audio_playback_callback: None,
             enable_credssp: true,
             enable_server_pointer: true,
             legacy_graphics: false,
@@ -254,6 +256,10 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder {
             |vmconnect: String| { self.0.borrow_mut().vmconnect = Some(vmconnect) };
             |kdc_proxy_url: String| { self.0.borrow_mut().kdc_proxy_url = Some(kdc_proxy_url) };
             |display_control: bool| { self.0.borrow_mut().use_display_control = display_control };
+            |audio_playback: JsValue| {
+                // A function receiving the audio (see audio::to_js); anything else turns playback off.
+                self.0.borrow_mut().audio_playback_callback = audio_playback.dyn_into::<js_sys::Function>().ok();
+            };
             |enable_credssp: bool| { self.0.borrow_mut().enable_credssp = enable_credssp };
             |enable_server_pointer: bool| { self.0.borrow_mut().enable_server_pointer = enable_server_pointer };
             |legacy_graphics: bool| { self.0.borrow_mut().legacy_graphics = legacy_graphics };
@@ -426,6 +432,9 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder {
 
         let enable_credssp = self.0.borrow().enable_credssp;
         config.enable_credssp = enable_credssp;
+        // Audio playback (RDPSND) when a callback takes the audio.
+        let audio_playback_callback = self.0.borrow().audio_playback_callback.clone();
+        config.enable_audio_playback = audio_playback_callback.is_some();
 
         let enable_server_pointer = self.0.borrow().enable_server_pointer;
         config.enable_server_pointer = enable_server_pointer;
@@ -516,6 +525,9 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder {
             printer_driver_name,
             computer_name: client_name.clone(),
             use_display_control,
+            audio_backend: audio_playback_callback
+                .as_ref()
+                .map(|_| crate::audio::WasmAudioBackend::new(input_events_tx.clone())),
         })
         .await?;
 
@@ -536,6 +548,7 @@ impl iron_remote_desktop::SessionBuilder for SessionBuilder {
             render_canvas,
             set_cursor_style_callback,
             set_cursor_style_callback_context,
+            audio_playback_callback,
 
             input_events_rx: RefCell::new(Some(input_events_rx)),
             rdp_reader: RefCell::new(Some(rdp_reader)),
@@ -555,6 +568,8 @@ pub(crate) enum RdpInputEvent {
     /// Printer backend → event loop: a print job finished and its bytes are
     /// ready for delivery to JS. See [`crate::printer::PrinterBackendMessage`].
     Printer(crate::printer::PrinterBackendMessage),
+    /// RDPSND backend → event loop: audio for the JS callback. See [`crate::audio::AudioMessage`].
+    Audio(crate::audio::AudioMessage),
     FastPath(FastPathInputEvents),
     Resize {
         width: u32,
@@ -584,6 +599,7 @@ pub(crate) struct Session {
     render_canvas: HtmlCanvasElement,
     set_cursor_style_callback: js_sys::Function,
     set_cursor_style_callback_context: JsValue,
+    audio_playback_callback: Option<js_sys::Function>,
 
     // Consumed when `run` is called
     input_events_rx: RefCell<Option<mpsc::UnboundedReceiver<RdpInputEvent>>>,
@@ -886,6 +902,14 @@ impl iron_remote_desktop::Session for Session {
                             }
                             Vec::new()
                         }
+                        RdpInputEvent::Audio(message) => {
+                            if let Some(callback) = &self.audio_playback_callback {
+                                if let Err(error) = callback.call1(&JsValue::NULL, &crate::audio::to_js(message)) {
+                                    warn!(?error, "audio playback callback failed");
+                                }
+                            }
+                            Vec::new()
+                        }
                         RdpInputEvent::TerminateSession => {
                             active_stage.graceful_shutdown()
                                 .context("graceful shutdown")?
@@ -1584,6 +1608,7 @@ struct ConnectParams {
     /// `computer_name` when constructing the `Rdpdr` processor.
     computer_name: String,
     use_display_control: bool,
+    audio_backend: Option<crate::audio::WasmAudioBackend>,
 }
 
 fn default_printer_driver_name() -> String {
@@ -1634,6 +1659,7 @@ async fn connect(
         printer_driver_name,
         computer_name,
         use_display_control,
+        audio_backend,
     }: ConnectParams,
 ) -> Result<(connector::ConnectionResult, WebSocket), IronError> {
     let mut framed = ironrdp_futures::LocalFuturesFramed::new(ws);
@@ -1647,11 +1673,26 @@ async fn connect(
         connector.attach_static_channel(CliprdrClient::new(Box::new(clipboard_backend)));
     }
 
+    let audio = audio_backend.is_some();
+    if let Some(audio_backend) = audio_backend {
+        connector.attach_static_channel(Rdpsnd::new(Box::new(audio_backend)));
+        // Windows servers only start audio playback once RDPDR is up too (FreeRDP enables device redirection
+        // along with audio for the same reason); without a printer, a device-less RDPDR does it.
+        if printer_backend.is_none() {
+            connector.attach_static_channel(Rdpdr::new(
+                Box::new(ironrdp::rdpdr::NoopRdpdrBackend),
+                computer_name.clone(),
+            ));
+        }
+    }
+
     if let Some(printer_backend) = printer_backend {
         // Windows servers only speak on RDPDR when RDPSND is advertised too
-        // (MS-RDPEFS Appendix A<1>). We do not play audio in the web client,
-        // but the no-op RDPSND processor satisfies that channel dependency.
-        connector.attach_static_channel(Rdpsnd::new(Box::new(NoopRdpsndBackend)));
+        // (MS-RDPEFS Appendix A<1>). Without audio playback, the no-op RDPSND
+        // processor satisfies that channel dependency.
+        if !audio {
+            connector.attach_static_channel(Rdpsnd::new(Box::new(NoopRdpsndBackend)));
+        }
         connector.attach_static_channel(
             Rdpdr::new(Box::new(printer_backend), computer_name).with_printer_driver(
                 printer_device_id,

web-client/iron-remote-desktop-rdp/src/main.ts

@@ -45,6 +45,14 @@ export function displayControl(enable: boolean): Extension {
     return new Extension('display_control', enable);
 }
 
+/**
+ * Plays the remote desktop's audio: `callback` receives `{ type: 'wave', sampleRate, channels, data }`
+ * (interleaved 16-bit little-endian PCM), `{ type: 'volume', left, right }` (0..1) and `{ type: 'close' }`.
+ */
+export function audioPlayback(callback: (message: unknown) => void): Extension {
+    return new Extension('audio_playback', callback);
+}
+
 export function kdcProxyUrl(url: string): Extension {
     return new Extension('kdc_proxy_url', url);
 }