fix(capture): align Windows cursor coordinates with DPI

Author: albi

packages/capture/Cargo.toml

@@ -36,6 +36,8 @@ windows = { version = "0.62", features = [
   "Win32_System_Threading",
   "Win32_UI_HiDpi",
   "Win32_UI_Input_KeyboardAndMouse",
+  "Win32_UI_Shell",
+  "Win32_UI_Shell_Common",
   "Win32_UI_WindowsAndMessaging",
 ] }
 

packages/capture/src/bin/capture-engine.rs

@@ -22,13 +22,50 @@ fn main() {
 }
 
 fn run() -> Result<(), capture::CaptureError> {
+    #[cfg(windows)]
+    configure_windows_dpi_awareness()?;
     capture::parent_watch::install_parent_death_guard()?;
     #[cfg(target_os = "macos")]
     return run_with_main_thread_cursor_sampling();
     #[cfg(not(target_os = "macos"))]
     run_blocking_protocol()
 }
 
+#[cfg(windows)]
+fn configure_windows_dpi_awareness() -> Result<(), capture::CaptureError> {
+    use windows::Win32::{
+        System::Threading::GetCurrentProcess,
+        UI::HiDpi::{
+            AreDpiAwarenessContextsEqual, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2,
+            GetDpiAwarenessContextForProcess, SetProcessDpiAwarenessContext,
+        },
+    };
+
+    // This process owns no UI. Set its coordinate system before discovery creates
+    // WGC items or worker threads so frames, source bounds and cursor positions
+    // all use physical per-monitor pixels.
+    // SAFETY: PMv2 is supported by every Windows version that supports WGC.
+    let error = match unsafe {
+        SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)
+    } {
+        Ok(()) => return Ok(()),
+        Err(error) => error,
+    };
+    // SAFETY: the current-process pseudo-handle is always valid and needs no close.
+    let process = unsafe { GetCurrentProcess() };
+    // SAFETY: querying the current process needs no caller-owned storage.
+    let current = unsafe { GetDpiAwarenessContextForProcess(process) };
+    // SAFETY: both handles are valid queried/predefined DPI contexts.
+    if unsafe { AreDpiAwarenessContextsEqual(current, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }
+        .as_bool()
+    {
+        return Ok(());
+    }
+    Err(capture::CaptureError::Backend(format!(
+        "Windows capture engine could not enable per-monitor DPI awareness: {error}",
+    )))
+}
+
 #[cfg(not(target_os = "macos"))]
 fn run_blocking_protocol() -> Result<(), capture::CaptureError> {
     let mut reader = BufReader::new(io::stdin().lock());

packages/capture/src/bin/capture_engine_tests/mod.rs

@@ -9,6 +9,18 @@ use capture::{
 use super::prepare_snapshot;
 use super::{Engine, handle};
 
+#[cfg(windows)]
+use std::process::{Command as ProcessCommand, Stdio};
+
+#[cfg(windows)]
+use windows::Win32::{
+    System::Threading::GetCurrentProcess,
+    UI::HiDpi::{
+        AreDpiAwarenessContextsEqual, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2,
+        GetDpiAwarenessContextForProcess, GetThreadDpiAwarenessContext,
+    },
+};
+
 #[test]
 fn idle_status_reports_screen_available() {
     let mut engine = Engine::default();
@@ -58,3 +70,100 @@ fn prepare_reuses_the_successful_portal_discovery_snapshot() -> Result<(), captu
     assert_eq!(prepared, discovered);
     Ok(())
 }
+
+#[cfg(windows)]
+#[test]
+fn capture_engine_sets_process_dpi_awareness_in_an_isolated_process() {
+    const CHILD_MARKER: &str = "BEAM_CAPTURE_ENGINE_DPI_TEST_CHILD";
+
+    if std::env::var_os(CHILD_MARKER).is_some() {
+        let run_result = super::run();
+        assert!(
+            run_result.is_ok(),
+            "capture-engine startup failed: {:?}",
+            run_result.err()
+        );
+
+        // SAFETY: the current process pseudo-handle is valid and needs no close.
+        let process = unsafe { GetCurrentProcess() };
+        // SAFETY: querying the current process DPI context requires no extra storage.
+        let process_context = unsafe { GetDpiAwarenessContextForProcess(process) };
+        // SAFETY: both handles are valid queried/predefined context values.
+        assert!(
+            unsafe {
+                AreDpiAwarenessContextsEqual(
+                    process_context,
+                    DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2,
+                )
+            }
+            .as_bool(),
+            "capture-engine process must use per-monitor-v2 coordinates"
+        );
+
+        let new_thread_is_per_monitor_v2 = std::thread::spawn(|| {
+            // SAFETY: reading this thread's current DPI context requires no extra storage.
+            let context = unsafe { GetThreadDpiAwarenessContext() };
+            // SAFETY: both handles are valid DPI-awareness context values.
+            unsafe {
+                AreDpiAwarenessContextsEqual(context, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)
+            }
+            .as_bool()
+        })
+        .join();
+        assert!(
+            matches!(new_thread_is_per_monitor_v2, Ok(true)),
+            "new capture threads must inherit per-monitor-v2 awareness"
+        );
+        return;
+    }
+
+    let executable = std::env::current_exe();
+    assert!(
+        executable.is_ok(),
+        "failed to resolve the capture-engine test executable: {:?}",
+        executable.as_ref().err()
+    );
+    let Ok(executable) = executable else {
+        return;
+    };
+
+    let child_result = ProcessCommand::new(executable)
+        .args([
+            "--exact",
+            "tests::capture_engine_sets_process_dpi_awareness_in_an_isolated_process",
+            "--nocapture",
+        ])
+        .env(CHILD_MARKER, "1")
+        .env_remove(capture::parent_watch::PARENT_PID_ENV)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn();
+    assert!(
+        child_result.is_ok(),
+        "failed to launch the isolated capture-engine test process: {:?}",
+        child_result.as_ref().err()
+    );
+    let Ok(mut child) = child_result else {
+        return;
+    };
+
+    // EOF lets run_blocking_protocol finish; the process-global DPI change stays
+    // in this subprocess and cannot affect the parallel test harness.
+    drop(child.stdin.take());
+    let output = child.wait_with_output();
+    assert!(
+        output.is_ok(),
+        "failed to collect the isolated capture-engine test output: {:?}",
+        output.as_ref().err()
+    );
+    let Ok(output) = output else {
+        return;
+    };
+    assert!(
+        output.status.success(),
+        "isolated capture-engine DPI test failed\nstdout:\n{}\nstderr:\n{}",
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr)
+    );
+}

packages/capture/src/cursor/backend.rs

@@ -82,6 +82,10 @@ pub struct CursorShapeCatalogEntry {
     rename_all_fields = "camelCase"
 )]
 pub enum CursorEvent {
+    Metadata {
+        session_ns: u64,
+        display_scale_factor: f64,
+    },
     Move {
         session_ns: u64,
         #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -176,7 +180,8 @@ pub fn telemetry_from_events(events: &[CursorEvent]) -> CursorTelemetrySidecar {
                     interaction_type: Some(interaction_type),
                 });
             }
-            CursorEvent::Shape { .. }
+            CursorEvent::Metadata { .. }
+            | CursorEvent::Shape { .. }
             | CursorEvent::Visibility { .. }
             | CursorEvent::CropChanged { .. } => {}
         }

packages/capture/src/cursor/recording_support_tests.rs

@@ -257,6 +257,79 @@ fn successful_worker_publishes_cursor_events_telemetry_shapes_and_input() {
     assert!(!temporary.path().join("input.partial.jsonl").exists());
 }
 
+#[test]
+fn display_scale_metadata_survives_finalization_without_telemetry_or_shapes() {
+    let temporary = tempfile::tempdir().expect("temporary directory");
+    let paths = paths(temporary.path());
+    let events = [
+        CursorEvent::Metadata {
+            session_ns: 0,
+            display_scale_factor: 1.5,
+        },
+        CursorEvent::Move {
+            session_ns: 33_000_000,
+            cursor_id: None,
+            pixel_x: 300,
+            pixel_y: 200,
+            normalized_x: 0.3,
+            normalized_y: 0.2,
+            visible: true,
+        },
+    ];
+    let cursor_lines = events
+        .iter()
+        .map(|event| serde_json::to_string(event).expect("cursor event JSON"))
+        .collect::<Vec<_>>()
+        .join("\n");
+    std::fs::write(&paths.partial, format!("{cursor_lines}\n")).expect("cursor partial");
+    std::fs::write(&paths.input_partial, "").expect("input partial");
+
+    finalize_after_worker(Ok(()), paths).expect("finalization");
+
+    let cursor_json = std::fs::read(temporary.path().join("cursor.json")).expect("cursor JSON");
+    let raw_events: Vec<serde_json::Value> =
+        serde_json::from_slice(&cursor_json).expect("raw cursor events");
+    assert_eq!(raw_events[0]["event"], "metadata");
+    assert_eq!(raw_events[0]["displayScaleFactor"], 1.5);
+    let events: Vec<CursorEvent> = serde_json::from_slice(&cursor_json).expect("cursor events");
+    assert!(matches!(
+        events.first(),
+        Some(CursorEvent::Metadata {
+            session_ns: 0,
+            display_scale_factor,
+        }) if *display_scale_factor == 1.5
+    ));
+    assert!(matches!(
+        events.get(1),
+        Some(CursorEvent::Move {
+            session_ns: 33_000_000,
+            pixel_x: 300,
+            pixel_y: 200,
+            normalized_x,
+            normalized_y,
+            ..
+        }) if *normalized_x == 0.3 && *normalized_y == 0.2
+    ));
+
+    let telemetry: CursorTelemetrySidecar = serde_json::from_slice(
+        &std::fs::read(temporary.path().join("telemetry.json")).expect("telemetry JSON"),
+    )
+    .expect("telemetry sidecar");
+    assert_eq!(telemetry.samples.len(), 1);
+    assert_eq!(telemetry.samples[0].time_ms, 33);
+    assert_eq!(
+        telemetry.samples[0].interaction_type,
+        Some(crate::cursor::CursorInteractionType::Move)
+    );
+
+    let shapes: std::collections::BTreeMap<String, CursorShapeCatalogEntry> =
+        serde_json::from_slice(
+            &std::fs::read(temporary.path().join("shapes.json")).expect("shape catalog JSON"),
+        )
+        .expect("shape catalog");
+    assert!(shapes.is_empty());
+}
+
 #[test]
 fn invalid_cursor_partial_is_preserved_for_recovery() {
     let temporary = tempfile::tempdir().expect("temporary directory");

packages/capture/src/cursor/win/capture.rs

@@ -3,14 +3,15 @@ use std::{ffi::c_void, mem::size_of, ptr, sync::OnceLock};
 use windows_capture::{monitor::Monitor, window::Window};
 
 use windows::Win32::{
-    Foundation::POINT,
+    Foundation::{HWND, POINT},
     Graphics::Gdi::{
         BI_RGB, BITMAP, BITMAPINFO, BITMAPINFOHEADER, CreateCompatibleDC, CreateDIBSection,
         DIB_RGB_COLORS, DeleteDC, DeleteObject, GetMonitorInfoW, GetObjectW, HGDIOBJ, HMONITOR,
         MONITORINFO, SelectObject,
     },
     UI::{
         Input::KeyboardAndMouse::{GetAsyncKeyState, VK_LBUTTON, VK_MBUTTON, VK_RBUTTON},
+        Shell::GetScaleFactorForMonitor,
         WindowsAndMessaging::{
             CURSOR_SHOWING, CURSORINFO, DI_NORMAL, DrawIconEx, GetCursorInfo, GetIconInfo,
             GetPhysicalCursorPos, HICON, ICONINFO, IDC_APPSTARTING, IDC_ARROW, IDC_CROSS, IDC_HAND,
@@ -44,6 +45,12 @@ pub struct WindowsCursorSample {
     pub shape: Option<WindowsCursorShape>,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct WindowsCursorSourceContext {
+    pub region: CaptureRegion,
+    pub display_scale_factor: Option<f64>,
+}
+
 pub fn sample_cursor(
     region: CaptureRegion,
     include_shape: bool,
@@ -78,7 +85,7 @@ pub fn sample_cursor(
     })
 }
 
-pub fn source_region(source_id: &SourceId) -> Result<CaptureRegion, CaptureError> {
+pub fn source_context(source_id: &SourceId) -> Result<WindowsCursorSourceContext, CaptureError> {
     let _physical_coordinates = super::dpi::PhysicalCoordinates::enter()?;
     if let Some(device_name) = source_id.as_str().strip_prefix("wgc:monitor:") {
         let monitor = Monitor::enumerate()
@@ -94,7 +101,11 @@ pub fn source_region(source_id: &SourceId) -> Result<CaptureRegion, CaptureError
         if !unsafe { GetMonitorInfoW(HMONITOR(monitor.as_raw_hmonitor()), &mut info) }.as_bool() {
             return Err(CaptureError::Backend("GetMonitorInfoW failed".into()));
         }
-        return region_from_rect(info.rcMonitor);
+        let monitor = HMONITOR(monitor.as_raw_hmonitor());
+        return Ok(WindowsCursorSourceContext {
+            region: region_from_rect(info.rcMonitor)?,
+            display_scale_factor: display_scale_factor(monitor),
+        });
     }
     if source_id.as_str().starts_with("wgc:window:") {
         let window = Window::enumerate()
@@ -104,13 +115,34 @@ pub fn source_region(source_id: &SourceId) -> Result<CaptureRegion, CaptureError
                 source_id.as_str() == format!("wgc:window:{:x}", window.as_raw_hwnd() as usize)
             })
             .ok_or_else(|| CaptureError::SourceNotFound(source_id.to_string()))?;
-        return region_from_rect(window.rect().map_err(backend_error)?);
+        let monitor = unsafe {
+            windows::Win32::Graphics::Gdi::MonitorFromWindow(
+                HWND(window.as_raw_hwnd()),
+                windows::Win32::Graphics::Gdi::MONITOR_DEFAULTTONEAREST,
+            )
+        };
+        return Ok(WindowsCursorSourceContext {
+            region: region_from_rect(window.rect().map_err(backend_error)?)?,
+            display_scale_factor: (!monitor.is_invalid())
+                .then(|| display_scale_factor(monitor))
+                .flatten(),
+        });
     }
     Err(CaptureError::InvalidConfiguration(format!(
         "{source_id} is not a Windows visual source"
     )))
 }
 
+pub fn source_region(source_id: &SourceId) -> Result<CaptureRegion, CaptureError> {
+    Ok(source_context(source_id)?.region)
+}
+
+fn display_scale_factor(monitor: HMONITOR) -> Option<f64> {
+    // SAFETY: the handle comes from monitor enumeration or MonitorFromWindow.
+    let percentage = unsafe { GetScaleFactorForMonitor(monitor) }.ok()?.0;
+    (percentage > 0).then(|| f64::from(percentage) / 100.0)
+}
+
 fn region_from_rect(rect: windows::Win32::Foundation::RECT) -> Result<CaptureRegion, CaptureError> {
     Ok(CaptureRegion {
         x: rect.left,