From 2d91104c66d69931524a12a89abc7602eb727606 Mon Sep 17 00:00:00 2001
From: Jeremy Whiting <jpwhiting@kde.org>
Date: Wed, 4 Jun 2025 15:24:58 -0600
Subject: [PATCH] screenreader: Use uinput to send keyboard events.

Orca has no API, so use keyboard events to trigger specific actions
like sticking to focus mode, browse mode, etc.
Also add new get and set methods to steamosctl for mode.
---
 Cargo.lock                           |  37 +++++-
 Cargo.toml                           |   1 +
 com.steampowered.SteamOSManager1.xml |   9 ++
 data/user/steamos-manager.service    |   1 +
 src/bin/steamosctl.rs                |  22 ++++
 src/lib.rs                           |   2 +-
 src/manager/user.rs                  |  24 +++-
 src/proxy/screenreader0.rs           |   6 +
 src/screenreader.rs                  | 168 +++++++++++++++++++++++++++
 9 files changed, 266 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a2e9c7d..83b93c5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -563,6 +563,26 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "input-linux"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7e8c4821c88b95582ca69234a1d233f87e44182c42e121f740efb0bec1142e0"
+dependencies = [
+ "input-linux-sys",
+ "nix 0.29.0",
+]
+
+[[package]]
+name = "input-linux-sys"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b91b2248b0eaf0a576ef5e60b7f2107a749e705a876bc0b9fe952ac8d83a724"
+dependencies = [
+ "libc",
+ "nix 0.29.0",
+]
+
 [[package]]
 name = "io-lifetimes"
 version = "1.0.11"
@@ -661,6 +681,18 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "nix"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
 [[package]]
 name = "nix"
 version = "0.30.1"
@@ -1000,10 +1032,11 @@ dependencies = [
  "config",
  "gio",
  "inotify",
+ "input-linux",
  "itertools",
  "lazy_static",
  "libc",
- "nix",
+ "nix 0.30.1",
  "num_enum",
  "regex",
  "serde",
@@ -1510,7 +1543,7 @@ dependencies = [
  "futures-core",
  "futures-lite",
  "hex",
- "nix",
+ "nix 0.30.1",
  "ordered-stream",
  "serde",
  "serde_repr",
diff --git a/Cargo.toml b/Cargo.toml
index b9dd2aa..3599834 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,6 +14,7 @@ clap = { version = "4.5", default-features = false, features = ["derive", "help"
 config = { version = "0.15", default-features = false, features = ["async", "ini", "toml"] }
 gio = "0.20"
 inotify = { version = "0.11", default-features = false, features = ["stream"] }
+input-linux = "0.7"
 itertools = "0.14"
 lazy_static = "1"
 libc = "0.2"
diff --git a/com.steampowered.SteamOSManager1.xml b/com.steampowered.SteamOSManager1.xml
index f5b8648..f3fcd04 100644
--- a/com.steampowered.SteamOSManager1.xml
+++ b/com.steampowered.SteamOSManager1.xml
@@ -333,6 +333,15 @@
         The volume for speech output. Valid values ar 0.0 for off, 10.0 for highest.
     -->
     <property name="Volume" type="d" access="readwrite"/>
+
+    <!--
+        Mode
+
+        Which mode the screen reader should operate in.
+
+        Valid modes: 0 - Browse mode, 1 - Focus mode.
+    -->
+    <property name="Mode" type="u" access="readwrite"/>
   </interface>
 
   <!--
diff --git a/data/user/steamos-manager.service b/data/user/steamos-manager.service
index c147e40..de25c51 100644
--- a/data/user/steamos-manager.service
+++ b/data/user/steamos-manager.service
@@ -9,3 +9,4 @@ Environment=RUST_LOG='INFO'
 ExecStart=/usr/lib/steamos-manager
 Restart=on-failure
 RestartSec=1
+EnvironmentFile=%t/gamescope-environment
diff --git a/src/bin/steamosctl.rs b/src/bin/steamosctl.rs
index d0b3627..cb70bd1 100644
--- a/src/bin/steamosctl.rs
+++ b/src/bin/steamosctl.rs
@@ -20,6 +20,7 @@ use steamos_manager::proxy::{
     TdpLimit1Proxy, UpdateBios1Proxy, UpdateDock1Proxy, WifiDebug1Proxy, WifiDebugDump1Proxy,
     WifiPowerManagement1Proxy,
 };
+use steamos_manager::screenreader::ScreenReaderMode;
 use steamos_manager::wifi::{WifiBackend, WifiDebugMode, WifiPowerManagement};
 use zbus::fdo::{IntrospectableProxy, PropertiesProxy};
 use zbus::{zvariant, Connection};
@@ -242,6 +243,15 @@ enum Commands {
         /// Valid volume between 0.0 for off, and 10.0 for loudest.
         volume: f64,
     },
+
+    /// Get screen reader mode
+    GetScreenReaderMode,
+
+    /// Set screen reader mode
+    SetScreenReaderMode {
+        /// Valid modes are `browse`, `focus`
+        mode: ScreenReaderMode,
+    },
 }
 
 async fn get_all_properties(conn: &Connection) -> Result<()> {
@@ -591,6 +601,18 @@ async fn main() -> Result<()> {
             let proxy = ScreenReader0Proxy::new(&conn).await?;
             proxy.set_volume(*volume).await?;
         }
+        Commands::GetScreenReaderMode => {
+            let proxy = ScreenReader0Proxy::new(&conn).await?;
+            let mode = proxy.mode().await?;
+            match ScreenReaderMode::try_from(mode) {
+                Ok(s) => println!("Screen Reader Mode: {s}"),
+                Err(_) => println!("Got unknown screen reader mode value {mode} from backend"),
+            }
+        }
+        Commands::SetScreenReaderMode { mode } => {
+            let proxy = ScreenReader0Proxy::new(&conn).await?;
+            proxy.set_mode(*mode as u32).await?;
+        }
     }
 
     Ok(())
diff --git a/src/lib.rs b/src/lib.rs
index d82f82c..82b1e94 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -25,7 +25,6 @@ mod job;
 mod manager;
 mod platform;
 mod process;
-mod screenreader;
 mod sls;
 mod systemd;
 mod udev;
@@ -35,6 +34,7 @@ pub mod daemon;
 pub mod hardware;
 pub mod power;
 pub mod proxy;
+pub mod screenreader;
 pub mod wifi;
 
 #[cfg(test)]
diff --git a/src/manager/user.rs b/src/manager/user.rs
index f4d5654..13ef6e1 100644
--- a/src/manager/user.rs
+++ b/src/manager/user.rs
@@ -31,7 +31,7 @@ use crate::power::{
     get_gpu_clocks, get_gpu_clocks_range, get_gpu_performance_level, get_gpu_power_profile,
     get_max_charge_level, get_platform_profile, TdpManagerCommand,
 };
-use crate::screenreader::OrcaManager;
+use crate::screenreader::{OrcaManager, ScreenReaderMode};
 use crate::wifi::{
     get_wifi_backend, get_wifi_power_management_state, list_wifi_interfaces, WifiBackend,
 };
@@ -685,6 +685,28 @@ impl ScreenReader0 {
             .await
             .map_err(to_zbus_fdo_error)
     }
+
+    #[zbus(property)]
+    async fn mode(&self) -> u32 {
+        self.screen_reader.mode() as u32
+    }
+
+    #[zbus(property)]
+    async fn set_mode(
+        &mut self,
+        m: u32,
+        #[zbus(signal_emitter)] ctx: SignalEmitter<'_>,
+    ) -> fdo::Result<()> {
+        let mode = match ScreenReaderMode::try_from(m) {
+            Ok(mode) => mode,
+            Err(err) => return Err(fdo::Error::InvalidArgs(err.to_string())),
+        };
+        self.screen_reader
+            .set_mode(mode)
+            .await
+            .map_err(to_zbus_fdo_error)?;
+        self.mode_changed(&ctx).await.map_err(to_zbus_fdo_error)
+    }
 }
 
 #[interface(name = "com.steampowered.SteamOSManager1.Storage1")]
diff --git a/src/proxy/screenreader0.rs b/src/proxy/screenreader0.rs
index 6edc553..22990bc 100644
--- a/src/proxy/screenreader0.rs
+++ b/src/proxy/screenreader0.rs
@@ -42,4 +42,10 @@ pub trait ScreenReader0 {
     fn volume(&self) -> zbus::Result<f64>;
     #[zbus(property)]
     fn set_volume(&self, value: f64) -> zbus::Result<()>;
+
+    /// Mode property
+    #[zbus(property)]
+    fn mode(&self) -> zbus::Result<u32>;
+    #[zbus(property)]
+    fn set_mode(&self, mode: u32) -> zbus::Result<()>;
 }
diff --git a/src/screenreader.rs b/src/screenreader.rs
index afb3666..cbcbcc5 100644
--- a/src/screenreader.rs
+++ b/src/screenreader.rs
@@ -7,12 +7,28 @@
 
 use anyhow::{anyhow, bail, ensure, Result};
 use gio::{prelude::SettingsExt, Settings};
+#[cfg(test)]
+use input_linux::InputEvent;
+#[cfg(not(test))]
+use input_linux::{EventKind, InputId, UInputHandle};
+use input_linux::{EventTime, Key, KeyEvent, KeyState, SynchronizeEvent};
 use lazy_static::lazy_static;
+#[cfg(not(test))]
+use nix::fcntl::{fcntl, FcntlArg, OFlag};
+use num_enum::TryFromPrimitive;
 use serde_json::{Map, Value};
 use std::collections::HashMap;
+#[cfg(test)]
+use std::collections::VecDeque;
+#[cfg(not(test))]
+use std::fs::OpenOptions;
 use std::io::ErrorKind;
 use std::ops::RangeInclusive;
+#[cfg(not(test))]
+use std::os::fd::OwnedFd;
 use std::path::PathBuf;
+use std::time::SystemTime;
+use strum::{Display, EnumString};
 use tokio::fs::{read_to_string, write};
 use tracing::{debug, error, info, trace, warn};
 #[cfg(not(test))]
@@ -37,6 +53,7 @@ const ENABLE_SETTING: &str = "enableSpeech";
 
 const A11Y_SETTING: &str = "org.gnome.desktop.a11y.applications";
 const SCREEN_READER_SETTING: &str = "screen-reader-enabled";
+const KEYBOARD_NAME: &str = "steamos-manager";
 
 const PITCH_DEFAULT: f64 = 5.0;
 const RATE_DEFAULT: f64 = 50.0;
@@ -50,12 +67,125 @@ lazy_static! {
     ]);
 }
 
+#[derive(Display, EnumString, PartialEq, Debug, Copy, Clone, TryFromPrimitive)]
+#[strum(serialize_all = "snake_case", ascii_case_insensitive)]
+#[repr(u32)]
+pub enum ScreenReaderMode {
+    Browse = 0,
+    Focus = 1,
+}
+
+pub(crate) struct UInputDevice {
+    #[cfg(not(test))]
+    handle: UInputHandle<OwnedFd>,
+    #[cfg(test)]
+    queue: VecDeque<InputEvent>,
+    name: String,
+    open: bool,
+}
+
+impl UInputDevice {
+    #[cfg(not(test))]
+    pub(crate) fn new() -> Result<UInputDevice> {
+        let fd = OpenOptions::new()
+            .write(true)
+            .create(false)
+            .open("/dev/uinput")?
+            .into();
+
+        let mut flags = OFlag::from_bits_retain(fcntl(&fd, FcntlArg::F_GETFL)?);
+        flags.set(OFlag::O_NONBLOCK, true);
+        fcntl(&fd, FcntlArg::F_SETFL(flags))?;
+
+        Ok(UInputDevice {
+            handle: UInputHandle::new(fd),
+            name: String::new(),
+            open: false,
+        })
+    }
+
+    #[cfg(test)]
+    pub(crate) fn new() -> Result<UInputDevice> {
+        Ok(UInputDevice {
+            queue: VecDeque::new(),
+            name: String::new(),
+            open: false,
+        })
+    }
+
+    pub(crate) fn set_name(&mut self, name: String) -> Result<()> {
+        ensure!(!self.open, "Cannot change name after opening");
+        self.name = name;
+        Ok(())
+    }
+
+    #[cfg(not(test))]
+    pub(crate) fn open(&mut self) -> Result<()> {
+        ensure!(!self.open, "Cannot reopen uinput handle");
+
+        self.handle.set_evbit(EventKind::Key)?;
+        self.handle.set_keybit(Key::Insert)?;
+        self.handle.set_keybit(Key::A)?;
+
+        let input_id = InputId {
+            bustype: input_linux::sys::BUS_VIRTUAL,
+            vendor: 0x28DE,
+            product: 0,
+            version: 0,
+        };
+        self.handle
+            .create(&input_id, self.name.as_bytes(), 0, &[])?;
+        self.open = true;
+        Ok(())
+    }
+
+    #[cfg(test)]
+    pub(crate) fn open(&mut self) -> Result<()> {
+        ensure!(!self.open, "Cannot reopen uinput handle");
+        self.open = true;
+        Ok(())
+    }
+
+    fn system_time() -> Result<EventTime> {
+        let duration = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?;
+        Ok(EventTime::new(
+            duration.as_secs().try_into()?,
+            duration.subsec_micros().into(),
+        ))
+    }
+
+    fn send_key_event(&mut self, key: Key, value: KeyState) -> Result<()> {
+        let tv = UInputDevice::system_time().unwrap_or_else(|err| {
+            warn!("System time error: {err}");
+            EventTime::default()
+        });
+
+        let ev = KeyEvent::new(tv, key, value);
+        let syn = SynchronizeEvent::report(tv);
+        #[cfg(not(test))]
+        self.handle.write(&[*ev.as_ref(), *syn.as_ref()])?;
+        #[cfg(test)]
+        self.queue.extend(&[*ev.as_ref(), *syn.as_ref()]);
+        Ok(())
+    }
+
+    pub(crate) fn key_down(&mut self, key: Key) -> Result<()> {
+        self.send_key_event(key, KeyState::PRESSED)
+    }
+
+    pub(crate) fn key_up(&mut self, key: Key) -> Result<()> {
+        self.send_key_event(key, KeyState::RELEASED)
+    }
+}
+
 pub(crate) struct OrcaManager<'dbus> {
     orca_unit: SystemdUnit<'dbus>,
     rate: f64,
     pitch: f64,
     volume: f64,
     enabled: bool,
+    mode: ScreenReaderMode,
+    keyboard: UInputDevice,
 }
 
 impl<'dbus> OrcaManager<'dbus> {
@@ -66,6 +196,9 @@ impl<'dbus> OrcaManager<'dbus> {
             pitch: PITCH_DEFAULT,
             volume: VOLUME_DEFAULT,
             enabled: true,
+            // Always start in browse mode for now, since we have no storage to remember this property
+            mode: ScreenReaderMode::Browse,
+            keyboard: UInputDevice::new()?,
         };
         let _ = manager
             .load_values()
@@ -73,6 +206,9 @@ impl<'dbus> OrcaManager<'dbus> {
             .inspect_err(|e| warn!("Failed to load orca configuration: {e}"));
         let a11ysettings = Settings::new(A11Y_SETTING);
         manager.enabled = a11ysettings.boolean(SCREEN_READER_SETTING);
+        manager.keyboard.set_name(KEYBOARD_NAME.to_string())?;
+        manager.keyboard.open()?;
+
         Ok(manager)
     }
 
@@ -157,6 +293,38 @@ impl<'dbus> OrcaManager<'dbus> {
         Ok(())
     }
 
+    pub fn mode(&self) -> ScreenReaderMode {
+        self.mode
+    }
+
+    pub async fn set_mode(&mut self, mode: ScreenReaderMode) -> Result<()> {
+        // Use insert+A twice to switch to focus mode sticky
+        // Use insert+A three times to switch to browse mode sticky
+        match mode {
+            ScreenReaderMode::Focus => {
+                self.keyboard.key_down(Key::Insert)?;
+                self.keyboard.key_down(Key::A)?;
+                self.keyboard.key_up(Key::A)?;
+                self.keyboard.key_down(Key::A)?;
+                self.keyboard.key_up(Key::A)?;
+                self.keyboard.key_up(Key::Insert)?;
+            }
+            ScreenReaderMode::Browse => {
+                self.keyboard.key_down(Key::Insert)?;
+                self.keyboard.key_down(Key::A)?;
+                self.keyboard.key_up(Key::A)?;
+                self.keyboard.key_down(Key::A)?;
+                self.keyboard.key_up(Key::A)?;
+                self.keyboard.key_down(Key::A)?;
+                self.keyboard.key_up(Key::A)?;
+                self.keyboard.key_up(Key::Insert)?;
+            }
+        }
+        self.mode = mode;
+
+        Ok(())
+    }
+
     async fn set_orca_enabled(&mut self, enabled: bool) -> Result<()> {
         // Change json file
         let data = read_to_string(self.settings_path()?).await?;