From 42e89ef3423fcd531817ed285069a29f32bf7ede Mon Sep 17 00:00:00 2001
From: Jeremy Whiting <jpwhiting@kde.org>
Date: Fri, 13 Jun 2025 16:57:08 -0600
Subject: [PATCH] Draft: screenreader: Add TriggerAction functionality.

Add functionality for various actions by sending left control key
press and release events.
Also add to steamosctl trigger_action support.
NOTE: Stop Speaking action should be changed to use orca's new dbus
api once that's packaged.
---
 Cargo.toml                           |  2 +-
 com.steampowered.SteamOSManager1.xml | 27 ++++++++
 src/bin/steamosctl.rs                | 27 +++++++-
 src/manager/user.rs                  | 13 +++-
 src/proxy/screenreader0.rs           |  2 +
 src/screenreader.rs                  | 97 +++++++++++++++++++++++++++-
 6 files changed, 164 insertions(+), 4 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 3599834..956a1ee 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,7 @@ input-linux = "0.7"
 itertools = "0.14"
 lazy_static = "1"
 libc = "0.2"
-nix = { version = "0.30", default-features = false, features = ["fs", "poll", "signal"] }
+nix = { version = "0.30", default-features = false, features = ["fs", "poll", "signal", "time"] }
 num_enum = "0.7"
 regex = "1"
 serde = { version = "1.0", default-features = false, features = ["derive"] }
diff --git a/com.steampowered.SteamOSManager1.xml b/com.steampowered.SteamOSManager1.xml
index f3fcd04..f54ca57 100644
--- a/com.steampowered.SteamOSManager1.xml
+++ b/com.steampowered.SteamOSManager1.xml
@@ -342,6 +342,33 @@
         Valid modes: 0 - Browse mode, 1 - Focus mode.
     -->
     <property name="Mode" type="u" access="readwrite"/>
+     
+    <!--
+        Trigger Action
+
+        Forward a user interaction event to the screen reader service.
+
+        @action: The action genereated via user input. Valid values:
+          0 = Stop Talking
+          1 = Read Next Word,
+          2 = Read Previous Word,
+          3 = Read Next Item,
+          4 = Read Previous Item,
+          5 = Move To Next Landmark,
+          6 = Move To Previous Landmark,
+          7 = Move To Next Heading,
+          8 = Move To Previous Heading,
+          9 = Toggle between Browse mode and Focus mode,
+
+        @timestamp: refers to the time the user interaction event
+        that triggered this event happened. E.g. when the user pressed
+        a key to request this action. (Encoded as nanoseconds since the
+        start of CLOCK_MONOTONIC_RAW).
+    -->
+    <method name="TriggerAction">
+      <arg type="u" name="action" direction="in"/>
+      <arg type="t" name="timestamp" direction="in"/>
+    </method>
   </interface>
 
   <!--
diff --git a/src/bin/steamosctl.rs b/src/bin/steamosctl.rs
index cb70bd1..c1bc142 100644
--- a/src/bin/steamosctl.rs
+++ b/src/bin/steamosctl.rs
@@ -8,6 +8,7 @@
 use anyhow::Result;
 use clap::{ArgAction, Parser, Subcommand};
 use itertools::Itertools;
+use nix::time::ClockId;
 use std::collections::HashMap;
 use std::io::Cursor;
 use steamos_manager::cec::HdmiCecState;
@@ -20,7 +21,7 @@ use steamos_manager::proxy::{
     TdpLimit1Proxy, UpdateBios1Proxy, UpdateDock1Proxy, WifiDebug1Proxy, WifiDebugDump1Proxy,
     WifiPowerManagement1Proxy,
 };
-use steamos_manager::screenreader::ScreenReaderMode;
+use steamos_manager::screenreader::{ScreenReaderAction, ScreenReaderMode};
 use steamos_manager::wifi::{WifiBackend, WifiDebugMode, WifiPowerManagement};
 use zbus::fdo::{IntrospectableProxy, PropertiesProxy};
 use zbus::{zvariant, Connection};
@@ -252,6 +253,23 @@ enum Commands {
         /// Valid modes are `browse`, `focus`
         mode: ScreenReaderMode,
     },
+
+    /// Trigger screen reader action
+    TriggerScreenReaderAction {
+        /// Valid actions are:
+        ///  stop_talking
+        ///  read_next_word
+        ///  read_previous_word
+        ///  read_next_item
+        ///  read_previous_item
+        ///  move_to_next_landmark,
+        ///  move_to_previous_landmark,
+        ///  move_to_next_heading,
+        ///  move_to_previous_heading,
+        ///  toggle_mode,
+        ///
+        action: ScreenReaderAction,
+    },
 }
 
 async fn get_all_properties(conn: &Connection) -> Result<()> {
@@ -613,6 +631,13 @@ async fn main() -> Result<()> {
             let proxy = ScreenReader0Proxy::new(&conn).await?;
             proxy.set_mode(*mode as u32).await?;
         }
+        Commands::TriggerScreenReaderAction { action } => {
+            let proxy = ScreenReader0Proxy::new(&conn).await?;
+            let timestamp = nix::time::clock_gettime(ClockId::CLOCK_MONOTONIC_RAW)?;
+            proxy
+                .trigger_action(*action as u32, timestamp.tv_nsec().try_into()?)
+                .await?;
+        }
     }
 
     Ok(())
diff --git a/src/manager/user.rs b/src/manager/user.rs
index 8572e52..cd47cd2 100644
--- a/src/manager/user.rs
+++ b/src/manager/user.rs
@@ -31,7 +31,7 @@ use crate::power::{
     get_gpu_clocks, get_gpu_clocks_range, get_gpu_performance_level, get_gpu_power_profile,
     get_max_charge_level, get_platform_profile, TdpManagerCommand,
 };
-use crate::screenreader::{OrcaManager, ScreenReaderMode};
+use crate::screenreader::{OrcaManager, ScreenReaderAction, ScreenReaderMode};
 use crate::wifi::{
     get_wifi_backend, get_wifi_power_management_state, list_wifi_interfaces, WifiBackend,
 };
@@ -707,6 +707,17 @@ impl ScreenReader0 {
             .map_err(to_zbus_fdo_error)?;
         self.mode_changed(&ctx).await.map_err(to_zbus_fdo_error)
     }
+
+    async fn trigger_action(&mut self, a: u32, timestamp: u64) -> fdo::Result<()> {
+        let action = match ScreenReaderAction::try_from(a) {
+            Ok(action) => action,
+            Err(err) => return Err(fdo::Error::InvalidArgs(err.to_string())),
+        };
+        self.screen_reader
+            .trigger_action(action, timestamp)
+            .await
+            .map_err(to_zbus_fdo_error)
+    }
 }
 
 #[interface(name = "com.steampowered.SteamOSManager1.Storage1")]
diff --git a/src/proxy/screenreader0.rs b/src/proxy/screenreader0.rs
index 22990bc..cffbefe 100644
--- a/src/proxy/screenreader0.rs
+++ b/src/proxy/screenreader0.rs
@@ -48,4 +48,6 @@ pub trait ScreenReader0 {
     fn mode(&self) -> zbus::Result<u32>;
     #[zbus(property)]
     fn set_mode(&self, mode: u32) -> zbus::Result<()>;
+
+    fn trigger_action(&self, action: u32, timestamp: u64) -> zbus::Result<()>;
 }
diff --git a/src/screenreader.rs b/src/screenreader.rs
index cbcbcc5..547dad3 100644
--- a/src/screenreader.rs
+++ b/src/screenreader.rs
@@ -75,6 +75,22 @@ pub enum ScreenReaderMode {
     Focus = 1,
 }
 
+#[derive(Display, EnumString, PartialEq, Debug, Copy, Clone, TryFromPrimitive)]
+#[strum(serialize_all = "snake_case", ascii_case_insensitive)]
+#[repr(u32)]
+pub enum ScreenReaderAction {
+    StopSpeaking = 0,
+    ReadNextWord = 1,
+    ReadPreviousWord = 2,
+    ReadNextItem = 3,
+    ReadPreviousItem = 4,
+    MoveToNextLandmark = 5,
+    MoveToPreviousLandmark = 6,
+    MoveToNextHeading = 7,
+    MoveToPreviousHeading = 8,
+    ToggleMode = 9,
+}
+
 pub(crate) struct UInputDevice {
     #[cfg(not(test))]
     handle: UInputHandle<OwnedFd>,
@@ -124,8 +140,16 @@ impl UInputDevice {
         ensure!(!self.open, "Cannot reopen uinput handle");
 
         self.handle.set_evbit(EventKind::Key)?;
-        self.handle.set_keybit(Key::Insert)?;
         self.handle.set_keybit(Key::A)?;
+        self.handle.set_keybit(Key::H)?;
+        self.handle.set_keybit(Key::M)?;
+        self.handle.set_keybit(Key::Insert)?;
+        self.handle.set_keybit(Key::LeftCtrl)?;
+        self.handle.set_keybit(Key::LeftShift)?;
+        self.handle.set_keybit(Key::Down)?;
+        self.handle.set_keybit(Key::Left)?;
+        self.handle.set_keybit(Key::Right)?;
+        self.handle.set_keybit(Key::Up)?;
 
         let input_id = InputId {
             bustype: input_linux::sys::BUS_VIRTUAL,
@@ -325,6 +349,77 @@ impl<'dbus> OrcaManager<'dbus> {
         Ok(())
     }
 
+    pub async fn trigger_action(
+        &mut self,
+        action: ScreenReaderAction,
+        _timestamp: u64,
+    ) -> Result<()> {
+        // TODO: Maybe filter events if the timestamp is too old?
+        match action {
+            ScreenReaderAction::StopSpeaking => {
+                // TODO: Use dbus method to stop orca from speaking instead once that's in a release/steamos package.
+                self.keyboard.key_down(Key::LeftCtrl)?;
+                self.keyboard.key_up(Key::LeftCtrl)?;
+            }
+            ScreenReaderAction::ReadNextWord => {
+                self.keyboard.key_down(Key::LeftCtrl)?;
+                self.keyboard.key_down(Key::Right)?;
+                self.keyboard.key_up(Key::Right)?;
+                self.keyboard.key_up(Key::LeftCtrl)?;
+            }
+            ScreenReaderAction::ReadPreviousWord => {
+                self.keyboard.key_down(Key::LeftCtrl)?;
+                self.keyboard.key_down(Key::Left)?;
+                self.keyboard.key_up(Key::Left)?;
+                self.keyboard.key_up(Key::LeftCtrl)?;
+            }
+            ScreenReaderAction::ReadNextItem => {
+                self.keyboard.key_down(Key::Down)?;
+                self.keyboard.key_up(Key::Down)?;
+            }
+            ScreenReaderAction::ReadPreviousItem => {
+                self.keyboard.key_down(Key::Up)?;
+                self.keyboard.key_up(Key::Up)?;
+            }
+            ScreenReaderAction::MoveToNextLandmark => {
+                self.keyboard.key_down(Key::M)?;
+                self.keyboard.key_up(Key::M)?;
+            }
+            ScreenReaderAction::MoveToPreviousLandmark => {
+                self.keyboard.key_down(Key::LeftShift)?;
+                self.keyboard.key_down(Key::M)?;
+                self.keyboard.key_up(Key::M)?;
+                self.keyboard.key_up(Key::LeftShift)?;
+            }
+            ScreenReaderAction::MoveToNextHeading => {
+                self.keyboard.key_down(Key::H)?;
+                self.keyboard.key_up(Key::H)?;
+            }
+            ScreenReaderAction::MoveToPreviousHeading => {
+                self.keyboard.key_down(Key::LeftShift)?;
+                self.keyboard.key_down(Key::H)?;
+                self.keyboard.key_up(Key::H)?;
+                self.keyboard.key_up(Key::LeftShift)?;
+            }
+            ScreenReaderAction::ToggleMode => {
+                self.keyboard.key_down(Key::Insert)?;
+                self.keyboard.key_down(Key::A)?;
+                self.keyboard.key_up(Key::A)?;
+                self.keyboard.key_up(Key::Insert)?;
+                // TODO: I guess we should emit that the mode changed here...
+                match self.mode {
+                    ScreenReaderMode::Browse => {
+                        self.mode = ScreenReaderMode::Focus;
+                    }
+                    ScreenReaderMode::Focus => {
+                        self.mode = ScreenReaderMode::Browse;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
     async fn set_orca_enabled(&mut self, enabled: bool) -> Result<()> {
         // Change json file
         let data = read_to_string(self.settings_path()?).await?;