mirror of
https://gitlab.steamos.cloud/holo/steamos-manager.git
synced 2025-07-15 10:46:41 -04:00
Merge branch 'work/whiting/screenreadermethods' into 'master'
screenreader: Add TriggerAction functionality. See merge request holo/steamos-manager!12
This commit is contained in:
Jeremy Whiting
commit
da2585f2d7
7 changed files with 333 additions and 4 deletions
154
Cargo.lock
generated
154
Cargo.lock
generated
|
|
@ -706,6 +706,15 @@ dependencies = [
|
|||
"memoffset",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.7.3"
|
||||
|
|
@ -727,6 +736,25 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-foundation"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-kit"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.36.7"
|
||||
|
|
@ -1042,6 +1070,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
|
|
@ -1088,6 +1117,20 @@ dependencies = [
|
|||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.35.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
"ntapi",
|
||||
"objc2-core-foundation",
|
||||
"objc2-io-kit",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-deps"
|
||||
version = "7.0.5"
|
||||
|
|
@ -1357,6 +1400,108 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.61.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-core",
|
||||
"windows-future",
|
||||
"windows-link",
|
||||
"windows-numerics",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-link",
|
||||
"windows-threading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
|
|
@ -1415,6 +1560,15 @@ dependencies = [
|
|||
"windows_x86_64_msvc 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.5"
|
||||
|
|
|
|||
|
|
@ -18,12 +18,13 @@ input-linux = "0.7"
|
|||
itertools = "0.14"
|
||||
lazy_static = "1"
|
||||
libc = "0.2"
|
||||
nix = { version = "0.30", default-features = false, features = ["fs", "poll", "signal"] }
|
||||
nix = { version = "0.30", default-features = false, features = ["fs", "poll", "signal", "time"] }
|
||||
num_enum = "0.7"
|
||||
regex = "1"
|
||||
serde = { version = "1.0", default-features = false, features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
strum = { version = "0.27", features = ["derive"] }
|
||||
sysinfo = "0.35"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1", default-features = false, features = ["fs", "io-std", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync"] }
|
||||
tokio-stream = { version = "0.1", default-features = false }
|
||||
|
|
|
|||
|
|
@ -342,6 +342,33 @@
|
|||
Valid modes: 0 - Browse mode, 1 - Focus mode.
|
||||
-->
|
||||
<property name="Mode" type="u" access="readwrite"/>
|
||||
|
||||
<!--
|
||||
Trigger Action
|
||||
|
||||
Forward a user interaction event to the screen reader service.
|
||||
|
||||
@action: The action genereated via user input. Valid values:
|
||||
0 = Stop Talking
|
||||
1 = Read Next Word,
|
||||
2 = Read Previous Word,
|
||||
3 = Read Next Item,
|
||||
4 = Read Previous Item,
|
||||
5 = Move To Next Landmark,
|
||||
6 = Move To Previous Landmark,
|
||||
7 = Move To Next Heading,
|
||||
8 = Move To Previous Heading,
|
||||
9 = Toggle between Browse mode and Focus mode,
|
||||
|
||||
@timestamp: refers to the time the user interaction event
|
||||
that triggered this event happened. E.g. when the user pressed
|
||||
a key to request this action. (Encoded as nanoseconds since the
|
||||
start of CLOCK_MONOTONIC_RAW).
|
||||
-->
|
||||
<method name="TriggerAction">
|
||||
<arg type="u" name="action" direction="in"/>
|
||||
<arg type="t" name="timestamp" direction="in"/>
|
||||
</method>
|
||||
</interface>
|
||||
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
use anyhow::Result;
|
||||
use clap::{ArgAction, Parser, Subcommand};
|
||||
use itertools::Itertools;
|
||||
use nix::time::{clock_gettime, ClockId};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use steamos_manager::cec::HdmiCecState;
|
||||
|
|
@ -20,7 +21,7 @@ use steamos_manager::proxy::{
|
|||
TdpLimit1Proxy, UpdateBios1Proxy, UpdateDock1Proxy, WifiDebug1Proxy, WifiDebugDump1Proxy,
|
||||
WifiPowerManagement1Proxy,
|
||||
};
|
||||
use steamos_manager::screenreader::ScreenReaderMode;
|
||||
use steamos_manager::screenreader::{ScreenReaderAction, ScreenReaderMode};
|
||||
use steamos_manager::wifi::{WifiBackend, WifiDebugMode, WifiPowerManagement};
|
||||
use zbus::fdo::{IntrospectableProxy, PropertiesProxy};
|
||||
use zbus::{zvariant, Connection};
|
||||
|
|
@ -252,6 +253,23 @@ enum Commands {
|
|||
/// Valid modes are `browse`, `focus`
|
||||
mode: ScreenReaderMode,
|
||||
},
|
||||
|
||||
/// Trigger screen reader action
|
||||
TriggerScreenReaderAction {
|
||||
/// Valid actions are:
|
||||
/// stop_talking
|
||||
/// read_next_word
|
||||
/// read_previous_word
|
||||
/// read_next_item
|
||||
/// read_previous_item
|
||||
/// move_to_next_landmark,
|
||||
/// move_to_previous_landmark,
|
||||
/// move_to_next_heading,
|
||||
/// move_to_previous_heading,
|
||||
/// toggle_mode,
|
||||
///
|
||||
action: ScreenReaderAction,
|
||||
},
|
||||
}
|
||||
|
||||
async fn get_all_properties(conn: &Connection) -> Result<()> {
|
||||
|
|
@ -613,6 +631,14 @@ async fn main() -> Result<()> {
|
|||
let proxy = ScreenReader0Proxy::new(&conn).await?;
|
||||
proxy.set_mode(*mode as u32).await?;
|
||||
}
|
||||
Commands::TriggerScreenReaderAction { action } => {
|
||||
let proxy = ScreenReader0Proxy::new(&conn).await?;
|
||||
let timestamp = clock_gettime(ClockId::CLOCK_MONOTONIC_RAW)?;
|
||||
let now = timestamp.tv_sec() * 1000000000 + timestamp.tv_nsec();
|
||||
proxy
|
||||
.trigger_action(*action as u32, now.try_into()?)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ use crate::power::{
|
|||
get_gpu_clocks, get_gpu_clocks_range, get_gpu_performance_level, get_gpu_power_profile,
|
||||
get_max_charge_level, get_platform_profile, TdpManagerCommand,
|
||||
};
|
||||
use crate::screenreader::{OrcaManager, ScreenReaderMode};
|
||||
use crate::screenreader::{OrcaManager, ScreenReaderAction, ScreenReaderMode};
|
||||
use crate::wifi::{
|
||||
get_wifi_backend, get_wifi_power_management_state, list_wifi_interfaces, WifiBackend,
|
||||
};
|
||||
|
|
@ -707,6 +707,17 @@ impl ScreenReader0 {
|
|||
.map_err(to_zbus_fdo_error)?;
|
||||
self.mode_changed(&ctx).await.map_err(to_zbus_fdo_error)
|
||||
}
|
||||
|
||||
async fn trigger_action(&mut self, a: u32, timestamp: u64) -> fdo::Result<()> {
|
||||
let action = match ScreenReaderAction::try_from(a) {
|
||||
Ok(action) => action,
|
||||
Err(err) => return Err(fdo::Error::InvalidArgs(err.to_string())),
|
||||
};
|
||||
self.screen_reader
|
||||
.trigger_action(action, timestamp)
|
||||
.await
|
||||
.map_err(to_zbus_fdo_error)
|
||||
}
|
||||
}
|
||||
|
||||
#[interface(name = "com.steampowered.SteamOSManager1.Storage1")]
|
||||
|
|
|
|||
|
|
@ -48,4 +48,6 @@ pub trait ScreenReader0 {
|
|||
fn mode(&self) -> zbus::Result<u32>;
|
||||
#[zbus(property)]
|
||||
fn set_mode(&self, mode: u32) -> zbus::Result<()>;
|
||||
|
||||
fn trigger_action(&self, action: u32, timestamp: u64) -> zbus::Result<()>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
use ::sysinfo::System;
|
||||
use anyhow::{anyhow, bail, ensure, Result};
|
||||
use gio::{prelude::SettingsExt, Settings};
|
||||
#[cfg(test)]
|
||||
|
|
@ -15,6 +16,8 @@ use input_linux::{EventTime, Key, KeyEvent, KeyState, SynchronizeEvent};
|
|||
use lazy_static::lazy_static;
|
||||
#[cfg(not(test))]
|
||||
use nix::fcntl::{fcntl, FcntlArg, OFlag};
|
||||
use nix::sys::signal;
|
||||
use nix::unistd::Pid;
|
||||
use num_enum::TryFromPrimitive;
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::HashMap;
|
||||
|
|
@ -75,6 +78,22 @@ pub enum ScreenReaderMode {
|
|||
Focus = 1,
|
||||
}
|
||||
|
||||
#[derive(Display, EnumString, PartialEq, Debug, Copy, Clone, TryFromPrimitive)]
|
||||
#[strum(serialize_all = "snake_case", ascii_case_insensitive)]
|
||||
#[repr(u32)]
|
||||
pub enum ScreenReaderAction {
|
||||
StopSpeaking = 0,
|
||||
ReadNextWord = 1,
|
||||
ReadPreviousWord = 2,
|
||||
ReadNextItem = 3,
|
||||
ReadPreviousItem = 4,
|
||||
MoveToNextLandmark = 5,
|
||||
MoveToPreviousLandmark = 6,
|
||||
MoveToNextHeading = 7,
|
||||
MoveToPreviousHeading = 8,
|
||||
ToggleMode = 9,
|
||||
}
|
||||
|
||||
pub(crate) struct UInputDevice {
|
||||
#[cfg(not(test))]
|
||||
handle: UInputHandle<OwnedFd>,
|
||||
|
|
@ -124,8 +143,16 @@ impl UInputDevice {
|
|||
ensure!(!self.open, "Cannot reopen uinput handle");
|
||||
|
||||
self.handle.set_evbit(EventKind::Key)?;
|
||||
self.handle.set_keybit(Key::Insert)?;
|
||||
self.handle.set_keybit(Key::A)?;
|
||||
self.handle.set_keybit(Key::H)?;
|
||||
self.handle.set_keybit(Key::M)?;
|
||||
self.handle.set_keybit(Key::Insert)?;
|
||||
self.handle.set_keybit(Key::LeftCtrl)?;
|
||||
self.handle.set_keybit(Key::LeftShift)?;
|
||||
self.handle.set_keybit(Key::Down)?;
|
||||
self.handle.set_keybit(Key::Left)?;
|
||||
self.handle.set_keybit(Key::Right)?;
|
||||
self.handle.set_keybit(Key::Up)?;
|
||||
|
||||
let input_id = InputId {
|
||||
bustype: input_linux::sys::BUS_VIRTUAL,
|
||||
|
|
@ -325,6 +352,87 @@ impl<'dbus> OrcaManager<'dbus> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn trigger_action(
|
||||
&mut self,
|
||||
action: ScreenReaderAction,
|
||||
_timestamp: u64,
|
||||
) -> Result<()> {
|
||||
// TODO: Maybe filter events if the timestamp is too old?
|
||||
match action {
|
||||
ScreenReaderAction::StopSpeaking => {
|
||||
// TODO: Use dbus method to stop orca from speaking instead once that's in a release/steamos package.
|
||||
let pid = self.get_orca_pid()?;
|
||||
signal::kill(pid, signal::Signal::SIGUSR2)?;
|
||||
}
|
||||
ScreenReaderAction::ReadNextWord => {
|
||||
self.keyboard.key_down(Key::LeftCtrl)?;
|
||||
self.keyboard.key_down(Key::Right)?;
|
||||
self.keyboard.key_up(Key::Right)?;
|
||||
self.keyboard.key_up(Key::LeftCtrl)?;
|
||||
}
|
||||
ScreenReaderAction::ReadPreviousWord => {
|
||||
self.keyboard.key_down(Key::LeftCtrl)?;
|
||||
self.keyboard.key_down(Key::Left)?;
|
||||
self.keyboard.key_up(Key::Left)?;
|
||||
self.keyboard.key_up(Key::LeftCtrl)?;
|
||||
}
|
||||
ScreenReaderAction::ReadNextItem => {
|
||||
self.keyboard.key_down(Key::Down)?;
|
||||
self.keyboard.key_up(Key::Down)?;
|
||||
}
|
||||
ScreenReaderAction::ReadPreviousItem => {
|
||||
self.keyboard.key_down(Key::Up)?;
|
||||
self.keyboard.key_up(Key::Up)?;
|
||||
}
|
||||
ScreenReaderAction::MoveToNextLandmark => {
|
||||
self.keyboard.key_down(Key::M)?;
|
||||
self.keyboard.key_up(Key::M)?;
|
||||
}
|
||||
ScreenReaderAction::MoveToPreviousLandmark => {
|
||||
self.keyboard.key_down(Key::LeftShift)?;
|
||||
self.keyboard.key_down(Key::M)?;
|
||||
self.keyboard.key_up(Key::M)?;
|
||||
self.keyboard.key_up(Key::LeftShift)?;
|
||||
}
|
||||
ScreenReaderAction::MoveToNextHeading => {
|
||||
self.keyboard.key_down(Key::H)?;
|
||||
self.keyboard.key_up(Key::H)?;
|
||||
}
|
||||
ScreenReaderAction::MoveToPreviousHeading => {
|
||||
self.keyboard.key_down(Key::LeftShift)?;
|
||||
self.keyboard.key_down(Key::H)?;
|
||||
self.keyboard.key_up(Key::H)?;
|
||||
self.keyboard.key_up(Key::LeftShift)?;
|
||||
}
|
||||
ScreenReaderAction::ToggleMode => {
|
||||
self.keyboard.key_down(Key::Insert)?;
|
||||
self.keyboard.key_down(Key::A)?;
|
||||
self.keyboard.key_up(Key::A)?;
|
||||
self.keyboard.key_up(Key::Insert)?;
|
||||
// TODO: I guess we should emit that the mode changed here...
|
||||
match self.mode {
|
||||
ScreenReaderMode::Browse => {
|
||||
self.mode = ScreenReaderMode::Focus;
|
||||
}
|
||||
ScreenReaderMode::Focus => {
|
||||
self.mode = ScreenReaderMode::Browse;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_orca_pid(&self) -> Result<Pid> {
|
||||
let mut system = System::new();
|
||||
system.refresh_all();
|
||||
|
||||
let mut p = system.processes_by_name("orca".as_ref());
|
||||
|
||||
let pid = p.next().expect("No orca process found");
|
||||
Ok(Pid::from_raw(pid.pid().as_u32().try_into()?))
|
||||
}
|
||||
|
||||
async fn set_orca_enabled(&mut self, enabled: bool) -> Result<()> {
|
||||
// Change json file
|
||||
let data = read_to_string(self.settings_path()?).await?;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue