azcomputerguru/guru-connect
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Initial GuruConnect implementation - Phase 1 MVP

Browse Source 
- Agent: DXGI/GDI screen capture, mouse/keyboard input, WebSocket transport
- Server: Axum relay, session management, REST API
- Dashboard: React viewer components with TypeScript
- Protocol: Protobuf definitions for all message types

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
AZ Computer Guru
2025-12-21 17:18:05 -07:00
commit 33893ea73b
38 changed files with 7724 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
agent/src/input/keyboard.rs Normal file
View File

@@ -0,0 +1,287 @@
//! Keyboard input simulation using Windows SendInput API
use anyhow::Result;
#[cfg(windows)]
use windows::Win32::UI::Input::KeyboardAndMouse::{
SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY,
KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, KEYEVENTF_UNICODE, KEYBDINPUT,
MapVirtualKeyW, MAPVK_VK_TO_VSC_EX,
};
/// Keyboard input controller
pub struct KeyboardController {
// Track modifier states for proper handling
#[allow(dead_code)]
modifiers: ModifierState,
}
#[derive(Default)]
struct ModifierState {
ctrl: bool,
alt: bool,
shift: bool,
meta: bool,
}
impl KeyboardController {
/// Create a new keyboard controller
pub fn new() -> Result<Self> {
Ok(Self {
modifiers: ModifierState::default(),
})
}
/// Press a key down by virtual key code
#[cfg(windows)]
pub fn key_down(&mut self, vk_code: u16) -> Result<()> {
self.send_key(vk_code, true)
}
/// Release a key by virtual key code
#[cfg(windows)]
pub fn key_up(&mut self, vk_code: u16) -> Result<()> {
self.send_key(vk_code, false)
}
/// Send a key event
#[cfg(windows)]
fn send_key(&mut self, vk_code: u16, down: bool) -> Result<()> {
// Get scan code from virtual key
let scan_code = unsafe { MapVirtualKeyW(vk_code as u32, MAPVK_VK_TO_VSC_EX) as u16 };
let mut flags = KEYBD_EVENT_FLAGS::default();
// Add extended key flag for certain keys
if Self::is_extended_key(vk_code) || (scan_code >> 8) == 0xE0 {
flags |= KEYEVENTF_EXTENDEDKEY;
}
if !down {
flags |= KEYEVENTF_KEYUP;
}
let input = INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: windows::Win32::UI::Input::KeyboardAndMouse::VIRTUAL_KEY(vk_code),
wScan: scan_code,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
};
self.send_input(&[input])
}
/// Type a unicode character
#[cfg(windows)]
pub fn type_char(&mut self, ch: char) -> Result<()> {
let mut inputs = Vec::new();
// For characters that fit in a single u16
for code_unit in ch.encode_utf16(&mut [0; 2]) {
// Key down
inputs.push(INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: windows::Win32::UI::Input::KeyboardAndMouse::VIRTUAL_KEY(0),
wScan: code_unit,
dwFlags: KEYEVENTF_UNICODE,
time: 0,
dwExtraInfo: 0,
},
},
});
// Key up
inputs.push(INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: windows::Win32::UI::Input::KeyboardAndMouse::VIRTUAL_KEY(0),
wScan: code_unit,
dwFlags: KEYEVENTF_UNICODE | KEYEVENTF_KEYUP,
time: 0,
dwExtraInfo: 0,
},
},
});
}
self.send_input(&inputs)
}
/// Type a string of text
#[cfg(windows)]
pub fn type_string(&mut self, text: &str) -> Result<()> {
for ch in text.chars() {
self.type_char(ch)?;
}
Ok(())
}
/// Send Secure Attention Sequence (Ctrl+Alt+Delete)
///
/// Note: This requires special privileges on Windows.
/// The agent typically needs to run as SYSTEM or use SAS API.
#[cfg(windows)]
pub fn send_sas(&mut self) -> Result<()> {
// Try using the SAS library if available
// For now, we'll attempt to send the key combination
// This won't work in all contexts due to Windows security
// Load the sas.dll and call SendSAS if available
use windows::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryW};
use windows::core::PCWSTR;
unsafe {
let dll_name: Vec<u16> = "sas.dll\0".encode_utf16().collect();
let lib = LoadLibraryW(PCWSTR(dll_name.as_ptr()));
if let Ok(lib) = lib {
let proc_name = b"SendSAS\0";
if let Some(proc) = GetProcAddress(lib, windows::core::PCSTR(proc_name.as_ptr())) {
// SendSAS takes a BOOL parameter: FALSE for Ctrl+Alt+Del
let send_sas: extern "system" fn(i32) = std::mem::transmute(proc);
send_sas(0); // FALSE = Ctrl+Alt+Del
return Ok(());
}
}
}
// Fallback: Try sending the keys (won't work without proper privileges)
tracing::warn!("SAS library not available, Ctrl+Alt+Del may not work");
// VK codes
const VK_CONTROL: u16 = 0x11;
const VK_MENU: u16 = 0x12; // Alt
const VK_DELETE: u16 = 0x2E;
// Press keys
self.key_down(VK_CONTROL)?;
self.key_down(VK_MENU)?;
self.key_down(VK_DELETE)?;
// Release keys
self.key_up(VK_DELETE)?;
self.key_up(VK_MENU)?;
self.key_up(VK_CONTROL)?;
Ok(())
}
/// Check if a virtual key code is an extended key
#[cfg(windows)]
fn is_extended_key(vk: u16) -> bool {
matches!(
vk,
0x21..=0x28 | // Page Up, Page Down, End, Home, Arrow keys
0x2D | 0x2E | // Insert, Delete
0x5B | 0x5C | // Left/Right Windows keys
0x5D | // Applications key
0x6F | // Numpad Divide
0x90 | // Num Lock
0x91 // Scroll Lock
)
}
/// Send input events
#[cfg(windows)]
fn send_input(&self, inputs: &[INPUT]) -> Result<()> {
let sent = unsafe { SendInput(inputs, std::mem::size_of::<INPUT>() as i32) };
if sent as usize != inputs.len() {
anyhow::bail!(
"SendInput failed: sent {} of {} inputs",
sent,
inputs.len()
);
}
Ok(())
}
#[cfg(not(windows))]
pub fn key_down(&mut self, _vk_code: u16) -> Result<()> {
anyhow::bail!("Keyboard input only supported on Windows")
}
#[cfg(not(windows))]
pub fn key_up(&mut self, _vk_code: u16) -> Result<()> {
anyhow::bail!("Keyboard input only supported on Windows")
}
#[cfg(not(windows))]
pub fn type_char(&mut self, _ch: char) -> Result<()> {
anyhow::bail!("Keyboard input only supported on Windows")
}
#[cfg(not(windows))]
pub fn send_sas(&mut self) -> Result<()> {
anyhow::bail!("SAS only supported on Windows")
}
}
/// Common Windows virtual key codes
#[allow(dead_code)]
pub mod vk {
pub const BACK: u16 = 0x08;
pub const TAB: u16 = 0x09;
pub const RETURN: u16 = 0x0D;
pub const SHIFT: u16 = 0x10;
pub const CONTROL: u16 = 0x11;
pub const MENU: u16 = 0x12; // Alt
pub const PAUSE: u16 = 0x13;
pub const CAPITAL: u16 = 0x14; // Caps Lock
pub const ESCAPE: u16 = 0x1B;
pub const SPACE: u16 = 0x20;
pub const PRIOR: u16 = 0x21; // Page Up
pub const NEXT: u16 = 0x22; // Page Down
pub const END: u16 = 0x23;
pub const HOME: u16 = 0x24;
pub const LEFT: u16 = 0x25;
pub const UP: u16 = 0x26;
pub const RIGHT: u16 = 0x27;
pub const DOWN: u16 = 0x28;
pub const INSERT: u16 = 0x2D;
pub const DELETE: u16 = 0x2E;
// 0-9 keys
pub const KEY_0: u16 = 0x30;
pub const KEY_9: u16 = 0x39;
// A-Z keys
pub const KEY_A: u16 = 0x41;
pub const KEY_Z: u16 = 0x5A;
// Windows keys
pub const LWIN: u16 = 0x5B;
pub const RWIN: u16 = 0x5C;
// Function keys
pub const F1: u16 = 0x70;
pub const F2: u16 = 0x71;
pub const F3: u16 = 0x72;
pub const F4: u16 = 0x73;
pub const F5: u16 = 0x74;
pub const F6: u16 = 0x75;
pub const F7: u16 = 0x76;
pub const F8: u16 = 0x77;
pub const F9: u16 = 0x78;
pub const F10: u16 = 0x79;
pub const F11: u16 = 0x7A;
pub const F12: u16 = 0x7B;
// Modifier keys
pub const LSHIFT: u16 = 0xA0;
pub const RSHIFT: u16 = 0xA1;
pub const LCONTROL: u16 = 0xA2;
pub const RCONTROL: u16 = 0xA3;
pub const LMENU: u16 = 0xA4; // Left Alt
pub const RMENU: u16 = 0xA5; // Right Alt
}

91
agent/src/input/mod.rs Normal file
View File

@@ -0,0 +1,91 @@
//! Input injection module
//!
//! Handles mouse and keyboard input simulation using Windows SendInput API.
mod mouse;
mod keyboard;
pub use mouse::MouseController;
pub use keyboard::KeyboardController;
use anyhow::Result;
/// Combined input controller for mouse and keyboard
pub struct InputController {
mouse: MouseController,
keyboard: KeyboardController,
}
impl InputController {
/// Create a new input controller
pub fn new() -> Result<Self> {
Ok(Self {
mouse: MouseController::new()?,
keyboard: KeyboardController::new()?,
})
}
/// Get mouse controller
pub fn mouse(&mut self) -> &mut MouseController {
&mut self.mouse
}
/// Get keyboard controller
pub fn keyboard(&mut self) -> &mut KeyboardController {
&mut self.keyboard
}
/// Move mouse to absolute position
pub fn mouse_move(&mut self, x: i32, y: i32) -> Result<()> {
self.mouse.move_to(x, y)
}
/// Click mouse button
pub fn mouse_click(&mut self, button: MouseButton, down: bool) -> Result<()> {
if down {
self.mouse.button_down(button)
} else {
self.mouse.button_up(button)
}
}
/// Scroll mouse wheel
pub fn mouse_scroll(&mut self, delta_x: i32, delta_y: i32) -> Result<()> {
self.mouse.scroll(delta_x, delta_y)
}
/// Press or release a key
pub fn key_event(&mut self, vk_code: u16, down: bool) -> Result<()> {
if down {
self.keyboard.key_down(vk_code)
} else {
self.keyboard.key_up(vk_code)
}
}
/// Type a unicode character
pub fn type_unicode(&mut self, ch: char) -> Result<()> {
self.keyboard.type_char(ch)
}
/// Send Ctrl+Alt+Delete (requires special handling on Windows)
pub fn send_ctrl_alt_del(&mut self) -> Result<()> {
self.keyboard.send_sas()
}
}
/// Mouse button types
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MouseButton {
Left,
Right,
Middle,
X1,
X2,
}
impl Default for InputController {
fn default() -> Self {
Self::new().expect("Failed to create input controller")
}
}

217
agent/src/input/mouse.rs Normal file
View File

@@ -0,0 +1,217 @@
//! Mouse input simulation using Windows SendInput API
use super::MouseButton;
use anyhow::Result;
#[cfg(windows)]
use windows::Win32::UI::Input::KeyboardAndMouse::{
SendInput, INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL,
MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP,
MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK,
MOUSEEVENTF_WHEEL, MOUSEEVENTF_XDOWN, MOUSEEVENTF_XUP, MOUSEINPUT, XBUTTON1, XBUTTON2,
};
#[cfg(windows)]
use windows::Win32::UI::WindowsAndMessaging::{
GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN,
SM_YVIRTUALSCREEN,
};
/// Mouse input controller
pub struct MouseController {
/// Virtual screen dimensions for coordinate translation
#[cfg(windows)]
virtual_screen: VirtualScreen,
}
#[cfg(windows)]
struct VirtualScreen {
x: i32,
y: i32,
width: i32,
height: i32,
}
impl MouseController {
/// Create a new mouse controller
pub fn new() -> Result<Self> {
#[cfg(windows)]
{
let virtual_screen = unsafe {
VirtualScreen {
x: GetSystemMetrics(SM_XVIRTUALSCREEN),
y: GetSystemMetrics(SM_YVIRTUALSCREEN),
width: GetSystemMetrics(SM_CXVIRTUALSCREEN),
height: GetSystemMetrics(SM_CYVIRTUALSCREEN),
}
};
Ok(Self { virtual_screen })
}
#[cfg(not(windows))]
{
anyhow::bail!("Mouse input only supported on Windows")
}
}
/// Move mouse to absolute screen coordinates
#[cfg(windows)]
pub fn move_to(&mut self, x: i32, y: i32) -> Result<()> {
// Convert screen coordinates to normalized absolute coordinates (0-65535)
let norm_x = ((x - self.virtual_screen.x) * 65535) / self.virtual_screen.width;
let norm_y = ((y - self.virtual_screen.y) * 65535) / self.virtual_screen.height;
let input = INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: norm_x,
dy: norm_y,
mouseData: 0,
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
time: 0,
dwExtraInfo: 0,
},
},
};
self.send_input(&[input])
}
/// Press mouse button down
#[cfg(windows)]
pub fn button_down(&mut self, button: MouseButton) -> Result<()> {
let (flags, data) = match button {
MouseButton::Left => (MOUSEEVENTF_LEFTDOWN, 0),
MouseButton::Right => (MOUSEEVENTF_RIGHTDOWN, 0),
MouseButton::Middle => (MOUSEEVENTF_MIDDLEDOWN, 0),
MouseButton::X1 => (MOUSEEVENTF_XDOWN, XBUTTON1 as u32),
MouseButton::X2 => (MOUSEEVENTF_XDOWN, XBUTTON2 as u32),
};
let input = INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: 0,
dy: 0,
mouseData: data,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
};
self.send_input(&[input])
}
/// Release mouse button
#[cfg(windows)]
pub fn button_up(&mut self, button: MouseButton) -> Result<()> {
let (flags, data) = match button {
MouseButton::Left => (MOUSEEVENTF_LEFTUP, 0),
MouseButton::Right => (MOUSEEVENTF_RIGHTUP, 0),
MouseButton::Middle => (MOUSEEVENTF_MIDDLEUP, 0),
MouseButton::X1 => (MOUSEEVENTF_XUP, XBUTTON1 as u32),
MouseButton::X2 => (MOUSEEVENTF_XUP, XBUTTON2 as u32),
};
let input = INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: 0,
dy: 0,
mouseData: data,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
};
self.send_input(&[input])
}
/// Scroll mouse wheel
#[cfg(windows)]
pub fn scroll(&mut self, delta_x: i32, delta_y: i32) -> Result<()> {
let mut inputs = Vec::new();
// Vertical scroll
if delta_y != 0 {
inputs.push(INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: 0,
dy: 0,
mouseData: delta_y as u32,
dwFlags: MOUSEEVENTF_WHEEL,
time: 0,
dwExtraInfo: 0,
},
},
});
}
// Horizontal scroll
if delta_x != 0 {
inputs.push(INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: 0,
dy: 0,
mouseData: delta_x as u32,
dwFlags: MOUSEEVENTF_HWHEEL,
time: 0,
dwExtraInfo: 0,
},
},
});
}
if !inputs.is_empty() {
self.send_input(&inputs)?;
}
Ok(())
}
/// Send input events
#[cfg(windows)]
fn send_input(&self, inputs: &[INPUT]) -> Result<()> {
let sent = unsafe {
SendInput(inputs, std::mem::size_of::<INPUT>() as i32)
};
if sent as usize != inputs.len() {
anyhow::bail!("SendInput failed: sent {} of {} inputs", sent, inputs.len());
}
Ok(())
}
#[cfg(not(windows))]
pub fn move_to(&mut self, _x: i32, _y: i32) -> Result<()> {
anyhow::bail!("Mouse input only supported on Windows")
}
#[cfg(not(windows))]
pub fn button_down(&mut self, _button: MouseButton) -> Result<()> {
anyhow::bail!("Mouse input only supported on Windows")
}
#[cfg(not(windows))]
pub fn button_up(&mut self, _button: MouseButton) -> Result<()> {
anyhow::bail!("Mouse input only supported on Windows")
}
#[cfg(not(windows))]
pub fn scroll(&mut self, _delta_x: i32, _delta_y: i32) -> Result<()> {
anyhow::bail!("Mouse input only supported on Windows")
}
}