Skip to content

[codex] add macOS controller support #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
a16797 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[codex] add macOS controller support #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src-tauri/src/commands/maa_core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,17 @@ pub async fn connect_controller_impl(
let uuid_str = uuid.as_deref().unwrap_or("");
Controller::new_playcover(address, uuid_str).map_err(|e| e.to_string())?
}
ControllerConfig::MacOS {
handle,
screencap_method,
input_method,
..
} => {
let window_id = u32::try_from(*handle)
.map_err(|_| format!("Invalid macOS window handle '{}'", handle))?;
Controller::new_macos(window_id, *screencap_method, *input_method)
.map_err(|e| e.to_string())?
}
ControllerConfig::Dummy {
display_short_side, ..
} => {
Expand Down Expand Up @@ -627,6 +638,9 @@ pub async fn connect_controller_impl(
| ControllerConfig::PlayCover {
display_short_side, ..
}
| ControllerConfig::MacOS {
display_short_side, ..
}
| ControllerConfig::Dummy {
display_short_side, ..
} => display_short_side.unwrap_or(720),
Expand Down
7 changes: 7 additions & 0 deletions src-tauri/src/commands/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,13 @@ pub enum ControllerConfig {
#[serde(default)]
display_short_side: Option<i32>,
},
MacOS {
handle: u64,
screencap_method: u64,
input_method: u64,
#[serde(default)]
display_short_side: Option<i32>,
},
/// 空 controller：截图返回纯黑图、输入 no-op。
/// 用于在游戏未连接/已关闭时执行不依赖游戏画面的 MXU 特殊任务。
Dummy {
Expand Down
79 changes: 57 additions & 22 deletions src/components/ConnectionPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@ import type { AdbDevice, Win32Window, ControllerConfig } from '@/types/maa';
import type { ControllerItem, ResourceItem } from '@/types/interface';
import { computeResourcePaths } from '@/utils/resourcePath';
import { getProcessNameFromPath } from '@/utils/paths';
import { parseWin32ScreencapMethod, parseWin32InputMethod } from '@/types/maa';
import {
parseWin32ScreencapMethod,
parseWin32InputMethod,
parseMacOSScreencapMethod,
parseMacOSInputMethod,
} from '@/types/maa';
import { getInterfaceLangKey } from '@/i18n';
import { generateId } from '@/stores/helpers';
import {
Expand Down Expand Up @@ -184,6 +189,16 @@ export function ConnectionPanel() {
const currentController =
controllers.find((c) => c.name === currentControllerName) || controllers[0];
const controllerType = currentController?.type;
const isDesktopWindowController =
controllerType === 'Win32' || controllerType === 'Gamepad' || controllerType === 'MacOS';
const desktopWindowClassRegex =
controllerType === 'MacOS'
? currentController?.macos?.class_regex
: currentController?.win32?.class_regex || currentController?.gamepad?.class_regex;
const desktopWindowRegex =
controllerType === 'MacOS'
? currentController?.macos?.window_regex
: currentController?.win32?.window_regex || currentController?.gamepad?.window_regex;

// 获取资源列表
const allResources = projectInterface?.resource || [];
Expand Down Expand Up @@ -315,6 +330,7 @@ export function ConnectionPanel() {
controllerType === 'Adb' ||
controllerType === 'Win32' ||
controllerType === 'Gamepad' ||
controllerType === 'MacOS' ||
controllerType === 'WlRoots';

// 记录上一次的控制器名称，用于检测切换
Expand Down Expand Up @@ -348,7 +364,7 @@ export function ConnectionPanel() {
const hasHistoricalDevice =
savedDevice &&
((controllerType === 'Adb' && savedDevice.adbDeviceName) ||
((controllerType === 'Win32' || controllerType === 'Gamepad') && savedDevice.windowName) ||
(isDesktopWindowController && savedDevice.windowName) ||
(controllerType === 'WlRoots' && savedDevice.wlrSocketPath) ||
(controllerType === 'PlayCover' && savedDevice.playcoverAddress));

Expand Down Expand Up @@ -425,12 +441,11 @@ export function ConnectionPanel() {
// 有保存设备但匹配失败，显示下拉框让用户选择
setShowDeviceDropdown(true);
}
} else if (controllerType === 'Win32' || controllerType === 'Gamepad') {
const classRegex =
currentController.win32?.class_regex || currentController.gamepad?.class_regex;
const windowRegex =
currentController.win32?.window_regex || currentController.gamepad?.window_regex;
const windows = await maaService.findWin32Windows(classRegex, windowRegex);
} else if (isDesktopWindowController) {
const windows = await maaService.findWin32Windows(
desktopWindowClassRegex,
desktopWindowRegex,
);
setCachedWin32Windows(windows);

// 自动连接策略（与 Adb 一致）
Expand Down Expand Up @@ -577,6 +592,16 @@ export function ConnectionPanel() {
};
deviceName = selectedWindow.window_name || selectedWindow.class_name;
targetType = 'window';
} else if (controllerType === 'MacOS' && selectedWindow) {
config = {
type: 'MacOS',
handle: selectedWindow.handle,
screencap_method: parseMacOSScreencapMethod(currentController.macos?.screencap || ''),
input_method: parseMacOSInputMethod(currentController.macos?.input || ''),
display_short_side: currentController?.display_short_side,
};
deviceName = selectedWindow.window_name || selectedWindow.class_name;
targetType = 'window';
} else if (controllerType === 'WlRoots' && selectedWlrootsSocket) {
config = {
type: 'WlRoots',
Expand Down Expand Up @@ -606,7 +631,7 @@ export function ConnectionPanel() {
targetType = 'window';
} else {
throw new Error(
controllerType === 'Win32' || controllerType === 'Gamepad'
isDesktopWindowController
? t('controller.selectWindow')
: t('controller.selectDevice'),
);
Expand Down Expand Up @@ -735,6 +760,7 @@ export function ConnectionPanel() {
case 'Win32':
case 'WlRoots':
return <Monitor className="w-4 h-4" />;
case 'MacOS':
case 'PlayCover':
return <Apple className="w-4 h-4" />;
case 'Gamepad':
Expand All @@ -758,7 +784,7 @@ export function ConnectionPanel() {
}
return t('controller.selectDevice');
}
if (controllerType === 'Win32' || controllerType === 'Gamepad') {
if (isDesktopWindowController) {
if (selectedWindow) {
return selectedWindow.window_name || selectedWindow.class_name;
}
Expand Down Expand Up @@ -904,6 +930,14 @@ export function ConnectionPanel() {
keyboard_method: parseWin32InputMethod(currentController?.win32?.keyboard || ''),
display_short_side: currentController?.display_short_side,
};
} else if (controllerType === 'MacOS') {
config = {
type: 'MacOS',
handle: win.handle,
screencap_method: parseMacOSScreencapMethod(currentController?.macos?.screencap || ''),
input_method: parseMacOSInputMethod(currentController?.macos?.input || ''),
display_short_side: currentController?.display_short_side,
};
} else {
config = {
type: 'Gamepad',
Expand All @@ -914,8 +948,10 @@ export function ConnectionPanel() {

await connectControllerInternal(config, win.window_name || win.class_name, 'window');

// 连接成功后异步获取进程路径（Win32 和 Gamepad 都基于窗口句柄）
void fetchAndStoreProcessPath(win.handle);
// 连接成功后异步获取进程路径（仅 Windows 窗口句柄支持该查询）
if (controllerType === 'Win32' || controllerType === 'Gamepad') {
void fetchAndStoreProcessPath(win.handle);
}
} catch (err) {
setDeviceError(err instanceof Error ? err.message : t('controller.connectionFailed'));
setIsConnected(false);
Expand Down Expand Up @@ -1003,12 +1039,11 @@ export function ConnectionPanel() {
if (devices.length > 0) {
setShowDeviceDropdown(true);
}
} else if (controllerType === 'Win32' || controllerType === 'Gamepad') {
const classRegex =
currentController.win32?.class_regex || currentController.gamepad?.class_regex;
const windowRegex =
currentController.win32?.window_regex || currentController.gamepad?.window_regex;
const windows = await maaService.findWin32Windows(classRegex, windowRegex);
} else if (isDesktopWindowController) {
const windows = await maaService.findWin32Windows(
desktopWindowClassRegex,
desktopWindowRegex,
);
setCachedWin32Windows(windows);

// 尝试匹配保存的窗口名称
Expand Down Expand Up @@ -1090,7 +1125,7 @@ export function ConnectionPanel() {

return [];
}
if (controllerType === 'Win32' || controllerType === 'Gamepad') {
if (isDesktopWindowController) {
// 如果缓存中有窗口，使用缓存列表
if (cachedWin32Windows.length > 0) {
return cachedWin32Windows.map((window) => ({
Expand Down Expand Up @@ -1154,7 +1189,7 @@ export function ConnectionPanel() {
// 判断是否可以连接
const canConnect = () => {
if (controllerType === 'Adb') return !!selectedAdbDevice;
if (controllerType === 'Win32' || controllerType === 'Gamepad') return !!selectedWindow;
if (isDesktopWindowController) return !!selectedWindow;
if (controllerType === 'WlRoots') return !!selectedWlrootsSocket;
if (controllerType === 'PlayCover') return playcoverAddress.trim().length > 0;
return false;
Expand Down Expand Up @@ -1505,7 +1540,7 @@ export function ConnectionPanel() {
<div className="px-3 py-3 text-center text-text-muted text-xs">
{isSearching
? t('common.loading')
: controllerType === 'Win32' || controllerType === 'Gamepad'
: isDesktopWindowController
? t('controller.noWindows')
: t('controller.noDevices')}
</div>
Expand All @@ -1526,7 +1561,7 @@ export function ConnectionPanel() {
: 'hover:bg-bg-hover hover:border-accent',
)}
title={
controllerType === 'Win32' || controllerType === 'Gamepad'
isDesktopWindowController
? t('controller.refreshWindows')
: t('controller.refreshDevices')
}
Expand Down
57 changes: 46 additions & 11 deletions src/components/DeviceSelector.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@ import { maaService } from '@/services/maaService';
import { useAppStore } from '@/stores/appStore';
import type { AdbDevice, Win32Window, ControllerConfig } from '@/types/maa';
import type { ControllerItem } from '@/types/interface';
import { parseWin32ScreencapMethod, parseWin32InputMethod } from '@/types/maa';
import {
parseWin32ScreencapMethod,
parseWin32InputMethod,
parseMacOSScreencapMethod,
parseMacOSInputMethod,
} from '@/types/maa';
import { loggers } from '@/utils/logger';

const log = loggers.device;
Expand Down Expand Up @@ -101,6 +106,16 @@ export function DeviceSelector({
}, [showDropdown]);

const controllerType = controllerDef.type;
const isDesktopWindowController =
controllerType === 'Win32' || controllerType === 'Gamepad' || controllerType === 'MacOS';
const desktopWindowClassRegex =
controllerType === 'MacOS'
? controllerDef.macos?.class_regex
: controllerDef.win32?.class_regex || controllerDef.gamepad?.class_regex;
const desktopWindowRegex =
controllerType === 'MacOS'
? controllerDef.macos?.window_regex
: controllerDef.win32?.window_regex || controllerDef.gamepad?.window_regex;

// PlayCover 地址输入
const [playcoverAddress, setPlaycoverAddress] = useState('127.0.0.1:1717');
Expand Down Expand Up @@ -143,6 +158,7 @@ export function DeviceSelector({
controllerType === 'Adb' ||
controllerType === 'Win32' ||
controllerType === 'WlRoots' ||
controllerType === 'MacOS' ||
controllerType === 'Gamepad';

// 初始化 MaaFramework（如果还没初始化）
Expand Down Expand Up @@ -180,11 +196,11 @@ export function DeviceSelector({
if (devices.length > 0) {
setShowDropdown(true);
}
} else if (controllerType === 'Win32' || controllerType === 'Gamepad') {
const classRegex = controllerDef.win32?.class_regex || controllerDef.gamepad?.class_regex;
const windowRegex =
controllerDef.win32?.window_regex || controllerDef.gamepad?.window_regex;
const windows = await maaService.findWin32Windows(classRegex, windowRegex);
} else if (isDesktopWindowController) {
const windows = await maaService.findWin32Windows(
desktopWindowClassRegex,
desktopWindowRegex,
);
setCachedWin32Windows(windows);
if (windows.length === 1) {
setSelectedWindow(windows[0]);
Expand Down Expand Up @@ -246,6 +262,14 @@ export function DeviceSelector({
keyboard_method: parseWin32InputMethod(controllerDef.win32?.keyboard || ''),
display_short_side: controllerDef.display_short_side,
};
} else if (controllerType === 'MacOS' && selectedWindow) {
config = {
type: 'MacOS',
handle: selectedWindow.handle,
screencap_method: parseMacOSScreencapMethod(controllerDef.macos?.screencap || ''),
input_method: parseMacOSInputMethod(controllerDef.macos?.input || ''),
display_short_side: controllerDef.display_short_side,
};
} else if (controllerType === 'WlRoots' && selectedWlrootsSocket) {
config = {
type: 'WlRoots',
Expand Down Expand Up @@ -276,7 +300,7 @@ export function DeviceSelector({
if (controllerType === 'Adb' && selectedAdbDevice) {
deviceName = selectedAdbDevice.name || selectedAdbDevice.address;
targetType = 'device';
} else if ((controllerType === 'Win32' || controllerType === 'Gamepad') && selectedWindow) {
} else if (isDesktopWindowController && selectedWindow) {
deviceName = selectedWindow.window_name || selectedWindow.class_name;
targetType = 'window';
} else if (controllerType === 'WlRoots' && selectedWlrootsSocket) {
Expand Down Expand Up @@ -317,7 +341,7 @@ export function DeviceSelector({
if (controllerType === 'Adb' && selectedAdbDevice) {
return `${selectedAdbDevice.name} (${selectedAdbDevice.address})`;
}
if ((controllerType === 'Win32' || controllerType === 'Gamepad') && selectedWindow) {
if (isDesktopWindowController && selectedWindow) {
return selectedWindow.window_name || selectedWindow.class_name;
}
if (controllerType === 'WlRoots' && selectedWlrootsSocket) {
Expand Down Expand Up @@ -396,6 +420,14 @@ export function DeviceSelector({
keyboard_method: parseWin32InputMethod(controllerDef.win32?.keyboard || ''),
display_short_side: controllerDef.display_short_side,
};
} else if (controllerType === 'MacOS') {
config = {
type: 'MacOS',
handle: win.handle,
screencap_method: parseMacOSScreencapMethod(controllerDef.macos?.screencap || ''),
input_method: parseMacOSInputMethod(controllerDef.macos?.input || ''),
display_short_side: controllerDef.display_short_side,
};
} else {
config = {
type: 'Gamepad',
Expand Down Expand Up @@ -470,7 +502,7 @@ export function DeviceSelector({
onClick: () => handleSelectAdbDevice(device),
}));
}
if (controllerType === 'Win32' || controllerType === 'Gamepad') {
if (isDesktopWindowController) {
return cachedWin32Windows.map((window) => ({
id: String(window.handle),
name: window.window_name || '(无标题)',
Expand All @@ -494,7 +526,7 @@ export function DeviceSelector({
// 判断是否可以连接
const canConnect = () => {
if (controllerType === 'Adb') return !!selectedAdbDevice;
if (controllerType === 'Win32' || controllerType === 'Gamepad') return !!selectedWindow;
if (isDesktopWindowController) return !!selectedWindow;
if (controllerType === 'WlRoots') return !!selectedWlrootsSocket;
if (controllerType === 'PlayCover') return playcoverAddress.trim().length > 0;
return false;
Expand All @@ -510,6 +542,7 @@ export function DeviceSelector({
case 'Win32':
case 'WlRoots':
return <Monitor className="w-4 h-4" />;
case 'MacOS':
case 'PlayCover':
return <Apple className="w-4 h-4" />;
case 'Gamepad':
Expand All @@ -530,6 +563,8 @@ export function DeviceSelector({
return t('controller.wlroots');
case 'PlayCover':
return t('controller.playcover');
case 'MacOS':
return t('controller.macos');
case 'Gamepad':
return t('controller.gamepad');
default:
Expand Down Expand Up @@ -672,7 +707,7 @@ export function DeviceSelector({
: 'hover:bg-bg-hover hover:border-accent',
)}
title={
controllerType === 'Win32' || controllerType === 'Gamepad'
isDesktopWindowController
? t('controller.refreshWindows')
: t('controller.refreshDevices')
}
Expand Down
Loading
Loading