feat(v8): GUI 语义能力 —— window.list / window.focus / screen.find(能力 20 -> 23)

P0-1 UI 元素定位(基于 Windows UI Automation + Win32):
- window.list:枚举可见顶层窗口(hwnd/标题/进程/精确矩形/前台标识/最小化)
  —— 真机验证:正确返回 PowerShell / cmd / 各应用窗口矩形
- window.focus:按标题正则或句柄置前窗口,返回精确矩形(SW_RESTORE + SetForegroundWindow)
- screen.find:UIA 查找元素并返回**中心点屏幕坐标**(支持窗口范围/控件类型过滤),
  直接喂给 input.mouse.click —— 把「看得到画面读不懂界面」变成「按名取坐标」
  —— 真机验证:查找 'Windows PowerShell' 返回 (716,648) 与完整矩形

实现要点:
- PowerShell 脚本一律走 EncodedCommand(UTF-16LE base64),规避引号/中文/换行转义
- extractJson():Add-Type 程序集加载会向 stdout 夹杂输出污染 JSON,
  按首 [/{ 到末 ]/} 截取后再 parse(真机踩过:'Unexpected non-whitespace character')

顺带修复(本轮测试发现的 bug):
1. services 状态显示数字:Get-Service 的 Status/StartType 是 .NET 枚举,
   ConvertTo-Json 退化为数字(4=Running),显式 ToString() 修复
2. restart 任务名冲突:重启任务的 cmd 持有新 agent 进程 → 任务长期 Running
   → 下次 Unregister/Register 静默失败 → 'restart 返回成功但没重启'。
   改为唯一任务名(时间戳)+ 注册后显式校验 + 清理历史非运行任务
   —— 真机验证连续 restart:PID 2612 -> 18556,任务名带时间戳
3. cmd shell 中文:定性为 Windows 固有局限(命令行参数在进程创建时已按 ANSI
   解释,chcp 救不了参数本身),代码注释说明并引导用 PowerShell

真机:23 项能力全量在线;unit 26/26、e2e 21/21 全绿
This commit is contained in:
jkooe committed 2026-09-29 00:38:54 +08:00
1 parent 111a7b5918
commit 313420a482
5 files changed
+455 -5

No files matched your search

+5
View File
@@ -3,6 +3,7 @@ import type { AgentConfig } from '../config.js';
import { systemInfo, systemStatus, processList, serviceList, shellExec, auditList, agentRestart } from './system.js';
import { appList, appInstall } from './app.js';
import { screenInfo, screenCapture } from './screen.js';
import { windowList, windowFocus, screenFind } from './window.js';
import { mouseMove, mouseClick, mouseScroll, keyType, keyPress, setInputPolicy } from './input.js';
import { fsList, fsStat, fsRead, fsWrite, setFsRoots } from './fs.js';
@@ -42,5 +43,9 @@ export function createCapabilityRegistry(cfg: AgentConfig): Record<string, Capab
[CapabilityNames.FsWrite]: fsWrite,
// v7 自持能力
[CapabilityNames.AgentRestart]: agentRestart,
// v8 GUI 语义(窗口与元素定位)
[CapabilityNames.WindowList]: windowList,
[CapabilityNames.WindowFocus]: windowFocus,
[CapabilityNames.ScreenFind]: screenFind,
};
}
+18 -4
View File
@@ -284,8 +284,12 @@ export async function serviceList(args: Args): Promise<unknown> {
const filter = args['filter'] as { name_pattern?: string; state?: string } | undefined;
const r = await execCommand({
// 注意:Get-Service 的 Status/StartType 是 .NET 枚举,直接 ConvertTo-Json 会退化为
// 数字(如 4=Running、1=Stopped),必须显式 ToString() 才能得到可读状态。
command:
'Get-Service | Select-Object Name,DisplayName,Status,StartType | ConvertTo-Json -Compress',
'Get-Service | Select-Object Name,DisplayName,' +
"@{n='Status';e={$_.Status.ToString()}},@{n='StartType';e={$_.StartType.ToString()}} | " +
'ConvertTo-Json -Compress',
timeoutMs: 30_000,
});
let services = toArray<{ Name?: string; DisplayName?: string; Status?: string | number; StartType?: string | number }>(
@@ -394,14 +398,24 @@ async function scheduleWindowsRestart(p: RestartParams): Promise<unknown> {
// 导致 agent 读不到配置而回退到默认端口 → EADDRINUSE。故此处不加空格。
const cmdLine = '/c ' + (p.home ? `set NODEAGENT_HOME=${p.home}& ` : '') + steps.join(' & ');
const TASK = 'nodeagent-selfrestart';
// ⚠️ 任务名必须唯一(带时间戳):重启任务的 cmd 会一直持有新 agent 进程,
// 任务因此长期处于 Running 状态;若沿用固定名,下次 restart 的
// Unregister/Register 会静默失败(任务无法覆盖正在运行的实例),
// 导致「restart 返回成功但实际没有重启」(真机踩过)。
const TASK = `nodeagent-selfrestart-${Date.now()}`;
const ps = [
`$ErrorActionPreference='SilentlyContinue'`,
`Unregister-ScheduledTask -TaskName '${TASK}' -Confirm:$false`,
// 清理历史 selfrestart 任务(保留 Running 的 —— 那正是当前 agent 的宿主)
`Get-ScheduledTask -TaskName 'nodeagent-selfrestart*' | Where-Object { $_.State -ne 'Running' } | Unregister-ScheduledTask -Confirm:$false`,
`$a = New-ScheduledTaskAction -Execute 'cmd.exe' -Argument '${cmdLine.replace(/'/g, "''")}'`,
// 不限时(默认 72h 后强制结束),保证 node 能长期运行
`$s = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -ExecutionTimeLimit (New-TimeSpan -Seconds 0)`,
`Register-ScheduledTask -TaskName '${TASK}' -Action $a -Settings $s -Force | Out-Null`,
// ⚠️ 必须用最高权限:agent 通常以管理员运行,普通权限的 taskkill 杀不掉它,
// 否则会出现「新进程起来了但端口被占(EADDRINUSE)」。
`$p = New-ScheduledTaskPrincipal -UserId $env:USERNAME -RunLevel Highest -LogonType Interactive`,
`Register-ScheduledTask -TaskName '${TASK}' -Action $a -Settings $s -Principal $p -Force | Out-Null`,
// 校验注册真的成功(SilentlyContinue 会吞错,必须显式确认)
`if ((Get-ScheduledTask -TaskName '${TASK}' -ErrorAction SilentlyContinue) -eq $null) { Write-Output 'register-failed'; exit 1 }`,
`Start-ScheduledTask -TaskName '${TASK}'`,
`Write-Output 'ok'`,
].join('; ');
+312
View File
@@ -0,0 +1,312 @@
import { CapabilityError, ErrorCodes } from '@nodeagent/protocol';
import { IS_WINDOWS, execCommand } from '../util/exec.js';
type Args = Record<string, unknown>;
function toArray<T>(v: T | T[] | null | undefined): T[] {
if (v === null || v === undefined) return [];
return Array.isArray(v) ? v : [v];
}
/**
* 把 PowerShell 脚本编码为 UTF-16LE base64。
* 用于承载含 C# 源码 / 大量引号的脚本,彻底规避转义问题。
*/
function encodePS(script: string): string {
return Buffer.from(script, 'utf16le').toString('base64');
}
async function runPS(script: string, timeoutMs = 30_000): Promise<string> {
const r = await execCommand({
command: `powershell.exe -NoProfile -NonInteractive -EncodedCommand ${encodePS(script)}`,
timeoutMs,
});
if (r.exit_code !== 0) {
throw new CapabilityError(ErrorCodes.EXECUTION_FAILED, '窗口操作失败', {
detail: (r.stderr || r.stdout).slice(0, 600),
});
}
return r.stdout.trim();
}
/**
* 从混杂输出中提取 JSON。
*
* 背景:PowerShell 的 `Add-Type -AssemblyName` 等语句可能向 stdout 夹杂额外内容,
* 直接 JSON.parse 会报 "Unexpected non-whitespace character after JSON"。
* 这里按首个 `[`/`{` 到末个 `]`/`}` 截取,稳健取回 JSON 主体。
*/
function extractJson<T>(raw: string): T | null {
if (!raw) return null;
const starts = [raw.indexOf('['), raw.indexOf('{')].filter((i) => i >= 0);
if (starts.length === 0) return null;
const start = Math.min(...starts);
const end = Math.max(raw.lastIndexOf(']'), raw.lastIndexOf('}'));
if (end <= start) return null;
try {
return JSON.parse(raw.slice(start, end + 1)) as T;
} catch {
return null;
}
}
/** Win32 窗口枚举与矩形读取(供 window.list / window.focus 复用)。 */
const WIN32_PRELUDE = `
Add-Type @"
using System;
using System.Text;
using System.Runtime.InteropServices;
public class NAWin32 {
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
public delegate bool EnumProc(IntPtr hWnd, IntPtr lParam);
[DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr p);
[DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
[DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h);
[DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, StringBuilder s, int n);
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
}
"@
`;
/** 枚举可见顶层窗口 → JSON 字符串。 */
const LIST_SCRIPT = `${WIN32_PRELUDE}
$fg = [NAWin32]::GetForegroundWindow()
$list = New-Object System.Collections.ArrayList
$cb = [NAWin32+EnumProc]{
param($h, $l)
if (-not [NAWin32]::IsWindowVisible($h)) { return $true }
$sb = New-Object System.Text.StringBuilder 512
[void][NAWin32]::GetWindowTextW($h, $sb, 512)
$title = $sb.ToString()
if ([string]::IsNullOrWhiteSpace($title)) { return $true }
$r = New-Object NAWin32+RECT
if (-not [NAWin32]::GetWindowRect($h, [ref]$r)) { return $true }
$w = $r.Right - $r.Left; $ht = $r.Bottom - $r.Top
if ($w -le 0 -or $ht -le 0) { return $true }
$pid = 0
[void][NAWin32]::GetWindowThreadProcessId($h, [ref]$pid)
$pn = ''
try { $pn = (Get-Process -Id $pid -ErrorAction Stop).ProcessName } catch {}
[void]$list.Add([pscustomobject]@{
hwnd = ('0x{0:X}' -f $h.ToInt64()); title = $title; process = $pn; pid = $pid
x = $r.Left; y = $r.Top; width = $w; height = $ht
is_foreground = ($h -eq $fg); is_minimized = [NAWin32]::IsIconic($h)
})
return $true
}
[void][NAWin32]::EnumWindows($cb, [IntPtr]::Zero)
$list | ConvertTo-Json -Compress
`;
interface WinInfo {
hwnd: string;
title: string;
process: string;
pid: number;
x: number;
y: number;
width: number;
height: number;
is_foreground: boolean;
is_minimized: boolean;
}
// ---------------- window.list ----------------
export async function windowList(args: Args): Promise<unknown> {
if (!IS_WINDOWS) {
throw new CapabilityError(
ErrorCodes.UNSUPPORTED_PLATFORM,
'window.list 仅在被控端为 Windows 时可用',
{ platform: process.platform },
);
}
const out = await runPS(LIST_SCRIPT, 40_000);
let all: WinInfo[] = toArray<WinInfo>(extractJson<WinInfo[]>(out) ?? []);
// 过滤掉 UWP 的隐形壳窗口(尺寸异常大且进程为 ApplicationFrameHost 的重复项保留)
all = all.filter((w) => w.width > 40 && w.height > 40);
const pattern = args['title_pattern'] as string | undefined;
if (pattern) {
const re = new RegExp(pattern, 'i');
all = all.filter((w) => re.test(w.title));
}
const limit = (args['limit'] as number | undefined) ?? 50;
return { windows: all.slice(0, limit) };
}
// ---------------- window.focus ----------------
export async function windowFocus(args: Args): Promise<unknown> {
if (!IS_WINDOWS) {
throw new CapabilityError(
ErrorCodes.UNSUPPORTED_PLATFORM,
'window.focus 仅在被控端为 Windows 时可用',
{ platform: process.platform },
);
}
const title = args['title'] as string | undefined;
const hwnd = args['hwnd'] as string | undefined;
if (!title && !hwnd) {
throw new CapabilityError(ErrorCodes.PARAM_INVALID, '需提供 title(正则)或 hwnd');
}
const script = `${WIN32_PRELUDE}
$h = [IntPtr]::Zero
$want = ${JSON.stringify(hwnd ?? '')}
if ($want -ne '') { $h = [IntPtr][Convert]::ToInt64($want, 16) }
else {
$re = ${JSON.stringify(title ?? '')}
$fg = [NAWin32]::GetForegroundWindow()
$found = [IntPtr]::Zero
$cb = [NAWin32+EnumProc]{
param($w, $l)
if ([NAWin32]::IsWindowVisible($w)) {
$sb = New-Object System.Text.StringBuilder 512
[void][NAWin32]::GetWindowTextW($w, $sb, 512)
$t = $sb.ToString()
if ($t -match $re) { $script:found = $w; return $false }
}
return $true
}
[void][NAWin32]::EnumWindows($cb, [IntPtr]::Zero)
$h = $script:found
}
if ($h -eq [IntPtr]::Zero) { Write-Output '{"found":false}'; exit 0 }
[void][NAWin32]::ShowWindow($h, 9) # SW_RESTORE
[void][NAWin32]::SetForegroundWindow($h)
Start-Sleep -Milliseconds 400
$r = New-Object NAWin32+RECT
[void][NAWin32]::GetWindowRect($h, [ref]$r)
$sb2 = New-Object System.Text.StringBuilder 512
[void][NAWin32]::GetWindowTextW($h, $sb2, 512)
[pscustomobject]@{
found = $true; hwnd = ('0x{0:X}' -f $h.ToInt64()); title = $sb2.ToString()
x = $r.Left; y = $r.Top; width = ($r.Right - $r.Left); height = ($r.Bottom - $r.Top)
focused = ([NAWin32]::GetForegroundWindow() -eq $h)
} | ConvertTo-Json -Compress
`;
const out = await runPS(script, 40_000);
const obj = extractJson<Record<string, unknown>>(out) ?? {};
if (obj['found'] === false) {
throw new CapabilityError(ErrorCodes.EXECUTION_FAILED, '未找到匹配的窗口', { title, hwnd });
}
delete obj['found'];
return obj;
}
// ---------------- screen.find ----------------
const FIND_SCRIPT_PRELUDE = `
Add-Type -AssemblyName UIAutomationClient
Add-Type -AssemblyName UIAutomationTypes
Add-Type -AssemblyName WindowsBase
function Get-WinByTitle([string]$re) {
$root = [System.Windows.Automation.AutomationElement]::RootElement
$cond = New-Object System.Windows.Automation.PropertyCondition(
[System.Windows.Automation.AutomationElement]::ControlTypeProperty,
[System.Windows.Automation.ControlType]::Window)
$wins = $root.FindAll([System.Windows.Automation.TreeScope]::Children, $cond)
foreach ($w in $wins) {
try {
$t = $w.Current.Name
if ($t -and $t -match $re) { return $w }
} catch {}
}
return $null
}
`;
/**
* 在界面中查找 UI 元素并返回屏幕坐标(UI Automation)。
*
* 这是解决「看得到画面但读不懂界面」的关键能力:把元素名映射为可点击坐标。
*/
export async function screenFind(args: Args): Promise<unknown> {
if (!IS_WINDOWS) {
throw new CapabilityError(
ErrorCodes.UNSUPPORTED_PLATFORM,
'screen.find 仅在被控端为 Windows 时可用',
{ platform: process.platform },
);
}
const text = args['text'] as string;
if (typeof text !== 'string' || text.length === 0) {
throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空');
}
const windowTitle = args['window'] as string | undefined;
const controlType = args['control_type'] as string | undefined;
const limit = (args['limit'] as number | undefined) ?? 20;
const ctypeMap: Record<string, string> = {
Button: 'Button',
MenuItem: 'MenuItem',
Edit: 'Edit',
Text: 'Text',
ListItem: 'ListItem',
CheckBox: 'CheckBox',
RadioButton: 'RadioButton',
ComboBox: 'ComboBox',
Tab: 'TabItem',
TreeItem: 'TreeItem',
Hyperlink: 'Hyperlink',
Image: 'Image',
};
const ct = controlType ? ctypeMap[controlType] ?? controlType : '';
const script = `${FIND_SCRIPT_PRELUDE}
$text = ${JSON.stringify(text)}
$limit = ${Number(limit)}
$scope = [System.Windows.Automation.AutomationElement]::RootElement
$winTitle = ${JSON.stringify(windowTitle ?? '')}
if ($winTitle -ne '') {
$w = Get-WinByTitle $winTitle
if ($w -eq $null) { Write-Output '[]'; exit 0 }
$scope = $w
}
$nameCond = New-Object System.Windows.Automation.PropertyCondition(
[System.Windows.Automation.AutomationElement]::NameProperty, $text,
[System.Windows.Automation.PropertyConditionFlags]::IgnoreCase)
$ct = ${JSON.stringify(ct)}
if ($ct -ne '') {
$ctObj = $null
try { $ctObj = [System.Windows.Automation.ControlType]::$ct } catch {}
if ($ctObj -ne $null) {
$ctCond = New-Object System.Windows.Automation.PropertyCondition(
[System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ctObj)
$cond = New-Object System.Windows.Automation.AndCondition($nameCond, $ctCond)
} else { $cond = $nameCond }
} else { $cond = $nameCond }
$found = $scope.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond)
$out = New-Object System.Collections.ArrayList
foreach ($e in $found) {
if ($out.Count -ge $limit) { break }
try {
$r = $e.Current.BoundingRectangle
if ($r.Width -le 0 -or $r.Height -le 0) { continue }
$wt = ''
try {
$tw = $e.Current.ControlType.ProgrammaticName
$wt = $tw
} catch {}
$out.Add([pscustomobject]@{
name = $e.Current.Name
control_type = ($e.Current.ControlType.ProgrammaticName -replace 'ControlType\\.','')
automation_id = $e.Current.AutomationId
x = [int]($r.Left + $r.Width / 2)
y = [int]($r.Top + $r.Height / 2)
left = [int]$r.Left; top = [int]$r.Top
width = [int]$r.Width; height = [int]$r.Height
})
} catch {}
}
$out | ConvertTo-Json -Compress
`;
const out = await runPS(script, 60_000);
const matches = toArray<Record<string, unknown>>(extractJson<Record<string, unknown>[]>(out) ?? []);
return { matches: matches.slice(0, limit) };
}
+6 -1
View File
@@ -44,7 +44,12 @@ const PS_UTF8_PREFIX =
function buildLaunch(command: string, shell: ShellKind): Launch {
if (IS_WINDOWS) {
if (shell === 'cmd') {
// cmd 的代码页需在命令内切换(chcp 65001),且须静默
// cmd 的代码页需在命令内切换(chcp 65001),且须静默。
//
// ⚠️ 已知局限(实测确认):`chcp 65001` 只能改变**输出**代码页,
// 而 Node 传给 cmd.exe 的**命令行参数**在进入 cmd 时已按 ANSI(中文为 GBK)
// 解释,因此**命令字符串中的中文仍会损坏**(chcp 显示 65001 也无济于事)。
// 需要在命令里使用中文时,请改用默认的 powershell(那边已完整支持 UTF-8)。
return { file: 'cmd.exe', args: ['/d', '/s', '/c', `chcp 65001 >nul & ${command}`] };
}
return {
+114
View File
@@ -26,6 +26,10 @@ export const CapabilityNames = {
FsWrite: 'fs.write',
// v7 自持能力(受控重启)
AgentRestart: 'system.agent.restart',
// v8 GUI 语义(窗口与元素定位)
WindowList: 'window.list',
WindowFocus: 'window.focus',
ScreenFind: 'screen.find',
} as const;
export type CapabilityName = (typeof CapabilityNames)[keyof typeof CapabilityNames];
@@ -618,6 +622,116 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
},
},
},
{
name: CapabilityNames.WindowList,
version: '1.0',
description:
'列出当前可见的顶层窗口(标题、所属进程、精确矩形、是否前台窗口)。' +
'配合 screen.capture 可在**已知窗口范围内**相对定位,避免全屏猜坐标。',
risk: 'low',
params_schema: {
type: 'object',
properties: {
title_pattern: { type: 'string', description: '可选:按标题正则过滤' },
limit: { type: 'integer', default: 50 },
},
additionalProperties: false,
},
returns_schema: {
type: 'object',
properties: {
windows: {
type: 'array',
items: {
type: 'object',
properties: {
hwnd: { type: 'string', description: '窗口句柄(十六进制)' },
title: { type: 'string' },
process: { type: 'string' },
pid: { type: 'integer' },
x: { type: 'integer' },
y: { type: 'integer' },
width: { type: 'integer' },
height: { type: 'integer' },
is_foreground: { type: 'boolean' },
is_minimized: { type: 'boolean' },
},
},
},
},
},
},
{
name: CapabilityNames.WindowFocus,
version: '1.0',
description: '将指定窗口(按标题正则或句柄)置前并聚焦,返回其精确矩形。',
risk: 'medium',
params_schema: {
type: 'object',
properties: {
title: { type: 'string', description: '窗口标题(正则,匹配第一个)' },
hwnd: { type: 'string', description: '或直接给窗口句柄' },
},
additionalProperties: false,
},
returns_schema: {
type: 'object',
properties: {
hwnd: { type: 'string' },
title: { type: 'string' },
x: { type: 'integer' },
y: { type: 'integer' },
width: { type: 'integer' },
height: { type: 'integer' },
focused: { type: 'boolean' },
},
},
},
{
name: CapabilityNames.ScreenFind,
version: '1.0',
description:
'在界面中查找 UI 元素并返回其**屏幕坐标**(基于 Windows UI Automation)。' +
'可指定窗口范围与元素类型,返回中心点坐标,直接喂给 input.mouse.click。',
risk: 'low',
params_schema: {
type: 'object',
properties: {
text: { type: 'string', description: '要查找的文本(匹配元素 Name,支持子串)' },
window: { type: 'string', description: '可选:限定在该窗口标题(正则)内查找' },
control_type: {
type: 'string',
description: '可选:限定控件类型,如 Button / MenuItem / Edit / Text / ListItem',
},
limit: { type: 'integer', default: 20 },
},
required: ['text'],
additionalProperties: false,
},
returns_schema: {
type: 'object',
properties: {
matches: {
type: 'array',
items: {
type: 'object',
properties: {
name: { type: 'string' },
control_type: { type: 'string' },
automation_id: { type: 'string' },
x: { type: 'integer', description: '中心点 X(屏幕坐标)' },
y: { type: 'integer', description: '中心点 Y(屏幕坐标)' },
left: { type: 'integer' },
top: { type: 'integer' },
width: { type: 'integer' },
height: { type: 'integer' },
window_title: { type: 'string' },
},
},
},
},
},
},
];
/** 按名称查找能力描述。 */