feat(v8): GUI 语义能力 —— window.list / window.focus / screen.find(能力 20 -> 23)
P0-1 UI 元素定位(基于 Windows UI Automation + Win32):
- window.list:枚举可见顶层窗口(hwnd/标题/进程/精确矩形/前台标识/最小化)
—— 真机验证:正确返回 PowerShell / cmd / 各应用窗口矩形
- window.focus:按标题正则或句柄置前窗口,返回精确矩形(SW_RESTORE + SetForegroundWindow)
- screen.find:UIA 查找元素并返回**中心点屏幕坐标**(支持窗口范围/控件类型过滤),
直接喂给 input.mouse.click —— 把「看得到画面读不懂界面」变成「按名取坐标」
—— 真机验证:查找 'Windows PowerShell' 返回 (716,648) 与完整矩形
实现要点:
- PowerShell 脚本一律走 EncodedCommand(UTF-16LE base64),规避引号/中文/换行转义
- extractJson():Add-Type 程序集加载会向 stdout 夹杂输出污染 JSON,
按首 [/{ 到末 ]/} 截取后再 parse(真机踩过:'Unexpected non-whitespace character')
顺带修复(本轮测试发现的 bug):
1. services 状态显示数字:Get-Service 的 Status/StartType 是 .NET 枚举,
ConvertTo-Json 退化为数字(4=Running),显式 ToString() 修复
2. restart 任务名冲突:重启任务的 cmd 持有新 agent 进程 → 任务长期 Running
→ 下次 Unregister/Register 静默失败 → 'restart 返回成功但没重启'。
改为唯一任务名(时间戳)+ 注册后显式校验 + 清理历史非运行任务
—— 真机验证连续 restart:PID 2612 -> 18556,任务名带时间戳
3. cmd shell 中文:定性为 Windows 固有局限(命令行参数在进程创建时已按 ANSI
解释,chcp 救不了参数本身),代码注释说明并引导用 PowerShell
真机:23 项能力全量在线;unit 26/26、e2e 21/21 全绿
This commit is contained in:
1 parent
111a7b5918
commit
313420a482
5 files changed
+455
-5
No files matched your search
@@ -3,6 +3,7 @@ import type { AgentConfig } from '../config.js';
|
||||
import { systemInfo, systemStatus, processList, serviceList, shellExec, auditList, agentRestart } from './system.js';
|
||||
import { appList, appInstall } from './app.js';
|
||||
import { screenInfo, screenCapture } from './screen.js';
|
||||
import { windowList, windowFocus, screenFind } from './window.js';
|
||||
import { mouseMove, mouseClick, mouseScroll, keyType, keyPress, setInputPolicy } from './input.js';
|
||||
import { fsList, fsStat, fsRead, fsWrite, setFsRoots } from './fs.js';
|
||||
|
||||
@@ -42,5 +43,9 @@ export function createCapabilityRegistry(cfg: AgentConfig): Record<string, Capab
|
||||
[CapabilityNames.FsWrite]: fsWrite,
|
||||
// v7 自持能力
|
||||
[CapabilityNames.AgentRestart]: agentRestart,
|
||||
// v8 GUI 语义(窗口与元素定位)
|
||||
[CapabilityNames.WindowList]: windowList,
|
||||
[CapabilityNames.WindowFocus]: windowFocus,
|
||||
[CapabilityNames.ScreenFind]: screenFind,
|
||||
};
|
||||
}
|
||||
@@ -284,8 +284,12 @@ export async function serviceList(args: Args): Promise<unknown> {
|
||||
const filter = args['filter'] as { name_pattern?: string; state?: string } | undefined;
|
||||
|
||||
const r = await execCommand({
|
||||
// 注意:Get-Service 的 Status/StartType 是 .NET 枚举,直接 ConvertTo-Json 会退化为
|
||||
// 数字(如 4=Running、1=Stopped),必须显式 ToString() 才能得到可读状态。
|
||||
command:
|
||||
'Get-Service | Select-Object Name,DisplayName,Status,StartType | ConvertTo-Json -Compress',
|
||||
'Get-Service | Select-Object Name,DisplayName,' +
|
||||
"@{n='Status';e={$_.Status.ToString()}},@{n='StartType';e={$_.StartType.ToString()}} | " +
|
||||
'ConvertTo-Json -Compress',
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
let services = toArray<{ Name?: string; DisplayName?: string; Status?: string | number; StartType?: string | number }>(
|
||||
@@ -394,14 +398,24 @@ async function scheduleWindowsRestart(p: RestartParams): Promise<unknown> {
|
||||
// 导致 agent 读不到配置而回退到默认端口 → EADDRINUSE。故此处不加空格。
|
||||
const cmdLine = '/c ' + (p.home ? `set NODEAGENT_HOME=${p.home}& ` : '') + steps.join(' & ');
|
||||
|
||||
const TASK = 'nodeagent-selfrestart';
|
||||
// ⚠️ 任务名必须唯一(带时间戳):重启任务的 cmd 会一直持有新 agent 进程,
|
||||
// 任务因此长期处于 Running 状态;若沿用固定名,下次 restart 的
|
||||
// Unregister/Register 会静默失败(任务无法覆盖正在运行的实例),
|
||||
// 导致「restart 返回成功但实际没有重启」(真机踩过)。
|
||||
const TASK = `nodeagent-selfrestart-${Date.now()}`;
|
||||
const ps = [
|
||||
`$ErrorActionPreference='SilentlyContinue'`,
|
||||
`Unregister-ScheduledTask -TaskName '${TASK}' -Confirm:$false`,
|
||||
// 清理历史 selfrestart 任务(保留 Running 的 —— 那正是当前 agent 的宿主)
|
||||
`Get-ScheduledTask -TaskName 'nodeagent-selfrestart*' | Where-Object { $_.State -ne 'Running' } | Unregister-ScheduledTask -Confirm:$false`,
|
||||
`$a = New-ScheduledTaskAction -Execute 'cmd.exe' -Argument '${cmdLine.replace(/'/g, "''")}'`,
|
||||
// 不限时(默认 72h 后强制结束),保证 node 能长期运行
|
||||
`$s = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -ExecutionTimeLimit (New-TimeSpan -Seconds 0)`,
|
||||
`Register-ScheduledTask -TaskName '${TASK}' -Action $a -Settings $s -Force | Out-Null`,
|
||||
// ⚠️ 必须用最高权限:agent 通常以管理员运行,普通权限的 taskkill 杀不掉它,
|
||||
// 否则会出现「新进程起来了但端口被占(EADDRINUSE)」。
|
||||
`$p = New-ScheduledTaskPrincipal -UserId $env:USERNAME -RunLevel Highest -LogonType Interactive`,
|
||||
`Register-ScheduledTask -TaskName '${TASK}' -Action $a -Settings $s -Principal $p -Force | Out-Null`,
|
||||
// 校验注册真的成功(SilentlyContinue 会吞错,必须显式确认)
|
||||
`if ((Get-ScheduledTask -TaskName '${TASK}' -ErrorAction SilentlyContinue) -eq $null) { Write-Output 'register-failed'; exit 1 }`,
|
||||
`Start-ScheduledTask -TaskName '${TASK}'`,
|
||||
`Write-Output 'ok'`,
|
||||
].join('; ');
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
import { CapabilityError, ErrorCodes } from '@nodeagent/protocol';
|
||||
import { IS_WINDOWS, execCommand } from '../util/exec.js';
|
||||
|
||||
type Args = Record<string, unknown>;
|
||||
|
||||
function toArray<T>(v: T | T[] | null | undefined): T[] {
|
||||
if (v === null || v === undefined) return [];
|
||||
return Array.isArray(v) ? v : [v];
|
||||
}
|
||||
|
||||
/**
|
||||
* 把 PowerShell 脚本编码为 UTF-16LE base64。
|
||||
* 用于承载含 C# 源码 / 大量引号的脚本,彻底规避转义问题。
|
||||
*/
|
||||
function encodePS(script: string): string {
|
||||
return Buffer.from(script, 'utf16le').toString('base64');
|
||||
}
|
||||
|
||||
async function runPS(script: string, timeoutMs = 30_000): Promise<string> {
|
||||
const r = await execCommand({
|
||||
command: `powershell.exe -NoProfile -NonInteractive -EncodedCommand ${encodePS(script)}`,
|
||||
timeoutMs,
|
||||
});
|
||||
if (r.exit_code !== 0) {
|
||||
throw new CapabilityError(ErrorCodes.EXECUTION_FAILED, '窗口操作失败', {
|
||||
detail: (r.stderr || r.stdout).slice(0, 600),
|
||||
});
|
||||
}
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从混杂输出中提取 JSON。
|
||||
*
|
||||
* 背景:PowerShell 的 `Add-Type -AssemblyName` 等语句可能向 stdout 夹杂额外内容,
|
||||
* 直接 JSON.parse 会报 "Unexpected non-whitespace character after JSON"。
|
||||
* 这里按首个 `[`/`{` 到末个 `]`/`}` 截取,稳健取回 JSON 主体。
|
||||
*/
|
||||
function extractJson<T>(raw: string): T | null {
|
||||
if (!raw) return null;
|
||||
const starts = [raw.indexOf('['), raw.indexOf('{')].filter((i) => i >= 0);
|
||||
if (starts.length === 0) return null;
|
||||
const start = Math.min(...starts);
|
||||
const end = Math.max(raw.lastIndexOf(']'), raw.lastIndexOf('}'));
|
||||
if (end <= start) return null;
|
||||
try {
|
||||
return JSON.parse(raw.slice(start, end + 1)) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Win32 窗口枚举与矩形读取(供 window.list / window.focus 复用)。 */
|
||||
const WIN32_PRELUDE = `
|
||||
Add-Type @"
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Runtime.InteropServices;
|
||||
public class NAWin32 {
|
||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
|
||||
public delegate bool EnumProc(IntPtr hWnd, IntPtr lParam);
|
||||
[DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr p);
|
||||
[DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
|
||||
[DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h);
|
||||
[DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, StringBuilder s, int n);
|
||||
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
|
||||
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
||||
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
||||
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
|
||||
}
|
||||
"@
|
||||
`;
|
||||
|
||||
/** 枚举可见顶层窗口 → JSON 字符串。 */
|
||||
const LIST_SCRIPT = `${WIN32_PRELUDE}
|
||||
$fg = [NAWin32]::GetForegroundWindow()
|
||||
$list = New-Object System.Collections.ArrayList
|
||||
$cb = [NAWin32+EnumProc]{
|
||||
param($h, $l)
|
||||
if (-not [NAWin32]::IsWindowVisible($h)) { return $true }
|
||||
$sb = New-Object System.Text.StringBuilder 512
|
||||
[void][NAWin32]::GetWindowTextW($h, $sb, 512)
|
||||
$title = $sb.ToString()
|
||||
if ([string]::IsNullOrWhiteSpace($title)) { return $true }
|
||||
$r = New-Object NAWin32+RECT
|
||||
if (-not [NAWin32]::GetWindowRect($h, [ref]$r)) { return $true }
|
||||
$w = $r.Right - $r.Left; $ht = $r.Bottom - $r.Top
|
||||
if ($w -le 0 -or $ht -le 0) { return $true }
|
||||
$pid = 0
|
||||
[void][NAWin32]::GetWindowThreadProcessId($h, [ref]$pid)
|
||||
$pn = ''
|
||||
try { $pn = (Get-Process -Id $pid -ErrorAction Stop).ProcessName } catch {}
|
||||
[void]$list.Add([pscustomobject]@{
|
||||
hwnd = ('0x{0:X}' -f $h.ToInt64()); title = $title; process = $pn; pid = $pid
|
||||
x = $r.Left; y = $r.Top; width = $w; height = $ht
|
||||
is_foreground = ($h -eq $fg); is_minimized = [NAWin32]::IsIconic($h)
|
||||
})
|
||||
return $true
|
||||
}
|
||||
[void][NAWin32]::EnumWindows($cb, [IntPtr]::Zero)
|
||||
$list | ConvertTo-Json -Compress
|
||||
`;
|
||||
|
||||
interface WinInfo {
|
||||
hwnd: string;
|
||||
title: string;
|
||||
process: string;
|
||||
pid: number;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
is_foreground: boolean;
|
||||
is_minimized: boolean;
|
||||
}
|
||||
|
||||
// ---------------- window.list ----------------
|
||||
|
||||
export async function windowList(args: Args): Promise<unknown> {
|
||||
if (!IS_WINDOWS) {
|
||||
throw new CapabilityError(
|
||||
ErrorCodes.UNSUPPORTED_PLATFORM,
|
||||
'window.list 仅在被控端为 Windows 时可用',
|
||||
{ platform: process.platform },
|
||||
);
|
||||
}
|
||||
const out = await runPS(LIST_SCRIPT, 40_000);
|
||||
let all: WinInfo[] = toArray<WinInfo>(extractJson<WinInfo[]>(out) ?? []);
|
||||
// 过滤掉 UWP 的隐形壳窗口(尺寸异常大且进程为 ApplicationFrameHost 的重复项保留)
|
||||
all = all.filter((w) => w.width > 40 && w.height > 40);
|
||||
|
||||
const pattern = args['title_pattern'] as string | undefined;
|
||||
if (pattern) {
|
||||
const re = new RegExp(pattern, 'i');
|
||||
all = all.filter((w) => re.test(w.title));
|
||||
}
|
||||
const limit = (args['limit'] as number | undefined) ?? 50;
|
||||
return { windows: all.slice(0, limit) };
|
||||
}
|
||||
|
||||
// ---------------- window.focus ----------------
|
||||
|
||||
export async function windowFocus(args: Args): Promise<unknown> {
|
||||
if (!IS_WINDOWS) {
|
||||
throw new CapabilityError(
|
||||
ErrorCodes.UNSUPPORTED_PLATFORM,
|
||||
'window.focus 仅在被控端为 Windows 时可用',
|
||||
{ platform: process.platform },
|
||||
);
|
||||
}
|
||||
const title = args['title'] as string | undefined;
|
||||
const hwnd = args['hwnd'] as string | undefined;
|
||||
if (!title && !hwnd) {
|
||||
throw new CapabilityError(ErrorCodes.PARAM_INVALID, '需提供 title(正则)或 hwnd');
|
||||
}
|
||||
|
||||
const script = `${WIN32_PRELUDE}
|
||||
$h = [IntPtr]::Zero
|
||||
$want = ${JSON.stringify(hwnd ?? '')}
|
||||
if ($want -ne '') { $h = [IntPtr][Convert]::ToInt64($want, 16) }
|
||||
else {
|
||||
$re = ${JSON.stringify(title ?? '')}
|
||||
$fg = [NAWin32]::GetForegroundWindow()
|
||||
$found = [IntPtr]::Zero
|
||||
$cb = [NAWin32+EnumProc]{
|
||||
param($w, $l)
|
||||
if ([NAWin32]::IsWindowVisible($w)) {
|
||||
$sb = New-Object System.Text.StringBuilder 512
|
||||
[void][NAWin32]::GetWindowTextW($w, $sb, 512)
|
||||
$t = $sb.ToString()
|
||||
if ($t -match $re) { $script:found = $w; return $false }
|
||||
}
|
||||
return $true
|
||||
}
|
||||
[void][NAWin32]::EnumWindows($cb, [IntPtr]::Zero)
|
||||
$h = $script:found
|
||||
}
|
||||
if ($h -eq [IntPtr]::Zero) { Write-Output '{"found":false}'; exit 0 }
|
||||
[void][NAWin32]::ShowWindow($h, 9) # SW_RESTORE
|
||||
[void][NAWin32]::SetForegroundWindow($h)
|
||||
Start-Sleep -Milliseconds 400
|
||||
$r = New-Object NAWin32+RECT
|
||||
[void][NAWin32]::GetWindowRect($h, [ref]$r)
|
||||
$sb2 = New-Object System.Text.StringBuilder 512
|
||||
[void][NAWin32]::GetWindowTextW($h, $sb2, 512)
|
||||
[pscustomobject]@{
|
||||
found = $true; hwnd = ('0x{0:X}' -f $h.ToInt64()); title = $sb2.ToString()
|
||||
x = $r.Left; y = $r.Top; width = ($r.Right - $r.Left); height = ($r.Bottom - $r.Top)
|
||||
focused = ([NAWin32]::GetForegroundWindow() -eq $h)
|
||||
} | ConvertTo-Json -Compress
|
||||
`;
|
||||
const out = await runPS(script, 40_000);
|
||||
const obj = extractJson<Record<string, unknown>>(out) ?? {};
|
||||
if (obj['found'] === false) {
|
||||
throw new CapabilityError(ErrorCodes.EXECUTION_FAILED, '未找到匹配的窗口', { title, hwnd });
|
||||
}
|
||||
delete obj['found'];
|
||||
return obj;
|
||||
}
|
||||
|
||||
// ---------------- screen.find ----------------
|
||||
|
||||
const FIND_SCRIPT_PRELUDE = `
|
||||
Add-Type -AssemblyName UIAutomationClient
|
||||
Add-Type -AssemblyName UIAutomationTypes
|
||||
Add-Type -AssemblyName WindowsBase
|
||||
function Get-WinByTitle([string]$re) {
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
$cond = New-Object System.Windows.Automation.PropertyCondition(
|
||||
[System.Windows.Automation.AutomationElement]::ControlTypeProperty,
|
||||
[System.Windows.Automation.ControlType]::Window)
|
||||
$wins = $root.FindAll([System.Windows.Automation.TreeScope]::Children, $cond)
|
||||
foreach ($w in $wins) {
|
||||
try {
|
||||
$t = $w.Current.Name
|
||||
if ($t -and $t -match $re) { return $w }
|
||||
} catch {}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* 在界面中查找 UI 元素并返回屏幕坐标(UI Automation)。
|
||||
*
|
||||
* 这是解决「看得到画面但读不懂界面」的关键能力:把元素名映射为可点击坐标。
|
||||
*/
|
||||
export async function screenFind(args: Args): Promise<unknown> {
|
||||
if (!IS_WINDOWS) {
|
||||
throw new CapabilityError(
|
||||
ErrorCodes.UNSUPPORTED_PLATFORM,
|
||||
'screen.find 仅在被控端为 Windows 时可用',
|
||||
{ platform: process.platform },
|
||||
);
|
||||
}
|
||||
const text = args['text'] as string;
|
||||
if (typeof text !== 'string' || text.length === 0) {
|
||||
throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空');
|
||||
}
|
||||
const windowTitle = args['window'] as string | undefined;
|
||||
const controlType = args['control_type'] as string | undefined;
|
||||
const limit = (args['limit'] as number | undefined) ?? 20;
|
||||
|
||||
const ctypeMap: Record<string, string> = {
|
||||
Button: 'Button',
|
||||
MenuItem: 'MenuItem',
|
||||
Edit: 'Edit',
|
||||
Text: 'Text',
|
||||
ListItem: 'ListItem',
|
||||
CheckBox: 'CheckBox',
|
||||
RadioButton: 'RadioButton',
|
||||
ComboBox: 'ComboBox',
|
||||
Tab: 'TabItem',
|
||||
TreeItem: 'TreeItem',
|
||||
Hyperlink: 'Hyperlink',
|
||||
Image: 'Image',
|
||||
};
|
||||
const ct = controlType ? ctypeMap[controlType] ?? controlType : '';
|
||||
|
||||
const script = `${FIND_SCRIPT_PRELUDE}
|
||||
$text = ${JSON.stringify(text)}
|
||||
$limit = ${Number(limit)}
|
||||
$scope = [System.Windows.Automation.AutomationElement]::RootElement
|
||||
$winTitle = ${JSON.stringify(windowTitle ?? '')}
|
||||
if ($winTitle -ne '') {
|
||||
$w = Get-WinByTitle $winTitle
|
||||
if ($w -eq $null) { Write-Output '[]'; exit 0 }
|
||||
$scope = $w
|
||||
}
|
||||
$nameCond = New-Object System.Windows.Automation.PropertyCondition(
|
||||
[System.Windows.Automation.AutomationElement]::NameProperty, $text,
|
||||
[System.Windows.Automation.PropertyConditionFlags]::IgnoreCase)
|
||||
$ct = ${JSON.stringify(ct)}
|
||||
if ($ct -ne '') {
|
||||
$ctObj = $null
|
||||
try { $ctObj = [System.Windows.Automation.ControlType]::$ct } catch {}
|
||||
if ($ctObj -ne $null) {
|
||||
$ctCond = New-Object System.Windows.Automation.PropertyCondition(
|
||||
[System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ctObj)
|
||||
$cond = New-Object System.Windows.Automation.AndCondition($nameCond, $ctCond)
|
||||
} else { $cond = $nameCond }
|
||||
} else { $cond = $nameCond }
|
||||
$found = $scope.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond)
|
||||
$out = New-Object System.Collections.ArrayList
|
||||
foreach ($e in $found) {
|
||||
if ($out.Count -ge $limit) { break }
|
||||
try {
|
||||
$r = $e.Current.BoundingRectangle
|
||||
if ($r.Width -le 0 -or $r.Height -le 0) { continue }
|
||||
$wt = ''
|
||||
try {
|
||||
$tw = $e.Current.ControlType.ProgrammaticName
|
||||
$wt = $tw
|
||||
} catch {}
|
||||
$out.Add([pscustomobject]@{
|
||||
name = $e.Current.Name
|
||||
control_type = ($e.Current.ControlType.ProgrammaticName -replace 'ControlType\\.','')
|
||||
automation_id = $e.Current.AutomationId
|
||||
x = [int]($r.Left + $r.Width / 2)
|
||||
y = [int]($r.Top + $r.Height / 2)
|
||||
left = [int]$r.Left; top = [int]$r.Top
|
||||
width = [int]$r.Width; height = [int]$r.Height
|
||||
})
|
||||
} catch {}
|
||||
}
|
||||
$out | ConvertTo-Json -Compress
|
||||
`;
|
||||
const out = await runPS(script, 60_000);
|
||||
const matches = toArray<Record<string, unknown>>(extractJson<Record<string, unknown>[]>(out) ?? []);
|
||||
return { matches: matches.slice(0, limit) };
|
||||
}
|
||||
@@ -44,7 +44,12 @@ const PS_UTF8_PREFIX =
|
||||
function buildLaunch(command: string, shell: ShellKind): Launch {
|
||||
if (IS_WINDOWS) {
|
||||
if (shell === 'cmd') {
|
||||
// cmd 的代码页需在命令内切换(chcp 65001),且须静默
|
||||
// cmd 的代码页需在命令内切换(chcp 65001),且须静默。
|
||||
//
|
||||
// ⚠️ 已知局限(实测确认):`chcp 65001` 只能改变**输出**代码页,
|
||||
// 而 Node 传给 cmd.exe 的**命令行参数**在进入 cmd 时已按 ANSI(中文为 GBK)
|
||||
// 解释,因此**命令字符串中的中文仍会损坏**(chcp 显示 65001 也无济于事)。
|
||||
// 需要在命令里使用中文时,请改用默认的 powershell(那边已完整支持 UTF-8)。
|
||||
return { file: 'cmd.exe', args: ['/d', '/s', '/c', `chcp 65001 >nul & ${command}`] };
|
||||
}
|
||||
return {
|
||||
|
||||
@@ -26,6 +26,10 @@ export const CapabilityNames = {
|
||||
FsWrite: 'fs.write',
|
||||
// v7 自持能力(受控重启)
|
||||
AgentRestart: 'system.agent.restart',
|
||||
// v8 GUI 语义(窗口与元素定位)
|
||||
WindowList: 'window.list',
|
||||
WindowFocus: 'window.focus',
|
||||
ScreenFind: 'screen.find',
|
||||
} as const;
|
||||
|
||||
export type CapabilityName = (typeof CapabilityNames)[keyof typeof CapabilityNames];
|
||||
@@ -618,6 +622,116 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: CapabilityNames.WindowList,
|
||||
version: '1.0',
|
||||
description:
|
||||
'列出当前可见的顶层窗口(标题、所属进程、精确矩形、是否前台窗口)。' +
|
||||
'配合 screen.capture 可在**已知窗口范围内**相对定位,避免全屏猜坐标。',
|
||||
risk: 'low',
|
||||
params_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title_pattern: { type: 'string', description: '可选:按标题正则过滤' },
|
||||
limit: { type: 'integer', default: 50 },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
returns_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
windows: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
hwnd: { type: 'string', description: '窗口句柄(十六进制)' },
|
||||
title: { type: 'string' },
|
||||
process: { type: 'string' },
|
||||
pid: { type: 'integer' },
|
||||
x: { type: 'integer' },
|
||||
y: { type: 'integer' },
|
||||
width: { type: 'integer' },
|
||||
height: { type: 'integer' },
|
||||
is_foreground: { type: 'boolean' },
|
||||
is_minimized: { type: 'boolean' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: CapabilityNames.WindowFocus,
|
||||
version: '1.0',
|
||||
description: '将指定窗口(按标题正则或句柄)置前并聚焦,返回其精确矩形。',
|
||||
risk: 'medium',
|
||||
params_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: { type: 'string', description: '窗口标题(正则,匹配第一个)' },
|
||||
hwnd: { type: 'string', description: '或直接给窗口句柄' },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
returns_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
hwnd: { type: 'string' },
|
||||
title: { type: 'string' },
|
||||
x: { type: 'integer' },
|
||||
y: { type: 'integer' },
|
||||
width: { type: 'integer' },
|
||||
height: { type: 'integer' },
|
||||
focused: { type: 'boolean' },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: CapabilityNames.ScreenFind,
|
||||
version: '1.0',
|
||||
description:
|
||||
'在界面中查找 UI 元素并返回其**屏幕坐标**(基于 Windows UI Automation)。' +
|
||||
'可指定窗口范围与元素类型,返回中心点坐标,直接喂给 input.mouse.click。',
|
||||
risk: 'low',
|
||||
params_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
text: { type: 'string', description: '要查找的文本(匹配元素 Name,支持子串)' },
|
||||
window: { type: 'string', description: '可选:限定在该窗口标题(正则)内查找' },
|
||||
control_type: {
|
||||
type: 'string',
|
||||
description: '可选:限定控件类型,如 Button / MenuItem / Edit / Text / ListItem',
|
||||
},
|
||||
limit: { type: 'integer', default: 20 },
|
||||
},
|
||||
required: ['text'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
returns_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
matches: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
control_type: { type: 'string' },
|
||||
automation_id: { type: 'string' },
|
||||
x: { type: 'integer', description: '中心点 X(屏幕坐标)' },
|
||||
y: { type: 'integer', description: '中心点 Y(屏幕坐标)' },
|
||||
left: { type: 'integer' },
|
||||
top: { type: 'integer' },
|
||||
width: { type: 'integer' },
|
||||
height: { type: 'integer' },
|
||||
window_title: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/** 按名称查找能力描述。 */
|
||||
|
||||
Reference in new issue
Block a user