diff --git a/apps/agent/src/capabilities/index.ts b/apps/agent/src/capabilities/index.ts index d267813..cb54e0e 100644 --- a/apps/agent/src/capabilities/index.ts +++ b/apps/agent/src/capabilities/index.ts @@ -3,6 +3,7 @@ import type { AgentConfig } from '../config.js'; import { systemInfo, systemStatus, processList, serviceList, shellExec, auditList, agentRestart } from './system.js'; import { appList, appInstall } from './app.js'; import { screenInfo, screenCapture } from './screen.js'; +import { windowList, windowFocus, screenFind } from './window.js'; import { mouseMove, mouseClick, mouseScroll, keyType, keyPress, setInputPolicy } from './input.js'; import { fsList, fsStat, fsRead, fsWrite, setFsRoots } from './fs.js'; @@ -42,5 +43,9 @@ export function createCapabilityRegistry(cfg: AgentConfig): Record { const filter = args['filter'] as { name_pattern?: string; state?: string } | undefined; const r = await execCommand({ + // 注意：Get-Service 的 Status/StartType 是 .NET 枚举，直接 ConvertTo-Json 会退化为 + // 数字（如 4=Running、1=Stopped），必须显式 ToString() 才能得到可读状态。 command: - 'Get-Service | Select-Object Name,DisplayName,Status,StartType | ConvertTo-Json -Compress', + 'Get-Service | Select-Object Name,DisplayName,' + + "@{n='Status';e={$_.Status.ToString()}},@{n='StartType';e={$_.StartType.ToString()}} | " + + 'ConvertTo-Json -Compress', timeoutMs: 30_000, }); let services = toArray<{ Name?: string; DisplayName?: string; Status?: string | number; StartType?: string | number }>( @@ -394,14 +398,24 @@ async function scheduleWindowsRestart(p: RestartParams): Promise { // 导致 agent 读不到配置而回退到默认端口 → EADDRINUSE。故此处不加空格。 const cmdLine = '/c ' + (p.home ? `set NODEAGENT_HOME=${p.home}& ` : '') + steps.join(' & '); - const TASK = 'nodeagent-selfrestart'; + // ⚠️ 任务名必须唯一（带时间戳）：重启任务的 cmd 会一直持有新 agent 进程， + // 任务因此长期处于 Running 状态；若沿用固定名，下次 restart 的 + // Unregister/Register 会静默失败（任务无法覆盖正在运行的实例）， + // 导致「restart 返回成功但实际没有重启」（真机踩过）。 + const TASK = `nodeagent-selfrestart-${Date.now()}`; const ps = [ `$ErrorActionPreference='SilentlyContinue'`, - `Unregister-ScheduledTask -TaskName '${TASK}' -Confirm:$false`, + // 清理历史 selfrestart 任务（保留 Running 的 —— 那正是当前 agent 的宿主） + `Get-ScheduledTask -TaskName 'nodeagent-selfrestart*' | Where-Object { $_.State -ne 'Running' } | Unregister-ScheduledTask -Confirm:$false`, `$a = New-ScheduledTaskAction -Execute 'cmd.exe' -Argument '${cmdLine.replace(/'/g, "''")}'`, // 不限时（默认 72h 后强制结束），保证 node 能长期运行 `$s = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -ExecutionTimeLimit (New-TimeSpan -Seconds 0)`, - `Register-ScheduledTask -TaskName '${TASK}' -Action $a -Settings $s -Force | Out-Null`, + // ⚠️ 必须用最高权限：agent 通常以管理员运行，普通权限的 taskkill 杀不掉它， + // 否则会出现「新进程起来了但端口被占（EADDRINUSE）」。 + `$p = New-ScheduledTaskPrincipal -UserId $env:USERNAME -RunLevel Highest -LogonType Interactive`, + `Register-ScheduledTask -TaskName '${TASK}' -Action $a -Settings $s -Principal $p -Force | Out-Null`, + // 校验注册真的成功（SilentlyContinue 会吞错，必须显式确认） + `if ((Get-ScheduledTask -TaskName '${TASK}' -ErrorAction SilentlyContinue) -eq $null) { Write-Output 'register-failed'; exit 1 }`, `Start-ScheduledTask -TaskName '${TASK}'`, `Write-Output 'ok'`, ].join('; '); diff --git a/apps/agent/src/capabilities/window.ts b/apps/agent/src/capabilities/window.ts new file mode 100644 index 0000000..b4384c2 --- /dev/null +++ b/apps/agent/src/capabilities/window.ts @@ -0,0 +1,312 @@ +import { CapabilityError, ErrorCodes } from '@nodeagent/protocol'; +import { IS_WINDOWS, execCommand } from '../util/exec.js'; + +type Args = Record; + +function toArray(v: T | T[] | null | undefined): T[] { + if (v === null || v === undefined) return []; + return Array.isArray(v) ? v : [v]; +} + +/** + * 把 PowerShell 脚本编码为 UTF-16LE base64。 + * 用于承载含 C# 源码 / 大量引号的脚本，彻底规避转义问题。 + */ +function encodePS(script: string): string { + return Buffer.from(script, 'utf16le').toString('base64'); +} + +async function runPS(script: string, timeoutMs = 30_000): Promise { + const r = await execCommand({ + command: `powershell.exe -NoProfile -NonInteractive -EncodedCommand ${encodePS(script)}`, + timeoutMs, + }); + if (r.exit_code !== 0) { + throw new CapabilityError(ErrorCodes.EXECUTION_FAILED, '窗口操作失败', { + detail: (r.stderr || r.stdout).slice(0, 600), + }); + } + return r.stdout.trim(); +} + +/** + * 从混杂输出中提取 JSON。 + * + * 背景：PowerShell 的 `Add-Type -AssemblyName` 等语句可能向 stdout 夹杂额外内容， + * 直接 JSON.parse 会报 "Unexpected non-whitespace character after JSON"。 + * 这里按首个 `[`/`{` 到末个 `]`/`}` 截取，稳健取回 JSON 主体。 + */ +function extractJson(raw: string): T | null { + if (!raw) return null; + const starts = [raw.indexOf('['), raw.indexOf('{')].filter((i) => i >= 0); + if (starts.length === 0) return null; + const start = Math.min(...starts); + const end = Math.max(raw.lastIndexOf(']'), raw.lastIndexOf('}')); + if (end <= start) return null; + try { + return JSON.parse(raw.slice(start, end + 1)) as T; + } catch { + return null; + } +} + +/** Win32 窗口枚举与矩形读取（供 window.list / window.focus 复用）。 */ +const WIN32_PRELUDE = ` +Add-Type @" +using System; +using System.Text; +using System.Runtime.InteropServices; +public class NAWin32 { + [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; } + public delegate bool EnumProc(IntPtr hWnd, IntPtr lParam); + [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr p); + [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h); + [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h); + [DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, StringBuilder s, int n); + [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h); + [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd); + [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid); +} +"@ +`; + +/** 枚举可见顶层窗口 → JSON 字符串。 */ +const LIST_SCRIPT = `${WIN32_PRELUDE} +$fg = [NAWin32]::GetForegroundWindow() +$list = New-Object System.Collections.ArrayList +$cb = [NAWin32+EnumProc]{ + param($h, $l) + if (-not [NAWin32]::IsWindowVisible($h)) { return $true } + $sb = New-Object System.Text.StringBuilder 512 + [void][NAWin32]::GetWindowTextW($h, $sb, 512) + $title = $sb.ToString() + if ([string]::IsNullOrWhiteSpace($title)) { return $true } + $r = New-Object NAWin32+RECT + if (-not [NAWin32]::GetWindowRect($h, [ref]$r)) { return $true } + $w = $r.Right - $r.Left; $ht = $r.Bottom - $r.Top + if ($w -le 0 -or $ht -le 0) { return $true } + $pid = 0 + [void][NAWin32]::GetWindowThreadProcessId($h, [ref]$pid) + $pn = '' + try { $pn = (Get-Process -Id $pid -ErrorAction Stop).ProcessName } catch {} + [void]$list.Add([pscustomobject]@{ + hwnd = ('0x{0:X}' -f $h.ToInt64()); title = $title; process = $pn; pid = $pid + x = $r.Left; y = $r.Top; width = $w; height = $ht + is_foreground = ($h -eq $fg); is_minimized = [NAWin32]::IsIconic($h) + }) + return $true +} +[void][NAWin32]::EnumWindows($cb, [IntPtr]::Zero) +$list | ConvertTo-Json -Compress +`; + +interface WinInfo { + hwnd: string; + title: string; + process: string; + pid: number; + x: number; + y: number; + width: number; + height: number; + is_foreground: boolean; + is_minimized: boolean; +} + +// ---------------- window.list ---------------- + +export async function windowList(args: Args): Promise { + if (!IS_WINDOWS) { + throw new CapabilityError( + ErrorCodes.UNSUPPORTED_PLATFORM, + 'window.list 仅在被控端为 Windows 时可用', + { platform: process.platform }, + ); + } + const out = await runPS(LIST_SCRIPT, 40_000); + let all: WinInfo[] = toArray(extractJson(out) ?? []); + // 过滤掉 UWP 的隐形壳窗口（尺寸异常大且进程为 ApplicationFrameHost 的重复项保留） + all = all.filter((w) => w.width > 40 && w.height > 40); + + const pattern = args['title_pattern'] as string | undefined; + if (pattern) { + const re = new RegExp(pattern, 'i'); + all = all.filter((w) => re.test(w.title)); + } + const limit = (args['limit'] as number | undefined) ?? 50; + return { windows: all.slice(0, limit) }; +} + +// ---------------- window.focus ---------------- + +export async function windowFocus(args: Args): Promise { + if (!IS_WINDOWS) { + throw new CapabilityError( + ErrorCodes.UNSUPPORTED_PLATFORM, + 'window.focus 仅在被控端为 Windows 时可用', + { platform: process.platform }, + ); + } + const title = args['title'] as string | undefined; + const hwnd = args['hwnd'] as string | undefined; + if (!title && !hwnd) { + throw new CapabilityError(ErrorCodes.PARAM_INVALID, '需提供 title（正则）或 hwnd'); + } + + const script = `${WIN32_PRELUDE} +$h = [IntPtr]::Zero +$want = ${JSON.stringify(hwnd ?? '')} +if ($want -ne '') { $h = [IntPtr][Convert]::ToInt64($want, 16) } +else { + $re = ${JSON.stringify(title ?? '')} + $fg = [NAWin32]::GetForegroundWindow() + $found = [IntPtr]::Zero + $cb = [NAWin32+EnumProc]{ + param($w, $l) + if ([NAWin32]::IsWindowVisible($w)) { + $sb = New-Object System.Text.StringBuilder 512 + [void][NAWin32]::GetWindowTextW($w, $sb, 512) + $t = $sb.ToString() + if ($t -match $re) { $script:found = $w; return $false } + } + return $true + } + [void][NAWin32]::EnumWindows($cb, [IntPtr]::Zero) + $h = $script:found +} +if ($h -eq [IntPtr]::Zero) { Write-Output '{"found":false}'; exit 0 } +[void][NAWin32]::ShowWindow($h, 9) # SW_RESTORE +[void][NAWin32]::SetForegroundWindow($h) +Start-Sleep -Milliseconds 400 +$r = New-Object NAWin32+RECT +[void][NAWin32]::GetWindowRect($h, [ref]$r) +$sb2 = New-Object System.Text.StringBuilder 512 +[void][NAWin32]::GetWindowTextW($h, $sb2, 512) +[pscustomobject]@{ + found = $true; hwnd = ('0x{0:X}' -f $h.ToInt64()); title = $sb2.ToString() + x = $r.Left; y = $r.Top; width = ($r.Right - $r.Left); height = ($r.Bottom - $r.Top) + focused = ([NAWin32]::GetForegroundWindow() -eq $h) +} | ConvertTo-Json -Compress +`; + const out = await runPS(script, 40_000); + const obj = extractJson>(out) ?? {}; + if (obj['found'] === false) { + throw new CapabilityError(ErrorCodes.EXECUTION_FAILED, '未找到匹配的窗口', { title, hwnd }); + } + delete obj['found']; + return obj; +} + +// ---------------- screen.find ---------------- + +const FIND_SCRIPT_PRELUDE = ` +Add-Type -AssemblyName UIAutomationClient +Add-Type -AssemblyName UIAutomationTypes +Add-Type -AssemblyName WindowsBase +function Get-WinByTitle([string]$re) { + $root = [System.Windows.Automation.AutomationElement]::RootElement + $cond = New-Object System.Windows.Automation.PropertyCondition( + [System.Windows.Automation.AutomationElement]::ControlTypeProperty, + [System.Windows.Automation.ControlType]::Window) + $wins = $root.FindAll([System.Windows.Automation.TreeScope]::Children, $cond) + foreach ($w in $wins) { + try { + $t = $w.Current.Name + if ($t -and $t -match $re) { return $w } + } catch {} + } + return $null +} +`; + +/** + * 在界面中查找 UI 元素并返回屏幕坐标（UI Automation）。 + * + * 这是解决「看得到画面但读不懂界面」的关键能力：把元素名映射为可点击坐标。 + */ +export async function screenFind(args: Args): Promise { + if (!IS_WINDOWS) { + throw new CapabilityError( + ErrorCodes.UNSUPPORTED_PLATFORM, + 'screen.find 仅在被控端为 Windows 时可用', + { platform: process.platform }, + ); + } + const text = args['text'] as string; + if (typeof text !== 'string' || text.length === 0) { + throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空'); + } + const windowTitle = args['window'] as string | undefined; + const controlType = args['control_type'] as string | undefined; + const limit = (args['limit'] as number | undefined) ?? 20; + + const ctypeMap: Record = { + Button: 'Button', + MenuItem: 'MenuItem', + Edit: 'Edit', + Text: 'Text', + ListItem: 'ListItem', + CheckBox: 'CheckBox', + RadioButton: 'RadioButton', + ComboBox: 'ComboBox', + Tab: 'TabItem', + TreeItem: 'TreeItem', + Hyperlink: 'Hyperlink', + Image: 'Image', + }; + const ct = controlType ? ctypeMap[controlType] ?? controlType : ''; + + const script = `${FIND_SCRIPT_PRELUDE} +$text = ${JSON.stringify(text)} +$limit = ${Number(limit)} +$scope = [System.Windows.Automation.AutomationElement]::RootElement +$winTitle = ${JSON.stringify(windowTitle ?? '')} +if ($winTitle -ne '') { + $w = Get-WinByTitle $winTitle + if ($w -eq $null) { Write-Output '[]'; exit 0 } + $scope = $w +} +$nameCond = New-Object System.Windows.Automation.PropertyCondition( + [System.Windows.Automation.AutomationElement]::NameProperty, $text, + [System.Windows.Automation.PropertyConditionFlags]::IgnoreCase) +$ct = ${JSON.stringify(ct)} +if ($ct -ne '') { + $ctObj = $null + try { $ctObj = [System.Windows.Automation.ControlType]::$ct } catch {} + if ($ctObj -ne $null) { + $ctCond = New-Object System.Windows.Automation.PropertyCondition( + [System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ctObj) + $cond = New-Object System.Windows.Automation.AndCondition($nameCond, $ctCond) + } else { $cond = $nameCond } +} else { $cond = $nameCond } +$found = $scope.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond) +$out = New-Object System.Collections.ArrayList +foreach ($e in $found) { + if ($out.Count -ge $limit) { break } + try { + $r = $e.Current.BoundingRectangle + if ($r.Width -le 0 -or $r.Height -le 0) { continue } + $wt = '' + try { + $tw = $e.Current.ControlType.ProgrammaticName + $wt = $tw + } catch {} + $out.Add([pscustomobject]@{ + name = $e.Current.Name + control_type = ($e.Current.ControlType.ProgrammaticName -replace 'ControlType\\.','') + automation_id = $e.Current.AutomationId + x = [int]($r.Left + $r.Width / 2) + y = [int]($r.Top + $r.Height / 2) + left = [int]$r.Left; top = [int]$r.Top + width = [int]$r.Width; height = [int]$r.Height + }) + } catch {} +} +$out | ConvertTo-Json -Compress +`; + const out = await runPS(script, 60_000); + const matches = toArray>(extractJson[]>(out) ?? []); + return { matches: matches.slice(0, limit) }; +} diff --git a/apps/agent/src/util/exec.ts b/apps/agent/src/util/exec.ts index 24b0dc5..fa3c3a5 100644 --- a/apps/agent/src/util/exec.ts +++ b/apps/agent/src/util/exec.ts @@ -44,7 +44,12 @@ const PS_UTF8_PREFIX = function buildLaunch(command: string, shell: ShellKind): Launch { if (IS_WINDOWS) { if (shell === 'cmd') { - // cmd 的代码页需在命令内切换（chcp 65001），且须静默 + // cmd 的代码页需在命令内切换（chcp 65001），且须静默。 + // + // ⚠️ 已知局限（实测确认）：`chcp 65001` 只能改变**输出**代码页， + // 而 Node 传给 cmd.exe 的**命令行参数**在进入 cmd 时已按 ANSI（中文为 GBK） + // 解释，因此**命令字符串中的中文仍会损坏**（chcp 显示 65001 也无济于事）。 + // 需要在命令里使用中文时，请改用默认的 powershell（那边已完整支持 UTF-8）。 return { file: 'cmd.exe', args: ['/d', '/s', '/c', `chcp 65001 >nul & ${command}`] }; } return { diff --git a/packages/protocol/src/capabilities.ts b/packages/protocol/src/capabilities.ts index dd18f9a..ad30ef3 100644 --- a/packages/protocol/src/capabilities.ts +++ b/packages/protocol/src/capabilities.ts @@ -26,6 +26,10 @@ export const CapabilityNames = { FsWrite: 'fs.write', // v7 自持能力（受控重启） AgentRestart: 'system.agent.restart', + // v8 GUI 语义（窗口与元素定位） + WindowList: 'window.list', + WindowFocus: 'window.focus', + ScreenFind: 'screen.find', } as const; export type CapabilityName = (typeof CapabilityNames)[keyof typeof CapabilityNames]; @@ -618,6 +622,116 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ }, }, }, + { + name: CapabilityNames.WindowList, + version: '1.0', + description: + '列出当前可见的顶层窗口（标题、所属进程、精确矩形、是否前台窗口）。' + + '配合 screen.capture 可在**已知窗口范围内**相对定位，避免全屏猜坐标。', + risk: 'low', + params_schema: { + type: 'object', + properties: { + title_pattern: { type: 'string', description: '可选：按标题正则过滤' }, + limit: { type: 'integer', default: 50 }, + }, + additionalProperties: false, + }, + returns_schema: { + type: 'object', + properties: { + windows: { + type: 'array', + items: { + type: 'object', + properties: { + hwnd: { type: 'string', description: '窗口句柄（十六进制）' }, + title: { type: 'string' }, + process: { type: 'string' }, + pid: { type: 'integer' }, + x: { type: 'integer' }, + y: { type: 'integer' }, + width: { type: 'integer' }, + height: { type: 'integer' }, + is_foreground: { type: 'boolean' }, + is_minimized: { type: 'boolean' }, + }, + }, + }, + }, + }, + }, + { + name: CapabilityNames.WindowFocus, + version: '1.0', + description: '将指定窗口（按标题正则或句柄）置前并聚焦，返回其精确矩形。', + risk: 'medium', + params_schema: { + type: 'object', + properties: { + title: { type: 'string', description: '窗口标题（正则，匹配第一个）' }, + hwnd: { type: 'string', description: '或直接给窗口句柄' }, + }, + additionalProperties: false, + }, + returns_schema: { + type: 'object', + properties: { + hwnd: { type: 'string' }, + title: { type: 'string' }, + x: { type: 'integer' }, + y: { type: 'integer' }, + width: { type: 'integer' }, + height: { type: 'integer' }, + focused: { type: 'boolean' }, + }, + }, + }, + { + name: CapabilityNames.ScreenFind, + version: '1.0', + description: + '在界面中查找 UI 元素并返回其**屏幕坐标**（基于 Windows UI Automation）。' + + '可指定窗口范围与元素类型，返回中心点坐标，直接喂给 input.mouse.click。', + risk: 'low', + params_schema: { + type: 'object', + properties: { + text: { type: 'string', description: '要查找的文本（匹配元素 Name，支持子串）' }, + window: { type: 'string', description: '可选：限定在该窗口标题（正则）内查找' }, + control_type: { + type: 'string', + description: '可选：限定控件类型，如 Button / MenuItem / Edit / Text / ListItem', + }, + limit: { type: 'integer', default: 20 }, + }, + required: ['text'], + additionalProperties: false, + }, + returns_schema: { + type: 'object', + properties: { + matches: { + type: 'array', + items: { + type: 'object', + properties: { + name: { type: 'string' }, + control_type: { type: 'string' }, + automation_id: { type: 'string' }, + x: { type: 'integer', description: '中心点 X（屏幕坐标）' }, + y: { type: 'integer', description: '中心点 Y（屏幕坐标）' }, + left: { type: 'integer' }, + top: { type: 'integer' }, + width: { type: 'integer' }, + height: { type: 'integer' }, + window_title: { type: 'string' }, + }, + }, + }, + }, + }, + }, ]; /** 按名称查找能力描述。 */