From a892b113ca992f08dac5a5d8b225116fd4f57c4c Mon Sep 17 00:00:00 2001 From: jkooe <62494645+jkooe@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:43:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(v12.2):=20GUI=20=E7=AD=89=E5=BE=85/?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E8=AF=AD=E4=B9=89=20=E2=80=94=E2=80=94=20?= =?UTF-8?q?=E7=BB=93=E6=9D=9F=E3=80=8C=E7=82=B9=E4=B8=8B=E5=8E=BB=E5=B0=B1?= =?UTF-8?q?=E4=B8=8D=E7=AE=A1=E3=80=8D=E7=9A=84=E5=81=87=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 动机 GUI 自动化最常见的失败不是「找不到」，而是「还没出来」：应用启动、页面切换、动画 都有延迟，一次性查找必然假失败。之前调用方只能自己在外面 sleep+重试（浪费往返）。 ## 实现（agent 侧内联轮询，避免多次往返） - screen.find 新增 wait_ms（0~30000，默认 0）与 interval_ms（100~2000，默认 400）： 轮询到命中或超时；无论成败都返回 waited_ms，让调用方知道是否命中等待窗口 - window.focus 新增 wait_ms：窗口未出现时每 500ms 重试，直到超时（仅 title 模式） - 宏步骤 find/assert 透传 wait_ms/interval_ms（写宏时可直接声明等待） - MCP na_screen_find 暴露 wait_ms ## 真机验证（决定性） 调用 window.focus(title=Notepad, wait_ms=15000) 时**记事本尚未运行**； 4 秒后才启动 -> 等待轮询捕获并成功聚焦： { title: "*install.ps1 - Notepad", focused: true, activated_by: "click" } 注意 activated_by=click：本次 SetForegroundWindow 又被系统静默否决， 点击兜底自动接管 —— 两个加固协同生效（等待 + 激活兜底）。 单测 89/89、e2e 21/21 全绿 --- apps/agent/src/capabilities/window.ts | 60 +++++++++++++++++++++------ apps/mcp/src/index.ts | 5 +++ packages/client/src/macro.ts | 16 +++++-- packages/protocol/src/capabilities.ts | 16 +++++++ 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/apps/agent/src/capabilities/window.ts b/apps/agent/src/capabilities/window.ts index 09fe3bf..dc20c0b 100644 --- a/apps/agent/src/capabilities/window.ts +++ b/apps/agent/src/capabilities/window.ts @@ -160,7 +160,29 @@ export async function windowFocus(args: Args): Promise { if (!title && !hwnd) { throw new CapabilityError(ErrorCodes.PARAM_INVALID, '需提供 title（正则）或 hwnd'); } + // v12.2：等窗口出现（应用启动有延迟，一次性查找极易假失败） + const waitMs = Math.max(0, Math.min(30_000, (args['wait_ms'] as number | undefined) ?? 0)); + if (waitMs > 0 && !hwnd) { + const t0 = Date.now(); + let lastErr: unknown = null; + while (Date.now() - t0 < waitMs) { + try { + return await focusOnce(title, hwnd); + } catch (err) { + lastErr = err; + await new Promise((r) => setTimeout(r, 500)); + } + } + throw lastErr instanceof Error + ? lastErr + : new CapabilityError(ErrorCodes.EXECUTION_FAILED, `等待 ${waitMs}ms 仍未找到窗口`, { title }); + } + return focusOnce(title, hwnd); +} + +/** 单次聚焦（供等待重试复用）。 */ +async function focusOnce(title: string | undefined, hwnd: string | undefined): Promise { const script = `${WIN32_PRELUDE} $h = [IntPtr]::Zero $want = ${JSON.stringify(hwnd ?? '')} @@ -271,6 +293,10 @@ export async function screenFind(args: Args): Promise { const controlType = args['control_type'] as string | undefined; const limit = (args['limit'] as number | undefined) ?? 20; const method = (args['method'] as string | undefined) ?? 'auto'; // auto | uia | ocr + // v12.2：等待语义 —— 界面常有动画/加载延迟，一次性查找极易假失败。 + // wait_ms > 0 时轮询直到命中或超时（返回 waited_ms 让调用方知情）。 + const waitMs = Math.max(0, Math.min(30_000, (args['wait_ms'] as number | undefined) ?? 0)); + const intervalMs = Math.max(100, Math.min(2000, (args['interval_ms'] as number | undefined) ?? 400)); // ---------- UIA 引擎 ---------- const uiaFind = async (): Promise[]> => { @@ -488,17 +514,27 @@ $out | ConvertTo-Json -Compress return toArray>(extractJson[]>(out) ?? []); }; - if (method === 'uia') { - return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' }; + /** 单次尝试（按 method 选择引擎）。 */ + const attempt = async (): Promise<{ matches: Record[]; engine: string }> => { + if (method === 'uia') return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' }; + if (method === 'ocr') return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' }; + // auto：UIA 优先（快且带控件语义），找不到再 OCR 兜底（自绘 UI） + const uiaMatches = await uiaFind(); + if (uiaMatches.length > 0) return { matches: uiaMatches.slice(0, limit), engine: 'uia' }; + const ocrMatches = await ocrFind(); + return { matches: ocrMatches.slice(0, limit), engine: 'ocr' }; + }; + + const startedAt = Date.now(); + let last = await attempt(); + while (last.matches.length === 0 && Date.now() - startedAt < waitMs) { + await new Promise((r) => setTimeout(r, intervalMs)); + last = await attempt(); } - if (method === 'ocr') { - return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' }; - } - // auto：UIA 优先（快且带控件语义），找不到再 OCR 兜底（自绘 UI） - const uiaMatches = await uiaFind(); - if (uiaMatches.length > 0) { - return { matches: uiaMatches.slice(0, limit), engine: 'uia' }; - } - const ocrMatches = await ocrFind(); - return { matches: ocrMatches.slice(0, limit), engine: 'ocr' }; + return { + matches: last.matches, + engine: last.engine, + waited_ms: Date.now() - startedAt, + ...(waitMs > 0 ? { wait_ms: waitMs } : {}), + }; } diff --git a/apps/mcp/src/index.ts b/apps/mcp/src/index.ts index 3d3b50a..3b37ad2 100644 --- a/apps/mcp/src/index.ts +++ b/apps/mcp/src/index.ts @@ -312,6 +312,7 @@ const TOOLS = [ type: 'object', properties: { title: { type: 'string', description: '窗口标题（正则，匹配第一个）' }, + wait_ms: { type: 'integer', description: '等待窗口出现（应用刚启动时用）' }, }, required: ['title'], additionalProperties: false, @@ -337,6 +338,10 @@ const TOOLS = [ description: 'auto=UIA 优先+OCR 兜底；uia=仅 UIA；ocr=仅截图识别', }, limit: { type: 'integer', description: '返回上限（默认 20）' }, + wait_ms: { + type: 'integer', + description: '等待元素出现的上限毫秒（0=只查一次）。界面有动画/加载时建议 3000~8000', + }, }, required: ['text'], additionalProperties: false, diff --git a/packages/client/src/macro.ts b/packages/client/src/macro.ts index a8bd960..0eac833 100644 --- a/packages/client/src/macro.ts +++ b/packages/client/src/macro.ts @@ -106,7 +106,10 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise { case 'focus': { const title = String(step['title'] ?? ''); if (!title) throw new Error('focus 需要 title'); - const d = (await invoke(c, CapabilityNames.WindowFocus, { title })) as { title?: string; x: number; y: number; width: number; height: number }; + const d = (await invoke(c, CapabilityNames.WindowFocus, { + title, + ...(step['wait_ms'] !== undefined ? { wait_ms: step['wait_ms'] } : {}), + }, Number(step['timeout_ms'] ?? 60_000))) as { title?: string; x: number; y: number; width: number; height: number; activated_by?: string }; return `已聚焦「${d.title ?? title}」(${d.x},${d.y} ${d.width}x${d.height})`; } case 'find': { @@ -116,13 +119,18 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise { if (step['method']) args['method'] = step['method']; if (step['window']) args['window'] = step['window']; if (step['control_type']) args['control_type'] = step['control_type']; + // v12.2：等待语义（界面有加载/动画时必备，避免假失败） + if (step['wait_ms'] !== undefined) args['wait_ms'] = step['wait_ms']; + if (step['interval_ms'] !== undefined) args['interval_ms'] = step['interval_ms']; const d = (await invoke(c, CapabilityNames.ScreenFind, args, 120_000)) as { engine: string; matches: Array<{ name: string; x: number; y: number }>; + waited_ms?: number; }; if (d.matches.length === 0) throw new Error(`未找到「${text}」（引擎 ${d.engine}）`); const m = d.matches[0]!; - const detail = `命中「${m.name}」@(${m.x},${m.y}) via ${d.engine}`; + const waited = d.waited_ms && d.waited_ms > 800 ? `（等待 ${d.waited_ms}ms）` : ''; + const detail = `命中「${m.name}」@(${m.x},${m.y}) via ${d.engine}${waited}`; if (step['click'] === true || step['dblclick'] === true) { await invoke(c, CapabilityNames.MouseClick, { x: m.x, @@ -221,11 +229,13 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise { if (d.windows.length === 0) throw new Error(`未找到标题匹配「${expect}」的窗口`); return `窗口断言通过（${d.windows.length} 个）`; } - // screen：用 screen.find 判定元素是否存在 + // screen：用 screen.find 判定元素是否存在（支持 wait_ms 等待出现） const d = (await invoke(c, CapabilityNames.ScreenFind, { text: expect, method: step['method'] ?? 'auto', limit: 1, + ...(step['wait_ms'] !== undefined ? { wait_ms: step['wait_ms'] } : {}), + ...(step['interval_ms'] !== undefined ? { interval_ms: step['interval_ms'] } : {}), }, 120_000)) as { engine: string; matches: unknown[] }; if (d.matches.length === 0) throw new Error(`屏幕上未找到「${expect}」（引擎 ${d.engine}）`); return `屏幕断言通过（via ${d.engine}）`; diff --git a/packages/protocol/src/capabilities.ts b/packages/protocol/src/capabilities.ts index 245854d..66905d4 100644 --- a/packages/protocol/src/capabilities.ts +++ b/packages/protocol/src/capabilities.ts @@ -733,6 +733,13 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ properties: { title: { type: 'string', description: '窗口标题（正则，匹配第一个）' }, hwnd: { type: 'string', description: '或直接给窗口句柄' }, + wait_ms: { + type: 'integer', + minimum: 0, + maximum: 30000, + default: 0, + description: '等待窗口出现（应用启动有延迟时用；仅 title 模式有效）', + }, }, additionalProperties: false, }, @@ -773,6 +780,14 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ description: 'auto=UIA 优先+OCR 兜底；uia=仅 UIA；ocr=仅截图识别', }, limit: { type: 'integer', default: 20 }, + wait_ms: { + type: 'integer', + minimum: 0, + maximum: 30000, + default: 0, + description: '等待元素出现的上限毫秒（0=只查一次）。界面有动画/加载时用', + }, + interval_ms: { type: 'integer', minimum: 100, maximum: 2000, default: 400 }, }, required: ['text'], additionalProperties: false, @@ -781,6 +796,7 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ type: 'object', properties: { engine: { type: 'string', description: '实际使用的引擎：uia | ocr' }, + waited_ms: { type: 'integer', description: '实际等待时长（便于诊断是否命中等待窗口）' }, matches: { type: 'array', items: {