feat(v12.2): GUI 等待/重试语义 —— 结束「点下去就不管」的假失败
## 动机
GUI 自动化最常见的失败不是「找不到」，而是「还没出来」：应用启动、页面切换、动画
都有延迟，一次性查找必然假失败。之前调用方只能自己在外面 sleep+重试（浪费往返）。
## 实现（agent 侧内联轮询，避免多次往返）
- screen.find 新增 wait_ms（0~30000，默认 0）与 interval_ms（100~2000，默认 400）：
轮询到命中或超时；无论成败都返回 waited_ms，让调用方知道是否命中等待窗口
- window.focus 新增 wait_ms：窗口未出现时每 500ms 重试，直到超时（仅 title 模式）
- 宏步骤 find/assert 透传 wait_ms/interval_ms（写宏时可直接声明等待）
- MCP na_screen_find 暴露 wait_ms
## 真机验证（决定性）
调用 window.focus(title=Notepad, wait_ms=15000) 时**记事本尚未运行**；
4 秒后才启动 -> 等待轮询捕获并成功聚焦：
{ title: "*install.ps1 - Notepad", focused: true, activated_by: "click" }
注意 activated_by=click：本次 SetForegroundWindow 又被系统静默否决，
点击兜底自动接管 —— 两个加固协同生效（等待 + 激活兜底）。
单测 89/89、e2e 21/21 全绿
This commit is contained in:
jkooe committed
1 parent
cd774ae0a6
commit
a892b113ca
4 files changed
+82
-15
No files matched your search
@@ -160,7 +160,29 @@ export async function windowFocus(args: Args): Promise<unknown> {
|
||||
if (!title && !hwnd) {
|
||||
throw new CapabilityError(ErrorCodes.PARAM_INVALID, '需提供 title（正则）或 hwnd');
|
||||
}
|
||||
// v12.2：等窗口出现（应用启动有延迟，一次性查找极易假失败）
|
||||
const waitMs = Math.max(0, Math.min(30_000, (args['wait_ms'] as number | undefined) ?? 0));
|
||||
if (waitMs > 0 && !hwnd) {
|
||||
const t0 = Date.now();
|
||||
let lastErr: unknown = null;
|
||||
while (Date.now() - t0 < waitMs) {
|
||||
try {
|
||||
return await focusOnce(title, hwnd);
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
}
|
||||
throw lastErr instanceof Error
|
||||
? lastErr
|
||||
: new CapabilityError(ErrorCodes.EXECUTION_FAILED, `等待 ${waitMs}ms 仍未找到窗口`, { title });
|
||||
}
|
||||
|
||||
return focusOnce(title, hwnd);
|
||||
}
|
||||
|
||||
/** 单次聚焦（供等待重试复用）。 */
|
||||
async function focusOnce(title: string | undefined, hwnd: string | undefined): Promise<unknown> {
|
||||
const script = `${WIN32_PRELUDE}
|
||||
$h = [IntPtr]::Zero
|
||||
$want = ${JSON.stringify(hwnd ?? '')}
|
||||
@@ -271,6 +293,10 @@ export async function screenFind(args: Args): Promise<unknown> {
|
||||
const controlType = args['control_type'] as string | undefined;
|
||||
const limit = (args['limit'] as number | undefined) ?? 20;
|
||||
const method = (args['method'] as string | undefined) ?? 'auto'; // auto | uia | ocr
|
||||
// v12.2：等待语义 —— 界面常有动画/加载延迟，一次性查找极易假失败。
|
||||
// wait_ms > 0 时轮询直到命中或超时（返回 waited_ms 让调用方知情）。
|
||||
const waitMs = Math.max(0, Math.min(30_000, (args['wait_ms'] as number | undefined) ?? 0));
|
||||
const intervalMs = Math.max(100, Math.min(2000, (args['interval_ms'] as number | undefined) ?? 400));
|
||||
|
||||
// ---------- UIA 引擎 ----------
|
||||
const uiaFind = async (): Promise<Record<string, unknown>[]> => {
|
||||
@@ -488,17 +514,27 @@ $out | ConvertTo-Json -Compress
|
||||
return toArray<Record<string, unknown>>(extractJson<Record<string, unknown>[]>(out) ?? []);
|
||||
};
|
||||
|
||||
if (method === 'uia') {
|
||||
return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' };
|
||||
/** 单次尝试（按 method 选择引擎）。 */
|
||||
const attempt = async (): Promise<{ matches: Record<string, unknown>[]; engine: string }> => {
|
||||
if (method === 'uia') return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' };
|
||||
if (method === 'ocr') return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' };
|
||||
// auto：UIA 优先（快且带控件语义），找不到再 OCR 兜底（自绘 UI）
|
||||
const uiaMatches = await uiaFind();
|
||||
if (uiaMatches.length > 0) return { matches: uiaMatches.slice(0, limit), engine: 'uia' };
|
||||
const ocrMatches = await ocrFind();
|
||||
return { matches: ocrMatches.slice(0, limit), engine: 'ocr' };
|
||||
};
|
||||
|
||||
const startedAt = Date.now();
|
||||
let last = await attempt();
|
||||
while (last.matches.length === 0 && Date.now() - startedAt < waitMs) {
|
||||
await new Promise((r) => setTimeout(r, intervalMs));
|
||||
last = await attempt();
|
||||
}
|
||||
if (method === 'ocr') {
|
||||
return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' };
|
||||
}
|
||||
// auto：UIA 优先（快且带控件语义），找不到再 OCR 兜底（自绘 UI）
|
||||
const uiaMatches = await uiaFind();
|
||||
if (uiaMatches.length > 0) {
|
||||
return { matches: uiaMatches.slice(0, limit), engine: 'uia' };
|
||||
}
|
||||
const ocrMatches = await ocrFind();
|
||||
return { matches: ocrMatches.slice(0, limit), engine: 'ocr' };
|
||||
return {
|
||||
matches: last.matches,
|
||||
engine: last.engine,
|
||||
waited_ms: Date.now() - startedAt,
|
||||
...(waitMs > 0 ? { wait_ms: waitMs } : {}),
|
||||
};
|
||||
}
|
||||
@@ -312,6 +312,7 @@ const TOOLS = [
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: { type: 'string', description: '窗口标题（正则，匹配第一个）' },
|
||||
wait_ms: { type: 'integer', description: '等待窗口出现（应用刚启动时用）' },
|
||||
},
|
||||
required: ['title'],
|
||||
additionalProperties: false,
|
||||
@@ -337,6 +338,10 @@ const TOOLS = [
|
||||
description: 'auto=UIA 优先+OCR 兜底；uia=仅 UIA；ocr=仅截图识别',
|
||||
},
|
||||
limit: { type: 'integer', description: '返回上限（默认 20）' },
|
||||
wait_ms: {
|
||||
type: 'integer',
|
||||
description: '等待元素出现的上限毫秒（0=只查一次）。界面有动画/加载时建议 3000~8000',
|
||||
},
|
||||
},
|
||||
required: ['text'],
|
||||
additionalProperties: false,
|
||||
|
||||
@@ -106,7 +106,10 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise<string> {
|
||||
case 'focus': {
|
||||
const title = String(step['title'] ?? '');
|
||||
if (!title) throw new Error('focus 需要 title');
|
||||
const d = (await invoke(c, CapabilityNames.WindowFocus, { title })) as { title?: string; x: number; y: number; width: number; height: number };
|
||||
const d = (await invoke(c, CapabilityNames.WindowFocus, {
|
||||
title,
|
||||
...(step['wait_ms'] !== undefined ? { wait_ms: step['wait_ms'] } : {}),
|
||||
}, Number(step['timeout_ms'] ?? 60_000))) as { title?: string; x: number; y: number; width: number; height: number; activated_by?: string };
|
||||
return `已聚焦「${d.title ?? title}」(${d.x},${d.y} ${d.width}x${d.height})`;
|
||||
}
|
||||
case 'find': {
|
||||
@@ -116,13 +119,18 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise<string> {
|
||||
if (step['method']) args['method'] = step['method'];
|
||||
if (step['window']) args['window'] = step['window'];
|
||||
if (step['control_type']) args['control_type'] = step['control_type'];
|
||||
// v12.2：等待语义（界面有加载/动画时必备，避免假失败）
|
||||
if (step['wait_ms'] !== undefined) args['wait_ms'] = step['wait_ms'];
|
||||
if (step['interval_ms'] !== undefined) args['interval_ms'] = step['interval_ms'];
|
||||
const d = (await invoke(c, CapabilityNames.ScreenFind, args, 120_000)) as {
|
||||
engine: string;
|
||||
matches: Array<{ name: string; x: number; y: number }>;
|
||||
waited_ms?: number;
|
||||
};
|
||||
if (d.matches.length === 0) throw new Error(`未找到「${text}」（引擎 ${d.engine}）`);
|
||||
const m = d.matches[0]!;
|
||||
const detail = `命中「${m.name}」@(${m.x},${m.y}) via ${d.engine}`;
|
||||
const waited = d.waited_ms && d.waited_ms > 800 ? `（等待 ${d.waited_ms}ms）` : '';
|
||||
const detail = `命中「${m.name}」@(${m.x},${m.y}) via ${d.engine}${waited}`;
|
||||
if (step['click'] === true || step['dblclick'] === true) {
|
||||
await invoke(c, CapabilityNames.MouseClick, {
|
||||
x: m.x,
|
||||
@@ -221,11 +229,13 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise<string> {
|
||||
if (d.windows.length === 0) throw new Error(`未找到标题匹配「${expect}」的窗口`);
|
||||
return `窗口断言通过（${d.windows.length} 个）`;
|
||||
}
|
||||
// screen：用 screen.find 判定元素是否存在
|
||||
// screen：用 screen.find 判定元素是否存在（支持 wait_ms 等待出现）
|
||||
const d = (await invoke(c, CapabilityNames.ScreenFind, {
|
||||
text: expect,
|
||||
method: step['method'] ?? 'auto',
|
||||
limit: 1,
|
||||
...(step['wait_ms'] !== undefined ? { wait_ms: step['wait_ms'] } : {}),
|
||||
...(step['interval_ms'] !== undefined ? { interval_ms: step['interval_ms'] } : {}),
|
||||
}, 120_000)) as { engine: string; matches: unknown[] };
|
||||
if (d.matches.length === 0) throw new Error(`屏幕上未找到「${expect}」（引擎 ${d.engine}）`);
|
||||
return `屏幕断言通过（via ${d.engine}）`;
|
||||
|
||||
@@ -733,6 +733,13 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
properties: {
|
||||
title: { type: 'string', description: '窗口标题（正则，匹配第一个）' },
|
||||
hwnd: { type: 'string', description: '或直接给窗口句柄' },
|
||||
wait_ms: {
|
||||
type: 'integer',
|
||||
minimum: 0,
|
||||
maximum: 30000,
|
||||
default: 0,
|
||||
description: '等待窗口出现（应用启动有延迟时用；仅 title 模式有效）',
|
||||
},
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
@@ -773,6 +780,14 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
description: 'auto=UIA 优先+OCR 兜底；uia=仅 UIA；ocr=仅截图识别',
|
||||
},
|
||||
limit: { type: 'integer', default: 20 },
|
||||
wait_ms: {
|
||||
type: 'integer',
|
||||
minimum: 0,
|
||||
maximum: 30000,
|
||||
default: 0,
|
||||
description: '等待元素出现的上限毫秒（0=只查一次）。界面有动画/加载时用',
|
||||
},
|
||||
interval_ms: { type: 'integer', minimum: 100, maximum: 2000, default: 400 },
|
||||
},
|
||||
required: ['text'],
|
||||
additionalProperties: false,
|
||||
@@ -781,6 +796,7 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
type: 'object',
|
||||
properties: {
|
||||
engine: { type: 'string', description: '实际使用的引擎：uia | ocr' },
|
||||
waited_ms: { type: 'integer', description: '实际等待时长（便于诊断是否命中等待窗口）' },
|
||||
matches: {
|
||||
type: 'array',
|
||||
items: {
|
||||
|
||||
Reference in new issue
Block a user