diff --git a/apps/agent/src/capabilities/window.ts b/apps/agent/src/capabilities/window.ts index 09fe3bf..dc20c0b 100644 --- a/apps/agent/src/capabilities/window.ts +++ b/apps/agent/src/capabilities/window.ts @@ -160,7 +160,29 @@ export async function windowFocus(args: Args): Promise { if (!title && !hwnd) { throw new CapabilityError(ErrorCodes.PARAM_INVALID, '需提供 title(正则)或 hwnd'); } + // v12.2:等窗口出现(应用启动有延迟,一次性查找极易假失败) + const waitMs = Math.max(0, Math.min(30_000, (args['wait_ms'] as number | undefined) ?? 0)); + if (waitMs > 0 && !hwnd) { + const t0 = Date.now(); + let lastErr: unknown = null; + while (Date.now() - t0 < waitMs) { + try { + return await focusOnce(title, hwnd); + } catch (err) { + lastErr = err; + await new Promise((r) => setTimeout(r, 500)); + } + } + throw lastErr instanceof Error + ? lastErr + : new CapabilityError(ErrorCodes.EXECUTION_FAILED, `等待 ${waitMs}ms 仍未找到窗口`, { title }); + } + return focusOnce(title, hwnd); +} + +/** 单次聚焦(供等待重试复用)。 */ +async function focusOnce(title: string | undefined, hwnd: string | undefined): Promise { const script = `${WIN32_PRELUDE} $h = [IntPtr]::Zero $want = ${JSON.stringify(hwnd ?? '')} @@ -271,6 +293,10 @@ export async function screenFind(args: Args): Promise { const controlType = args['control_type'] as string | undefined; const limit = (args['limit'] as number | undefined) ?? 20; const method = (args['method'] as string | undefined) ?? 'auto'; // auto | uia | ocr + // v12.2:等待语义 —— 界面常有动画/加载延迟,一次性查找极易假失败。 + // wait_ms > 0 时轮询直到命中或超时(返回 waited_ms 让调用方知情)。 + const waitMs = Math.max(0, Math.min(30_000, (args['wait_ms'] as number | undefined) ?? 0)); + const intervalMs = Math.max(100, Math.min(2000, (args['interval_ms'] as number | undefined) ?? 400)); // ---------- UIA 引擎 ---------- const uiaFind = async (): Promise[]> => { @@ -488,17 +514,27 @@ $out | ConvertTo-Json -Compress return toArray>(extractJson[]>(out) ?? []); }; - if (method === 'uia') { - return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' }; + /** 单次尝试(按 method 选择引擎)。 */ + const attempt = async (): Promise<{ matches: Record[]; engine: string }> => { + if (method === 'uia') return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' }; + if (method === 'ocr') return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' }; + // auto:UIA 优先(快且带控件语义),找不到再 OCR 兜底(自绘 UI) + const uiaMatches = await uiaFind(); + if (uiaMatches.length > 0) return { matches: uiaMatches.slice(0, limit), engine: 'uia' }; + const ocrMatches = await ocrFind(); + return { matches: ocrMatches.slice(0, limit), engine: 'ocr' }; + }; + + const startedAt = Date.now(); + let last = await attempt(); + while (last.matches.length === 0 && Date.now() - startedAt < waitMs) { + await new Promise((r) => setTimeout(r, intervalMs)); + last = await attempt(); } - if (method === 'ocr') { - return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' }; - } - // auto:UIA 优先(快且带控件语义),找不到再 OCR 兜底(自绘 UI) - const uiaMatches = await uiaFind(); - if (uiaMatches.length > 0) { - return { matches: uiaMatches.slice(0, limit), engine: 'uia' }; - } - const ocrMatches = await ocrFind(); - return { matches: ocrMatches.slice(0, limit), engine: 'ocr' }; + return { + matches: last.matches, + engine: last.engine, + waited_ms: Date.now() - startedAt, + ...(waitMs > 0 ? { wait_ms: waitMs } : {}), + }; } diff --git a/apps/mcp/src/index.ts b/apps/mcp/src/index.ts index 3d3b50a..3b37ad2 100644 --- a/apps/mcp/src/index.ts +++ b/apps/mcp/src/index.ts @@ -312,6 +312,7 @@ const TOOLS = [ type: 'object', properties: { title: { type: 'string', description: '窗口标题(正则,匹配第一个)' }, + wait_ms: { type: 'integer', description: '等待窗口出现(应用刚启动时用)' }, }, required: ['title'], additionalProperties: false, @@ -337,6 +338,10 @@ const TOOLS = [ description: 'auto=UIA 优先+OCR 兜底;uia=仅 UIA;ocr=仅截图识别', }, limit: { type: 'integer', description: '返回上限(默认 20)' }, + wait_ms: { + type: 'integer', + description: '等待元素出现的上限毫秒(0=只查一次)。界面有动画/加载时建议 3000~8000', + }, }, required: ['text'], additionalProperties: false, diff --git a/packages/client/src/macro.ts b/packages/client/src/macro.ts index a8bd960..0eac833 100644 --- a/packages/client/src/macro.ts +++ b/packages/client/src/macro.ts @@ -106,7 +106,10 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise { case 'focus': { const title = String(step['title'] ?? ''); if (!title) throw new Error('focus 需要 title'); - const d = (await invoke(c, CapabilityNames.WindowFocus, { title })) as { title?: string; x: number; y: number; width: number; height: number }; + const d = (await invoke(c, CapabilityNames.WindowFocus, { + title, + ...(step['wait_ms'] !== undefined ? { wait_ms: step['wait_ms'] } : {}), + }, Number(step['timeout_ms'] ?? 60_000))) as { title?: string; x: number; y: number; width: number; height: number; activated_by?: string }; return `已聚焦「${d.title ?? title}」(${d.x},${d.y} ${d.width}x${d.height})`; } case 'find': { @@ -116,13 +119,18 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise { if (step['method']) args['method'] = step['method']; if (step['window']) args['window'] = step['window']; if (step['control_type']) args['control_type'] = step['control_type']; + // v12.2:等待语义(界面有加载/动画时必备,避免假失败) + if (step['wait_ms'] !== undefined) args['wait_ms'] = step['wait_ms']; + if (step['interval_ms'] !== undefined) args['interval_ms'] = step['interval_ms']; const d = (await invoke(c, CapabilityNames.ScreenFind, args, 120_000)) as { engine: string; matches: Array<{ name: string; x: number; y: number }>; + waited_ms?: number; }; if (d.matches.length === 0) throw new Error(`未找到「${text}」(引擎 ${d.engine})`); const m = d.matches[0]!; - const detail = `命中「${m.name}」@(${m.x},${m.y}) via ${d.engine}`; + const waited = d.waited_ms && d.waited_ms > 800 ? `(等待 ${d.waited_ms}ms)` : ''; + const detail = `命中「${m.name}」@(${m.x},${m.y}) via ${d.engine}${waited}`; if (step['click'] === true || step['dblclick'] === true) { await invoke(c, CapabilityNames.MouseClick, { x: m.x, @@ -221,11 +229,13 @@ async function runStep(ctx: MacroContext, step: MacroStep): Promise { if (d.windows.length === 0) throw new Error(`未找到标题匹配「${expect}」的窗口`); return `窗口断言通过(${d.windows.length} 个)`; } - // screen:用 screen.find 判定元素是否存在 + // screen:用 screen.find 判定元素是否存在(支持 wait_ms 等待出现) const d = (await invoke(c, CapabilityNames.ScreenFind, { text: expect, method: step['method'] ?? 'auto', limit: 1, + ...(step['wait_ms'] !== undefined ? { wait_ms: step['wait_ms'] } : {}), + ...(step['interval_ms'] !== undefined ? { interval_ms: step['interval_ms'] } : {}), }, 120_000)) as { engine: string; matches: unknown[] }; if (d.matches.length === 0) throw new Error(`屏幕上未找到「${expect}」(引擎 ${d.engine})`); return `屏幕断言通过(via ${d.engine})`; diff --git a/packages/protocol/src/capabilities.ts b/packages/protocol/src/capabilities.ts index 245854d..66905d4 100644 --- a/packages/protocol/src/capabilities.ts +++ b/packages/protocol/src/capabilities.ts @@ -733,6 +733,13 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ properties: { title: { type: 'string', description: '窗口标题(正则,匹配第一个)' }, hwnd: { type: 'string', description: '或直接给窗口句柄' }, + wait_ms: { + type: 'integer', + minimum: 0, + maximum: 30000, + default: 0, + description: '等待窗口出现(应用启动有延迟时用;仅 title 模式有效)', + }, }, additionalProperties: false, }, @@ -773,6 +780,14 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ description: 'auto=UIA 优先+OCR 兜底;uia=仅 UIA;ocr=仅截图识别', }, limit: { type: 'integer', default: 20 }, + wait_ms: { + type: 'integer', + minimum: 0, + maximum: 30000, + default: 0, + description: '等待元素出现的上限毫秒(0=只查一次)。界面有动画/加载时用', + }, + interval_ms: { type: 'integer', minimum: 100, maximum: 2000, default: 400 }, }, required: ['text'], additionalProperties: false, @@ -781,6 +796,7 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ type: 'object', properties: { engine: { type: 'string', description: '实际使用的引擎:uia | ocr' }, + waited_ms: { type: 'integer', description: '实际等待时长(便于诊断是否命中等待窗口)' }, matches: { type: 'array', items: {