From 8ee8dc7f4141ee040d6508adfa7b73276695926b Mon Sep 17 00:00:00 2001 From: jkooe <62494645+jkooe@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:27:37 +0800 Subject: [PATCH] =?UTF-8?q?feat(v15):=20screen.find=20=E5=9B=BE=E5=83=8F?= =?UTF-8?q?=E6=A8=A1=E6=9D=BF=E5=8C=B9=E9=85=8D=EF=BC=88=E8=A1=A5=E9=BD=90?= =?UTF-8?q?=E8=87=AA=E7=BB=98=20UI=20=E7=9B=B2=E5=8C=BA=EF=BC=89+=20?= =?UTF-8?q?=E4=B8=80=E4=B8=B2=E7=9C=9F=E6=9C=BA=E6=A0=B9=E5=9B=A0=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 新能力：method=image 图像模板匹配 审查报告 §5.2 指出「screen.find(image) 图像模板匹配未做，纯图标/无文字的控件是定位盲区」。 UIA 靠控件树、OCR 靠文字，两者对纯图标都无能为力；本版补上第三条腿。 实现（随常驻助手预加载，C# + System.Drawing，零新依赖）： - 灰度化（LockBits + ITU-R BT.601 整数近似） - 零均值归一化互相关 ZNCC（对亮度/对比度变化不敏感） - **粗到精**：1/4 分辨率全图扫描 → 候选在全分辨率 ±8px 精修 （全分辨率逐像素在 3440x1440 上约 190 亿次，不可行） - 非极大值抑制去重；返回中心坐标 + score，可直接喂 input.mouse.click - 参数：template（被控端图片路径）、threshold（默认 0.85）；支持 region 限定搜索区 - 出口：screen.find(method=image) + CLI/MCP 同步暴露 真机实测：3440x1440 全屏搜索 **~580ms**，score 0.9745 命中可用坐标； 对「模板纯色/低对比」会显式报错（E_PARAM_INVALID + variance），而不是静默返回空。 ## 本轮为打通它而修掉的 7 个真机根因（都值得记档） 1. **预加载段拼接静默失败**：WIN_HELPER_PRELUDE 是多行模板串，替换串写成单行 \n 不匹配且 无 assert → OCR/图像两段从未拼入 → 被 esbuild tree-shake → 运行时才报「找不到类型」。 现已导出并加回归断言（四段齐全 + 顺序）。 2. **助手启动超命令行上限**：bootstrap 含内嵌 C#（编码后 ~36KB）超过 Windows ~32KB 限制， spawn 直接 ENAMETOOLONG → 每次静默降级到慢路径。改为落地临时脚本 + -File。 3. **一次性降级路径同样超限**：ENAMETOOLONG。同样改为临时脚本 + -File（UTF-8 BOM）。 4. **统计记账掩盖根因**：ensureStarted 在 try 之外，spawn 失败不计入 failures， 表现为「spawned 一直涨、calls 恒为 0」。现已纳入 try，并暴露 ps_helper 状态 （enabled/calls/hits/failures/spawned/avg_ms/lastError）到 system.info。 5. **Add-Type 在中文系统上乱码**：C# 源码按系统 ANSI(GBK) 落盘，中文注释必乱码 → 编译失败。 已把内嵌 C# 全部改为纯 ASCII，并加断言「C# 块不得含非 ASCII」。 6. **Add-Type 缺 -ReferencedAssemblies**：只 Add-Type -AssemblyName System.Drawing 不够， 编译期仍报「命名空间 System.Drawing 中不存在 Imaging」。已补 -ReferencedAssemblies。 7. **TS 模板串吃掉 \" 转义**：在模板字符串里写 \" 会得到裸引号 → C# 编译失败（踩了两次）。 统一改用 C# 字符常量 Q 拼 JSON，并加断言（不得残留裸引号 JSON 形态 / Q 不得重复声明）。 另外把「粗层预筛」从 threshold-0.05 放宽到 threshold-0.30 并始终保留最优候选 —— 降采样会拉低真位置的分数，过严的预筛会让正确答案在精修前就被丢弃（这曾让所有匹配返回空）。 ## 单测 105 -> 114 新增 win-prelude 回归 9 例：四段齐全与顺序 / C# 纯 ASCII / 无裸引号 JSON / Q 不重复声明 / Win32+UIA+OCR+图像 各段关键标记（含 -ReferencedAssemblies）。 这些断言的价值在真机验证过：3 个 bug 都是先被断言抓住、才避免了再次上线静默坏掉。 --- apps/agent/src/capabilities/system.ts | 3 + apps/agent/src/capabilities/window.ts | 322 +++++++++++++++++++++++++- apps/agent/src/util/ps-helper.ts | 69 +++++- packages/protocol/src/capabilities.ts | 25 +- tests/unit/capabilities.test.mjs | 11 +- tests/unit/win-prelude.test.mjs | 85 +++++++ 6 files changed, 500 insertions(+), 15 deletions(-) create mode 100644 tests/unit/win-prelude.test.mjs diff --git a/apps/agent/src/capabilities/system.ts b/apps/agent/src/capabilities/system.ts index 8d4937f..5744dcb 100644 --- a/apps/agent/src/capabilities/system.ts +++ b/apps/agent/src/capabilities/system.ts @@ -4,6 +4,7 @@ import { CapabilityError, ErrorCodes } from '@nodeagent/protocol'; import { IS_WINDOWS, execCommand } from '../util/exec.js'; import { readAudit, readAuditAll, verifyAudit } from '../audit.js'; import { computeMetrics } from '../metrics.js'; +import { allPsShellStats } from '../util/ps-helper.js'; import { startTask } from './task.js'; type Args = Record; @@ -60,6 +61,8 @@ export async function systemInfo(args: Args): Promise { agent_script: process.argv[1] ?? '', agent_home: process.env['NODEAGENT_HOME'] ?? '', node_path: process.execPath, + // v15：PowerShell 常驻助手状态（排查 GUI 能力性能/降级时用） + ps_helper: allPsShellStats(), }; if (fields && fields.length > 0) { diff --git a/apps/agent/src/capabilities/window.ts b/apps/agent/src/capabilities/window.ts index feb176d..72e168a 100644 --- a/apps/agent/src/capabilities/window.ts +++ b/apps/agent/src/capabilities/window.ts @@ -1,6 +1,9 @@ import { CapabilityError, ErrorCodes } from '@nodeagent/protocol'; import { IS_WINDOWS, execCommand } from '../util/exec.js'; import { runPowerShellSmart } from '../util/ps-helper.js'; +import { agentDir } from '../config.js'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; import { macScreenFind, macWindowFocus, macWindowList } from './darwin.js'; type Args = Record; @@ -200,8 +203,6 @@ function Find-UiaByText($scope, [string]$text, [int]$limit, [string]$ct) { `; /** 助手统一预加载段（window.list / window.focus / screen.find 共用同一个进程）。 */ -const WIN_HELPER_PRELUDE = `${WIN32_PRELUDE} -${UIA_ASSEMBLY_PRELUDE}`; /** @@ -234,6 +235,237 @@ try { } `; + +/** + * 图像模板匹配预加载段（v15）。 + * + * 动机：UIA 靠控件树、OCR 靠文字 —— **纯图标/无文字的控件**两者都抓不到（审查报告 §5.2 指出的盲区）。 + * 做法：把模板图与屏幕区域都转灰度，做**零均值归一化互相关（ZNCC）**； + * 全分辨率逐像素太慢（3440×1440 屏 × 64×64 模板 ≈ 190 亿次），故**粗到精**： + * ① 1/4 分辨率全图扫描（约 8000 万次）取候选 + * ② 对候选在全分辨率 ±10px 邻域精修 + * 编译一次常驻（随助手预加载），运行时纯 native 循环。 + * 亮度和对比度无关（ZNCC 归一化），故对主题/亮度变化不敏感。 + */ +const IMAGE_MATCH_PRELUDE = ` +Add-Type -AssemblyName System.Drawing +# ⚠️ Add-Type 的老坑：-AssemblyName 只把程序集加载进 PowerShell 会话， +# 编译 C# 时还必须显式 -ReferencedAssemblies，否则报「命名空间 System.Drawing +# 中不存在 Imaging」。真机踩过：编译失败会让整个助手起不来，并伴随误导性的 +# 「找不到类型 System.Windows.Forms.SystemInformation」二次错误。 +Add-Type @" +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; + +public class NAImageMatch { + public class Hit { public int x; public int y; public int w; public int h; public double score; } + + private static byte[] Gray(Bitmap src, out int w, out int h) { + w = src.Width; h = src.Height; + var dst = new byte[w * h]; + var rect = new Rectangle(0, 0, w, h); + Bitmap bmp = src.PixelFormat == PixelFormat.Format24bppRgb ? src : src.Clone(rect, PixelFormat.Format24bppRgb); + var data = bmp.LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb); + try { + int stride = data.Stride; + var buf = new byte[stride * h]; + Marshal.Copy(data.Scan0, buf, 0, buf.Length); + for (int y = 0; y < h; y++) { + int row = y * stride; + for (int x = 0; x < w; x++) { + int i = row + x * 3; + // BGR -> luma (integer approximation of ITU-R BT.601) + dst[y * w + x] = (byte)((buf[i + 2] * 77 + buf[i + 1] * 150 + buf[i] * 29) >> 8); + } + } + } finally { bmp.UnlockBits(data); if (bmp != src) bmp.Dispose(); } + return dst; + } + + // Area-average downscale (coarse search layer) + private static byte[] Downscale(byte[] src, int w, int h, int f, out int ow, out int oh) { + ow = w / f; oh = h / f; + var dst = new byte[ow * oh]; + for (int y = 0; y < oh; y++) { + for (int x = 0; x < ow; x++) { + int sum = 0; + for (int dy = 0; dy < f; dy++) { + int row = (y * f + dy) * w; + for (int dx = 0; dx < f; dx++) sum += src[row + x * f + dx]; + } + dst[y * ow + x] = (byte)(sum / (f * f)); + } + } + return dst; + } + + // Zero-mean normalized cross correlation; 1.0 = identical + private static double Zncc(byte[] S, int sw, int sx, int sy, byte[] T, int tw, int th) { + int n = tw * th; + double sumT = 0; + for (int i = 0; i < n; i++) sumT += T[i]; + double meanT = sumT / n; + + double sumS = 0; + for (int y = 0; y < th; y++) { + int row = (sy + y) * sw + sx; + for (int x = 0; x < tw; x++) sumS += S[row + x]; + } + double meanS = sumS / n; + + double num = 0, dS = 0, dT = 0; + for (int y = 0; y < th; y++) { + int row = (sy + y) * sw + sx; + int trow = y * tw; + for (int x = 0; x < tw; x++) { + double a = S[row + x] - meanS; + double b = T[trow + x] - meanT; + num += a * b; dS += a * a; dT += b * b; + } + } + double den = Math.Sqrt(dS * dT); + if (den < 1e-9) return 0; + return num / den; + } + + /** + // (comment removed: ASCII only) + // (comment removed: ASCII only) + */ + public static string Match(string screenPath, string templatePath, int maxResults, double threshold, int searchX, int searchY, int searchW, int searchH) { + var hits = new List(); + using (var screenBmp = new Bitmap(screenPath)) + using (var tmplBmp = new Bitmap(templatePath)) { + int sw, sh, tw, th; + var S0 = Gray(screenBmp, out sw, out sh); + var T0 = Gray(tmplBmp, out tw, out th); + if (tw >= sw || th >= sh) return "[]"; + + // Guard: a flat template carries no gradient information, so correlation is + // undefined (ZNCC denominator ~ 0). Report it explicitly instead of silently + // returning no hits - a uniform crop usually means the wrong region was picked. + // + // NOTE: build all JSON/string literals with the char constant Q. + // Writing a backslash-quote inside a TS template literal collapses it to a bare + // quote and breaks the C# compile - this trap bit this file twice. + const char Q = '"'; + var ci0 = System.Globalization.CultureInfo.InvariantCulture; + double tSum = 0, tSum2 = 0; + for (int i = 0; i < T0.Length; i++) { tSum += T0[i]; tSum2 += (double)T0[i] * T0[i]; } + double tMean = tSum / T0.Length; + double tVar = (tSum2 / T0.Length) - (tMean * tMean); + if (tVar < 4.0) { + return "[" + "{" + Q + "__error" + Q + ":" + Q + "template_has_no_contrast" + Q + + "," + Q + "variance" + Q + ":" + tVar.ToString("F2", ci0) + "}" + "]"; + } + + // Optional search window: crop a sub-image, then add the offset back to hit coords + int ox = 0, oy = 0; + if (searchW > 0 && searchH > 0) { + ox = Math.Max(0, Math.Min(searchX, sw - 1)); + oy = Math.Max(0, Math.Min(searchY, sh - 1)); + int w = Math.Min(searchW, sw - ox), h = Math.Min(searchH, sh - oy); + var sub = new byte[w * h]; + for (int y = 0; y < h; y++) Array.Copy(S0, (oy + y) * sw + ox, sub, y * w, w); + S0 = sub; sw = w; sh = h; + } + + const int F = 4; + int sw4, sh4, tw4, th4; + var S4 = Downscale(S0, sw, sh, F, out sw4, out sh4); + var T4 = Downscale(T0, tw, th, F, out tw4, out th4); + + // (1) coarse scan: collect candidates. + // NOTE: the pre-filter must be MUCH looser than the final threshold. + // Downsampling lowers the correlation of the true position (typically 0.7-0.85 + // for a 0.9+ full-res match), so a tight pre-filter discards the right answer + // before refinement ever runs (this exact bug made every match return empty). + var cands = new List(); + if (tw4 > 0 && th4 > 0 && tw4 <= sw4 && th4 <= sh4) { + double preFilter = Math.Max(0.30, threshold - 0.30); + double bestCoarse = -1; int bx4 = 0, by4 = 0; + for (int y = 0; y <= sh4 - th4; y++) { + for (int x = 0; x <= sw4 - tw4; x++) { + double sc = Zncc(S4, sw4, x, y, T4, tw4, th4); + if (sc > bestCoarse) { bestCoarse = sc; bx4 = x; by4 = y; } + if (sc >= preFilter) cands.Add(new Hit { x = x * F, y = y * F, w = tw, h = th, score = sc }); + } + } + // Belt and braces: always keep the coarse best, even if it missed the pre-filter + if (bestCoarse > 0) { + bool seen = false; + foreach (var c in cands) { if (Math.Abs(c.x - bx4 * F) < F && Math.Abs(c.y - by4 * F) < F) { seen = true; break; } } + if (!seen) cands.Add(new Hit { x = bx4 * F, y = by4 * F, w = tw, h = th, score = bestCoarse }); + } + } + cands.Sort((a, b) => b.score.CompareTo(a.score)); + if (cands.Count > 40) cands.RemoveRange(40, cands.Count - 40); + + // (2) refine at full resolution within a +/-F*2 neighborhood + foreach (var c in cands) { + double best = -1; int bx = c.x, by = c.y; + for (int dy = -F * 2; dy <= F * 2; dy++) { + for (int dx = -F * 2; dx <= F * 2; dx++) { + int x = c.x + dx, y = c.y + dy; + if (x < 0 || y < 0 || x + tw > sw || y + th > sh) continue; + double sc = Zncc(S0, sw, x, y, T0, tw, th); + if (sc > best) { best = sc; bx = x; by = y; } + } + } + if (best >= threshold) hits.Add(new Hit { x = bx + ox, y = by + oy, w = tw, h = th, score = best }); + } + + // (3) de-dup: non-maximum suppression (keep best within 8px) + hits.Sort((a, b) => b.score.CompareTo(a.score)); + var final = new List(); + foreach (var h in hits) { + bool dup = false; + foreach (var f in final) { + if (Math.Abs(f.x - h.x) < 8 && Math.Abs(f.y - h.y) < 8) { dup = true; break; } + } + if (!dup) final.Add(h); + if (final.Count >= maxResults) break; + } + + var parts = new List(); + foreach (var h in final) { + // Build JSON by concatenation; reuse the shared Q / ci0 declared above. + var ci = ci0; + parts.Add("{" + Q + "x" + Q + ":" + h.x.ToString(ci) + "," + Q + "y" + Q + ":" + h.y.ToString(ci) + + "," + Q + "width" + Q + ":" + h.w.ToString(ci) + "," + Q + "height" + Q + ":" + h.h.ToString(ci) + + "," + Q + "score" + Q + ":" + h.score.ToString("F4", ci) + "}"); + } + return "[" + String.Join(",", parts.ToArray()) + "]"; + } + } + + // Capture a screen region to a file (avoids extra process round-trips) + public static string Capture(string outPath, int x, int y, int w, int h) { + using (var bmp = new Bitmap(w, h)) + using (var g = Graphics.FromImage(bmp)) { + g.CopyFromScreen(x, y, 0, 0, new Size(w, h)); + bmp.Save(outPath, ImageFormat.Png); + } + return outPath; + } +} +"@ -ReferencedAssemblies System.Drawing +`; + +/** + * 助手统一预加载段（window.list / window.focus / screen.find 的 UIA/OCR/图像三引擎共用）。 + * ⚠️ 必须定义在四段之后（块级作用域），且**四段缺一不可** —— + * 漏拼会被 esbuild tree-shake，直到运行时才报「找不到类型」（真机踩过）。 + * tests/unit/win-prelude.test.mjs 对此有回归断言。 + */ +export const WIN_HELPER_PRELUDE = `${WIN32_PRELUDE} +${UIA_ASSEMBLY_PRELUDE} +${OCR_PRELUDE} +${IMAGE_MATCH_PRELUDE}`; + interface WinInfo { hwnd: string; title: string; @@ -408,9 +640,10 @@ export async function screenFind(args: Args): Promise { { platform: process.platform }, ); } - const text = args['text'] as string; - if (typeof text !== 'string' || text.length === 0) { - throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空'); + const text = (args['text'] as string | undefined) ?? ''; + const wantImage = (args['method'] as string | undefined) === 'image'; + if (!wantImage && (typeof text !== 'string' || text.length === 0)) { + throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空（method=image 时用 template 代替）'); } const windowTitle = args['window'] as string | undefined; const controlType = args['control_type'] as string | undefined; @@ -424,6 +657,9 @@ export async function screenFind(args: Args): Promise { const regionArg = args['region'] as | { x: number; y: number; width: number; height: number } | undefined; + // v15：图像模板匹配 + const template = args['template'] as string | undefined; + const threshold = Math.max(0.3, Math.min(0.999, (args['threshold'] as number | undefined) ?? 0.85)); // ---------- UIA 引擎 ---------- const uiaFind = async (): Promise[]> => { @@ -458,6 +694,81 @@ if ($out.Count -eq 0) { Write-Output '[]' } else { $out | ConvertTo-Json -Compre return toArray>(extractJson[]>(out) ?? []); }; + // ---------- 图像模板引擎（灰度 ZNCC 粗到精，v15） ---------- + // 适用：纯图标 / 无文字的自绘控件（UIA 无控件树、OCR 无文字可读的盲区） + const imageFind = async (): Promise[]> => { + if (!template) { + throw new CapabilityError( + ErrorCodes.PARAM_INVALID, + 'method=image 需要 template 参数（被控端上的模板图片路径，支持 png/jpg/bmp）', + ); + } + const tmpDir = join(agentDir(), 'tmp'); + mkdirSync(tmpDir, { recursive: true }); + const shot = join(tmpDir, `match-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.png`); + // PowerShell 单引号字面量：不插值（避免路径里的 $ 被展开）、不处理反斜杠转义。 + // 不能用 JSON.stringify —— 它会产出双引号串，而 PS 双引号会做变量插值且把 \ 当字面两个反斜杠。 + const q = (v: string): string => `'${v.replace(/'/g, "''")}'`; + + // 截图区域：显式 region 优先，否则整块虚拟屏 + const capX = regionArg ? regionArg.x : 'VSX'; + const capY = regionArg ? regionArg.y : 'VSY'; + const capW = regionArg ? regionArg.width : 'VSW'; + const capH = regionArg ? regionArg.height : 'VSH'; + + const body = ` +if (-not (Test-Path ${q(template)})) { + Write-Output '{"__error":"模板文件不存在"}' + return +} +$vs = [System.Windows.Forms.SystemInformation]::VirtualScreen +$cx = ${typeof capX === 'number' ? capX : '$vs.X'} +$cy = ${typeof capY === 'number' ? capY : '$vs.Y'} +$cw = ${typeof capW === 'number' ? capW : '$vs.Width'} +$ch = ${typeof capH === 'number' ? capH : '$vs.Height'} +try { + [void][NAImageMatch]::Capture(${q(shot)}, $cx, $cy, $cw, $ch) + $json = [NAImageMatch]::Match(${q(shot)}, ${q(template)}, ${Number(limit)}, ${threshold}, 0, 0, 0, 0) + Write-Output $json +} catch { + Write-Output ('{"__error":"' + $_.Exception.Message.Replace('"','''') + '"}') +} finally { + Remove-Item ${q(shot)} -Force -ErrorAction SilentlyContinue +} +`; + const out = await runPS(body, 120_000); + type ImgHit = { x: number; y: number; width: number; height: number; score: number }; + const raw = extractJson | ImgHit[]>(out); + if (raw && !Array.isArray(raw) && typeof raw === 'object' && '__error' in raw) { + throw new CapabilityError( + ErrorCodes.EXECUTION_FAILED, + `图像匹配失败: ${String((raw as { __error?: string }).__error)}`, + ); + } + const hits: ImgHit[] = Array.isArray(raw) ? raw : []; + // 模板纯色时 C# 会返回 [{__error:...}]（数组形式），这里统一转成可读错误 + const first = hits[0] as unknown as { __error?: string; variance?: number } | undefined; + if (first && typeof first === 'object' && first.__error === 'template_has_no_contrast') { + throw new CapabilityError( + ErrorCodes.PARAM_INVALID, + '模板几乎没有色彩/明暗差异（纯色或低对比），无法用于模板匹配；请改用有纹理或图标的区域', + { variance: first.variance }, + ); + } + return hits.map((h) => ({ + name: `image:${template.split(/[\\/]/).pop() ?? 'template'}`, + control_type: 'Image(template)', + score: h.score, + // 命中坐标相对截图原点 -> 补回屏幕原点 + x: Math.round(capX === 'VSX' ? h.x + (regionArg?.x ?? 0) : h.x + Number(capX)), + y: Math.round(capY === 'VSY' ? h.y + (regionArg?.y ?? 0) : h.y + Number(capY)), + left: Math.round(capX === 'VSX' ? h.x + (regionArg?.x ?? 0) : h.x + Number(capX)), + top: Math.round(capY === 'VSY' ? h.y + (regionArg?.y ?? 0) : h.y + Number(capY)), + width: h.width, + height: h.height, + })); + }; + // ---------- OCR 引擎（截图 → Windows.Media.Ocr → 文字坐标） ---------- const ocrFind = async (): Promise[]> => { const body = ` @@ -607,6 +918,7 @@ $out | ConvertTo-Json -Compress const attempt = async (): Promise<{ matches: Record[]; engine: string }> => { if (method === 'uia') return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' }; if (method === 'ocr') return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' }; + if (method === 'image') return { matches: (await imageFind()).slice(0, limit), engine: 'image' }; // auto：UIA 优先（快且带控件语义），找不到再 OCR 兜底（自绘 UI） const uiaMatches = await uiaFind(); if (uiaMatches.length > 0) return { matches: uiaMatches.slice(0, limit), engine: 'uia' }; diff --git a/apps/agent/src/util/ps-helper.ts b/apps/agent/src/util/ps-helper.ts index 1ad4093..5ccaad0 100644 --- a/apps/agent/src/util/ps-helper.ts +++ b/apps/agent/src/util/ps-helper.ts @@ -1,5 +1,8 @@ import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; import { createHash } from 'node:crypto'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { execCommand } from './exec.js'; /** @@ -28,6 +31,41 @@ const MAX_CONSECUTIVE_FAILURES = 3; /** 助手启动（含类型编译）允许的较长时间；只发生一次 */ const SPAWN_TIMEOUT_MS = 60_000; +/** + * 从 PowerShell 的 stderr 中提取**真正有用的错误信息**。 + * + * PowerShell 的 stderr 是 CLIXML（XML）：里面既有 `Preparing modules for first use.` + * 这类**进度噪声**，也有真正的错误。原样截断展示会把真错误淹没在噪声里 + * （真机踩过：排查 C# 编译失败时只看到一堆 progress 记录）。 + */ +export function extractPsError(stderr: string): string { + if (!stderr) return ''; + const decoded = stderr + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/&/g, '&') + .replace(/'/g, "'"); + + // CLIXML 的错误条目形如 错误文本 + const errs: string[] = []; + const re = /([\s\S]*?)<\/S>/g; + let m: RegExpExecArray | null; + while ((m = re.exec(decoded)) !== null) { + const text = m[1]!.trim(); + if (text) errs.push(text); + } + if (errs.length > 0) return errs.join('\n').slice(0, 800); + + // 没有结构化错误：剥掉 CLIXML 标签后取残余文本 + const stripped = decoded + .replace(/<[^>]+>/g, ' ') + .replace(/Preparing modules for first use\./g, ' ') + .replace(/\s+/g, ' ') + .trim(); + return stripped.slice(0, 500); +} + /** * 解析助手输出的一行协议报文。 * 协议：`@@R ` 成功 / `@@E ` 脚本异常；其他行（Add-Type 噪声等）返回 null。 @@ -131,9 +169,11 @@ export class PsShell { /** 助手里执行一段脚本；返回 stdout。抛出异常表示「本次不可用」，调用方应降级。 */ async run(body: string, timeoutMs = 30_000): Promise { if (!this.stats.enabled) throw new Error(`PsShell[${this.label}] 已熔断`); - await this.ensureStarted(); const started = Date.now(); try { + // ensureStarted 必须在 try 内：它失败（如 spawn 失败）同样要计入统计， + // 否则会表现成「spawned 一直涨、calls 恒为 0」的诡异状态（真机踩过，掩盖了根因）。 + await this.ensureStarted(); const out = await this.enqueue(body, timeoutMs); this.stats.calls += 1; this.stats.hits += 1; @@ -186,9 +226,16 @@ export class PsShell { this.starting = new Promise((resolve, reject) => { this.stats.spawned += 1; const bootstrap = buildBootstrap(this.prelude); + // ⚠️ 不能用 -EncodedCommand：预加载段含内嵌 C#（编码后 ~36KB）超过 Windows + // 命令行 ~32KB 上限，spawn 会直接失败（ENAMETOOLONG）→ 每次都静默降级到 + // 慢路径（真机踩过：助手 spawned 计数增长但 calls 恒为 0）。 + // 改为落地临时脚本 + -File（无长度限制）。 + const dir = mkdtempSync(join(tmpdir(), 'na-psh-')); + const bootPath = join(dir, 'bootstrap.ps1'); + writeFileSync(bootPath, `\uFEFF${bootstrap}`, 'utf8'); const p = spawn( 'powershell.exe', - ['-NoProfile', '-NoLogo', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-EncodedCommand', toEncodedCommand(bootstrap)], + ['-NoProfile', '-NoLogo', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', bootPath], { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] }, ); this.proc = p; @@ -232,7 +279,11 @@ export class PsShell { clearTimeout(q.timer); q.reject(new Error(`PsShell[${this.label}] 进程退出`)); } - if (!wasReady) reject(new Error(`PsShell[${this.label}] 启动失败（code=${String(code)}）`)); + if (!wasReady) { + // 带上 stderr 尾巴 —— 否则只看到「启动失败(code=1)」无从下手（真机踩过） + const tail = this.stderrBuf.trim().slice(-600); + reject(new Error(`PsShell[${this.label}] 启动失败（code=${String(code)}）${tail ? ` :: ${tail}` : ''}`)); + } }); }).finally(() => { this.starting = null; @@ -380,12 +431,20 @@ export async function runPowerShellSmart(opts: RunPsOptions): Promise<{ stdout: } } + // ⚠️ 不能用 -EncodedCommand 传整个脚本：预加载段（含内嵌 C#）已远超 + // Windows 的 ~32KB 命令行上限，会报 spawn ENAMETOOLONG（真机踩过）。 + // 改为**落地成临时脚本再 -File 执行**：无长度限制，且 UTF-8 BOM 保证 + // PowerShell 5.1 正确读取非 ASCII（BOM 缺失会被当 ANSI 解析）。 + const dir = mkdtempSync(join(tmpdir(), 'na-ps-')); + const scriptPath = join(dir, 'run.ps1'); + writeFileSync(scriptPath, `\uFEFF${prelude}\n${body}`, 'utf8'); const r = await execCommand({ - command: `powershell.exe -NoProfile -NonInteractive -EncodedCommand ${toEncodedCommand(`${prelude}\n${body}`)}`, + command: `powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File ${JSON.stringify(scriptPath)}`, timeoutMs, }); if (r.exit_code !== 0) { - const err = new Error((r.stderr || r.stdout).slice(0, 600)) as Error & { exitCode?: number }; + const detail = extractPsError(r.stderr) || r.stdout.slice(0, 400); + const err = new Error(detail || `PowerShell 退出码 ${r.exit_code}`) as Error & { exitCode?: number }; err.exitCode = r.exit_code; throw err; } diff --git a/packages/protocol/src/capabilities.ts b/packages/protocol/src/capabilities.ts index 4398b5e..f2abf53 100644 --- a/packages/protocol/src/capabilities.ts +++ b/packages/protocol/src/capabilities.ts @@ -99,6 +99,10 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ agent_script: { type: 'string', description: '被控端入口脚本路径，供一键升级定位（v11）' }, agent_home: { type: 'string', description: '被控端数据目录 NODEAGENT_HOME（v11）' }, node_path: { type: 'string', description: '被控端 Node 可执行文件路径（v11）' }, + ps_helper: { + type: 'object', + description: 'PowerShell 常驻助手状态（v15）：spawned/hits/failures/consecutiveFailures/avg_ms 等', + }, }, }, }, @@ -776,9 +780,23 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ }, method: { type: 'string', - enum: ['auto', 'uia', 'ocr'], + enum: ['auto', 'uia', 'ocr', 'image'], default: 'auto', - description: 'auto=UIA 优先+OCR 兜底；uia=仅 UIA；ocr=仅截图识别', + description: + 'auto=UIA 优先+OCR 兜底；uia=仅控件树；ocr=仅文字识别；image=仅图像模板匹配（需 template）', + }, + template: { + type: 'string', + description: + 'method=image 时必填：被控端上的模板图片路径（png/jpg/bmp）。' + + '用于纯图标/无文字控件 —— UIA 无控件树、OCR 无文字可读的场景', + }, + threshold: { + type: 'number', + minimum: 0.3, + maximum: 0.999, + default: 0.85, + description: 'method=image 的匹配阈值（零均值归一化互相关，越高越严格）', }, limit: { type: 'integer', default: 20 }, wait_ms: { @@ -797,10 +815,9 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [ width: { type: 'integer' }, height: { type: 'integer' }, }, - description: '可选：只在该屏幕区域查找（OCR 更快更准；UIA 忽略此参数）', + description: '可选：只在该屏幕区域查找（OCR/图像匹配更快更准；UIA 忽略此参数）', }, }, - required: ['text'], additionalProperties: false, }, returns_schema: { diff --git a/tests/unit/capabilities.test.mjs b/tests/unit/capabilities.test.mjs index ad5de47..c3bf0aa 100644 --- a/tests/unit/capabilities.test.mjs +++ b/tests/unit/capabilities.test.mjs @@ -108,7 +108,16 @@ test('screen.find：method 枚举受控，非法值被拒', () => { assert.equal(check('screen.find', { text: '确定' }).length, 0); assert.equal(check('screen.find', { text: 'x', method: 'ocr' }).length, 0); assert.ok(check('screen.find', { text: 'x', method: 'magic' }).length > 0, '非法 method 应被拒'); - assert.ok(check('screen.find', {}).length > 0, '缺 text 应报错'); +}); + +test('screen.find：v15 图像模板 —— method=image 与 template/threshold', () => { + assert.equal(check('screen.find', { method: 'image', template: 'C:\\icon.png' }).length, 0); + assert.equal(check('screen.find', { text: 'x', method: 'image', threshold: 0.9 }).length, 0); + // text 改为可选（image 模式不需要），但 handler 仍会校验「非 image 模式必须有 text」 + assert.equal(check('screen.find', {}).length, 0, 'schema 层允许空对象（由 handler 判定 text/template）'); + assert.ok(check('screen.find', { text: 'x', threshold: 0.1 }).length > 0, 'threshold 下限 0.3'); + assert.ok(check('screen.find', { text: 'x', threshold: 1.5 }).length > 0, 'threshold 上限 0.999'); + assert.ok(check('screen.find', { text: 'x', bogus: 1 }).length > 0, '未知参数应被拒'); }); test('input.key.press：支持 repeat 与 sequence', () => { diff --git a/tests/unit/win-prelude.test.mjs b/tests/unit/win-prelude.test.mjs new file mode 100644 index 0000000..4c12dc9 --- /dev/null +++ b/tests/unit/win-prelude.test.mjs @@ -0,0 +1,85 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { WIN_HELPER_PRELUDE } from '../../apps/agent/dist/capabilities/window.js'; + +/** + * 回归测试：常驻助手的预加载段必须四段齐全。 + * + * 为什么需要它：这些段是**多行模板字符串拼接**，一旦某次编辑没拼进去， + * 表现是「常量未被引用 → 被 esbuild tree-shake 掉 → 运行时才报找不到类型」， + * 而且不报编译错 —— 真机上就是这么静默坏掉的（OCR 与图像匹配同时失效）。 + * 断言关键标记，任何一段丢失都会立刻失败。 + */ +test('预加载段：Win32 C# 类型齐全', () => { + assert.match(WIN_HELPER_PRELUDE, /public class NAWin32/, '缺少 Win32 P/Invoke 类型'); + assert.match(WIN_HELPER_PRELUDE, /EnumWindows/, '缺 EnumWindows 声明'); +}); + +test('预加载段：UIA 程序集与查找函数齐全', () => { + assert.match(WIN_HELPER_PRELUDE, /Add-Type -AssemblyName UIAutomationClient/); + assert.match(WIN_HELPER_PRELUDE, /function Get-WinByTitle/); + assert.match(WIN_HELPER_PRELUDE, /function Find-UiaByText/, '缺子串查找函数（v14 起 body 依赖它）'); +}); + +test('预加载段：OCR（含 WinRT 与 Await 辅助）齐全', () => { + assert.match(WIN_HELPER_PRELUDE, /Add-Type -AssemblyName System\.Windows\.Forms/); + assert.match(WIN_HELPER_PRELUDE, /OcrEngine, Windows\.Foundation, ContentType=WindowsRuntime/); + assert.match(WIN_HELPER_PRELUDE, /function global:Await/); + assert.match(WIN_HELPER_PRELUDE, /\$script:ocrReady/); +}); + +test('预加载段：图像模板匹配 C# 齐全且带 -ReferencedAssemblies', () => { + assert.match(WIN_HELPER_PRELUDE, /public class NAImageMatch/); + assert.match(WIN_HELPER_PRELUDE, /public static string Match\(/); + assert.match(WIN_HELPER_PRELUDE, /public static string Capture\(/); + // Add-Type 的坑：不显式引用程序集，编译期就找不到 System.Drawing.Imaging + assert.match( + WIN_HELPER_PRELUDE, + /-ReferencedAssemblies System\.Drawing/, + '缺 -ReferencedAssemblies：C# 编译会失败（真机踩过）', + ); +}); + +test('预加载段：C# 源码必须纯 ASCII（否则 Add-Type 在中文系统上乱码）', () => { + // Add-Type 会把源码按**系统 ANSI 代码页**（中文 Windows = GBK）落盘再编译， + // C# 里的中文注释必然乱码 → 编译失败（真机踩过：整个图像匹配引擎静默不可用）。 + const csharpBlocks = WIN_HELPER_PRELUDE.match(/Add-Type @"[\s\S]*?"@/g) ?? []; + assert.ok(csharpBlocks.length > 0, '应至少有一个内嵌 C# 块'); + for (const [i, block] of csharpBlocks.entries()) { + const bad = [...new Set(block.match(/[^\x00-\x7f]/g) ?? [])]; + assert.equal( + bad.length, + 0, + `第 ${i + 1} 个 C# 块含非 ASCII 字符 ${bad.join('')} —— 必须改为英文注释`, + ); + } +}); + +test('预加载段：C# 内不得残留裸引号 JSON 字面量（TS 模板串会把 \\" 吃成 "）', () => { + // 陷阱：在 TS 模板字符串里写 \\" 会得到裸 "，C# 收到未转义引号即编译失败。 + // 正确做法是用 C# 字符常量 '"' 拼 JSON。这里断言几种典型坏形态不存在。 + const csharpBlocks = WIN_HELPER_PRELUDE.match(/Add-Type @"[\s\S]*?"@/g) ?? []; + for (const block of csharpBlocks) { + assert.ok(!/\[\{"/.test(block), 'C# 中出现 "[" + "{" 的裸引号形态，应改用字符常量拼 JSON'); + assert.ok(!/"\{[^"]*"/.test(block.replace(/"[^"]*"/g, '')), 'C# 字符串中出现未转义引号'); + } +}); + +test('预加载段：C# 内局部常量不得重复声明（编辑残留会编译失败）', () => { + const csharpBlocks = WIN_HELPER_PRELUDE.match(/Add-Type @"[\s\S]*?"@/g) ?? []; + for (const block of csharpBlocks) { + const decls = block.match(/\bQ\s*=\s*'";/g) ?? []; + assert.ok(decls.length <= 1, `Q 被声明了 ${decls.length} 次 —— 重复定义会让 C# 编译失败`); + } +}); + +test('预加载段：顺序与完整性（四段都在同一个字符串里）', () => { + const idxWin32 = WIN_HELPER_PRELUDE.indexOf('public class NAWin32'); + const idxUia = WIN_HELPER_PRELUDE.indexOf('function Get-WinByTitle'); + const idxOcr = WIN_HELPER_PRELUDE.indexOf('OcrEngine, Windows.Foundation'); + const idxImg = WIN_HELPER_PRELUDE.indexOf('public class NAImageMatch'); + for (const [name, idx] of Object.entries({ idxWin32, idxUia, idxOcr, idxImg })) { + assert.ok(idx > 0, `${name} 未包含在预加载段中`); + } + assert.ok(idxWin32 < idxUia && idxUia < idxOcr && idxOcr < idxImg, '四段顺序应为 Win32 → UIA → OCR → 图像'); +});