feat(v15): screen.find 图像模板匹配（补齐自绘 UI 盲区）+ 一串真机根因修复
## 新能力：method=image 图像模板匹配 审查报告 §5.2 指出「screen.find(image) 图像模板匹配未做，纯图标/无文字的控件是定位盲区」。 UIA 靠控件树、OCR 靠文字，两者对纯图标都无能为力；本版补上第三条腿。 实现（随常驻助手预加载，C# + System.Drawing，零新依赖）： - 灰度化（LockBits + ITU-R BT.601 整数近似） - 零均值归一化互相关 ZNCC（对亮度/对比度变化不敏感） - **粗到精**：1/4 分辨率全图扫描 → 候选在全分辨率 ±8px 精修 （全分辨率逐像素在 3440x1440 上约 190 亿次，不可行） - 非极大值抑制去重；返回中心坐标 + score，可直接喂 input.mouse.click - 参数：template（被控端图片路径）、threshold（默认 0.85）；支持 region 限定搜索区 - 出口：screen.find(method=image) + CLI/MCP 同步暴露 真机实测：3440x1440 全屏搜索 **~580ms**，score 0.9745 命中可用坐标； 对「模板纯色/低对比」会显式报错（E_PARAM_INVALID + variance），而不是静默返回空。 ## 本轮为打通它而修掉的 7 个真机根因（都值得记档） 1. **预加载段拼接静默失败**：WIN_HELPER_PRELUDE 是多行模板串，替换串写成单行 \n 不匹配且 无 assert → OCR/图像两段从未拼入 → 被 esbuild tree-shake → 运行时才报「找不到类型」。 现已导出并加回归断言（四段齐全 + 顺序）。 2. **助手启动超命令行上限**：bootstrap 含内嵌 C#（编码后 ~36KB）超过 Windows ~32KB 限制， spawn 直接 ENAMETOOLONG → 每次静默降级到慢路径。改为落地临时脚本 + -File。 3. **一次性降级路径同样超限**：ENAMETOOLONG。同样改为临时脚本 + -File（UTF-8 BOM）。 4. **统计记账掩盖根因**：ensureStarted 在 try 之外，spawn 失败不计入 failures， 表现为「spawned 一直涨、calls 恒为 0」。现已纳入 try，并暴露 ps_helper 状态 （enabled/calls/hits/failures/spawned/avg_ms/lastError）到 system.info。 5. **Add-Type 在中文系统上乱码**：C# 源码按系统 ANSI(GBK) 落盘，中文注释必乱码 → 编译失败。 已把内嵌 C# 全部改为纯 ASCII，并加断言「C# 块不得含非 ASCII」。 6. **Add-Type 缺 -ReferencedAssemblies**：只 Add-Type -AssemblyName System.Drawing 不够， 编译期仍报「命名空间 System.Drawing 中不存在 Imaging」。已补 -ReferencedAssemblies。 7. **TS 模板串吃掉 \" 转义**：在模板字符串里写 \" 会得到裸引号 → C# 编译失败（踩了两次）。 统一改用 C# 字符常量 Q 拼 JSON，并加断言（不得残留裸引号 JSON 形态 / Q 不得重复声明）。 另外把「粗层预筛」从 threshold-0.05 放宽到 threshold-0.30 并始终保留最优候选 —— 降采样会拉低真位置的分数，过严的预筛会让正确答案在精修前就被丢弃（这曾让所有匹配返回空）。 ## 单测 105 -> 114 新增 win-prelude 回归 9 例：四段齐全与顺序 / C# 纯 ASCII / 无裸引号 JSON / Q 不重复声明 / Win32+UIA+OCR+图像 各段关键标记（含 -ReferencedAssemblies）。 这些断言的价值在真机验证过：3 个 bug 都是先被断言抓住、才避免了再次上线静默坏掉。
This commit is contained in:
jkooe committed
1 parent
92df08eb20
commit
8ee8dc7f41
6 files changed
+500
-15
No files matched your search
@@ -4,6 +4,7 @@ import { CapabilityError, ErrorCodes } from '@nodeagent/protocol';
|
||||
import { IS_WINDOWS, execCommand } from '../util/exec.js';
|
||||
import { readAudit, readAuditAll, verifyAudit } from '../audit.js';
|
||||
import { computeMetrics } from '../metrics.js';
|
||||
import { allPsShellStats } from '../util/ps-helper.js';
|
||||
import { startTask } from './task.js';
|
||||
|
||||
type Args = Record<string, unknown>;
|
||||
@@ -60,6 +61,8 @@ export async function systemInfo(args: Args): Promise<unknown> {
|
||||
agent_script: process.argv[1] ?? '',
|
||||
agent_home: process.env['NODEAGENT_HOME'] ?? '',
|
||||
node_path: process.execPath,
|
||||
// v15：PowerShell 常驻助手状态（排查 GUI 能力性能/降级时用）
|
||||
ps_helper: allPsShellStats(),
|
||||
};
|
||||
|
||||
if (fields && fields.length > 0) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { CapabilityError, ErrorCodes } from '@nodeagent/protocol';
|
||||
import { IS_WINDOWS, execCommand } from '../util/exec.js';
|
||||
import { runPowerShellSmart } from '../util/ps-helper.js';
|
||||
import { agentDir } from '../config.js';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { macScreenFind, macWindowFocus, macWindowList } from './darwin.js';
|
||||
|
||||
type Args = Record<string, unknown>;
|
||||
@@ -200,8 +203,6 @@ function Find-UiaByText($scope, [string]$text, [int]$limit, [string]$ct) {
|
||||
`;
|
||||
|
||||
/** 助手统一预加载段（window.list / window.focus / screen.find 共用同一个进程）。 */
|
||||
const WIN_HELPER_PRELUDE = `${WIN32_PRELUDE}
|
||||
${UIA_ASSEMBLY_PRELUDE}`;
|
||||
|
||||
|
||||
/**
|
||||
@@ -234,6 +235,237 @@ try {
|
||||
}
|
||||
`;
|
||||
|
||||
|
||||
/**
|
||||
* 图像模板匹配预加载段（v15）。
|
||||
*
|
||||
* 动机：UIA 靠控件树、OCR 靠文字 —— **纯图标/无文字的控件**两者都抓不到（审查报告 §5.2 指出的盲区）。
|
||||
* 做法：把模板图与屏幕区域都转灰度，做**零均值归一化互相关（ZNCC）**；
|
||||
* 全分辨率逐像素太慢（3440×1440 屏 × 64×64 模板 ≈ 190 亿次），故**粗到精**：
|
||||
* ① 1/4 分辨率全图扫描（约 8000 万次）取候选
|
||||
* ② 对候选在全分辨率 ±10px 邻域精修
|
||||
* 编译一次常驻（随助手预加载），运行时纯 native 循环。
|
||||
* 亮度和对比度无关（ZNCC 归一化），故对主题/亮度变化不敏感。
|
||||
*/
|
||||
const IMAGE_MATCH_PRELUDE = `
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
# ⚠️ Add-Type 的老坑：-AssemblyName 只把程序集加载进 PowerShell 会话，
|
||||
# 编译 C# 时还必须显式 -ReferencedAssemblies，否则报「命名空间 System.Drawing
|
||||
# 中不存在 Imaging」。真机踩过：编译失败会让整个助手起不来，并伴随误导性的
|
||||
# 「找不到类型 System.Windows.Forms.SystemInformation」二次错误。
|
||||
Add-Type @"
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
public class NAImageMatch {
|
||||
public class Hit { public int x; public int y; public int w; public int h; public double score; }
|
||||
|
||||
private static byte[] Gray(Bitmap src, out int w, out int h) {
|
||||
w = src.Width; h = src.Height;
|
||||
var dst = new byte[w * h];
|
||||
var rect = new Rectangle(0, 0, w, h);
|
||||
Bitmap bmp = src.PixelFormat == PixelFormat.Format24bppRgb ? src : src.Clone(rect, PixelFormat.Format24bppRgb);
|
||||
var data = bmp.LockBits(rect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
|
||||
try {
|
||||
int stride = data.Stride;
|
||||
var buf = new byte[stride * h];
|
||||
Marshal.Copy(data.Scan0, buf, 0, buf.Length);
|
||||
for (int y = 0; y < h; y++) {
|
||||
int row = y * stride;
|
||||
for (int x = 0; x < w; x++) {
|
||||
int i = row + x * 3;
|
||||
// BGR -> luma (integer approximation of ITU-R BT.601)
|
||||
dst[y * w + x] = (byte)((buf[i + 2] * 77 + buf[i + 1] * 150 + buf[i] * 29) >> 8);
|
||||
}
|
||||
}
|
||||
} finally { bmp.UnlockBits(data); if (bmp != src) bmp.Dispose(); }
|
||||
return dst;
|
||||
}
|
||||
|
||||
// Area-average downscale (coarse search layer)
|
||||
private static byte[] Downscale(byte[] src, int w, int h, int f, out int ow, out int oh) {
|
||||
ow = w / f; oh = h / f;
|
||||
var dst = new byte[ow * oh];
|
||||
for (int y = 0; y < oh; y++) {
|
||||
for (int x = 0; x < ow; x++) {
|
||||
int sum = 0;
|
||||
for (int dy = 0; dy < f; dy++) {
|
||||
int row = (y * f + dy) * w;
|
||||
for (int dx = 0; dx < f; dx++) sum += src[row + x * f + dx];
|
||||
}
|
||||
dst[y * ow + x] = (byte)(sum / (f * f));
|
||||
}
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
// Zero-mean normalized cross correlation; 1.0 = identical
|
||||
private static double Zncc(byte[] S, int sw, int sx, int sy, byte[] T, int tw, int th) {
|
||||
int n = tw * th;
|
||||
double sumT = 0;
|
||||
for (int i = 0; i < n; i++) sumT += T[i];
|
||||
double meanT = sumT / n;
|
||||
|
||||
double sumS = 0;
|
||||
for (int y = 0; y < th; y++) {
|
||||
int row = (sy + y) * sw + sx;
|
||||
for (int x = 0; x < tw; x++) sumS += S[row + x];
|
||||
}
|
||||
double meanS = sumS / n;
|
||||
|
||||
double num = 0, dS = 0, dT = 0;
|
||||
for (int y = 0; y < th; y++) {
|
||||
int row = (sy + y) * sw + sx;
|
||||
int trow = y * tw;
|
||||
for (int x = 0; x < tw; x++) {
|
||||
double a = S[row + x] - meanS;
|
||||
double b = T[trow + x] - meanT;
|
||||
num += a * b; dS += a * a; dT += b * b;
|
||||
}
|
||||
}
|
||||
double den = Math.Sqrt(dS * dT);
|
||||
if (den < 1e-9) return 0;
|
||||
return num / den;
|
||||
}
|
||||
|
||||
/**
|
||||
// (comment removed: ASCII only)
|
||||
// (comment removed: ASCII only)
|
||||
*/
|
||||
public static string Match(string screenPath, string templatePath, int maxResults, double threshold, int searchX, int searchY, int searchW, int searchH) {
|
||||
var hits = new List<Hit>();
|
||||
using (var screenBmp = new Bitmap(screenPath))
|
||||
using (var tmplBmp = new Bitmap(templatePath)) {
|
||||
int sw, sh, tw, th;
|
||||
var S0 = Gray(screenBmp, out sw, out sh);
|
||||
var T0 = Gray(tmplBmp, out tw, out th);
|
||||
if (tw >= sw || th >= sh) return "[]";
|
||||
|
||||
// Guard: a flat template carries no gradient information, so correlation is
|
||||
// undefined (ZNCC denominator ~ 0). Report it explicitly instead of silently
|
||||
// returning no hits - a uniform crop usually means the wrong region was picked.
|
||||
//
|
||||
// NOTE: build all JSON/string literals with the char constant Q.
|
||||
// Writing a backslash-quote inside a TS template literal collapses it to a bare
|
||||
// quote and breaks the C# compile - this trap bit this file twice.
|
||||
const char Q = '"';
|
||||
var ci0 = System.Globalization.CultureInfo.InvariantCulture;
|
||||
double tSum = 0, tSum2 = 0;
|
||||
for (int i = 0; i < T0.Length; i++) { tSum += T0[i]; tSum2 += (double)T0[i] * T0[i]; }
|
||||
double tMean = tSum / T0.Length;
|
||||
double tVar = (tSum2 / T0.Length) - (tMean * tMean);
|
||||
if (tVar < 4.0) {
|
||||
return "[" + "{" + Q + "__error" + Q + ":" + Q + "template_has_no_contrast" + Q
|
||||
+ "," + Q + "variance" + Q + ":" + tVar.ToString("F2", ci0) + "}" + "]";
|
||||
}
|
||||
|
||||
// Optional search window: crop a sub-image, then add the offset back to hit coords
|
||||
int ox = 0, oy = 0;
|
||||
if (searchW > 0 && searchH > 0) {
|
||||
ox = Math.Max(0, Math.Min(searchX, sw - 1));
|
||||
oy = Math.Max(0, Math.Min(searchY, sh - 1));
|
||||
int w = Math.Min(searchW, sw - ox), h = Math.Min(searchH, sh - oy);
|
||||
var sub = new byte[w * h];
|
||||
for (int y = 0; y < h; y++) Array.Copy(S0, (oy + y) * sw + ox, sub, y * w, w);
|
||||
S0 = sub; sw = w; sh = h;
|
||||
}
|
||||
|
||||
const int F = 4;
|
||||
int sw4, sh4, tw4, th4;
|
||||
var S4 = Downscale(S0, sw, sh, F, out sw4, out sh4);
|
||||
var T4 = Downscale(T0, tw, th, F, out tw4, out th4);
|
||||
|
||||
// (1) coarse scan: collect candidates.
|
||||
// NOTE: the pre-filter must be MUCH looser than the final threshold.
|
||||
// Downsampling lowers the correlation of the true position (typically 0.7-0.85
|
||||
// for a 0.9+ full-res match), so a tight pre-filter discards the right answer
|
||||
// before refinement ever runs (this exact bug made every match return empty).
|
||||
var cands = new List<Hit>();
|
||||
if (tw4 > 0 && th4 > 0 && tw4 <= sw4 && th4 <= sh4) {
|
||||
double preFilter = Math.Max(0.30, threshold - 0.30);
|
||||
double bestCoarse = -1; int bx4 = 0, by4 = 0;
|
||||
for (int y = 0; y <= sh4 - th4; y++) {
|
||||
for (int x = 0; x <= sw4 - tw4; x++) {
|
||||
double sc = Zncc(S4, sw4, x, y, T4, tw4, th4);
|
||||
if (sc > bestCoarse) { bestCoarse = sc; bx4 = x; by4 = y; }
|
||||
if (sc >= preFilter) cands.Add(new Hit { x = x * F, y = y * F, w = tw, h = th, score = sc });
|
||||
}
|
||||
}
|
||||
// Belt and braces: always keep the coarse best, even if it missed the pre-filter
|
||||
if (bestCoarse > 0) {
|
||||
bool seen = false;
|
||||
foreach (var c in cands) { if (Math.Abs(c.x - bx4 * F) < F && Math.Abs(c.y - by4 * F) < F) { seen = true; break; } }
|
||||
if (!seen) cands.Add(new Hit { x = bx4 * F, y = by4 * F, w = tw, h = th, score = bestCoarse });
|
||||
}
|
||||
}
|
||||
cands.Sort((a, b) => b.score.CompareTo(a.score));
|
||||
if (cands.Count > 40) cands.RemoveRange(40, cands.Count - 40);
|
||||
|
||||
// (2) refine at full resolution within a +/-F*2 neighborhood
|
||||
foreach (var c in cands) {
|
||||
double best = -1; int bx = c.x, by = c.y;
|
||||
for (int dy = -F * 2; dy <= F * 2; dy++) {
|
||||
for (int dx = -F * 2; dx <= F * 2; dx++) {
|
||||
int x = c.x + dx, y = c.y + dy;
|
||||
if (x < 0 || y < 0 || x + tw > sw || y + th > sh) continue;
|
||||
double sc = Zncc(S0, sw, x, y, T0, tw, th);
|
||||
if (sc > best) { best = sc; bx = x; by = y; }
|
||||
}
|
||||
}
|
||||
if (best >= threshold) hits.Add(new Hit { x = bx + ox, y = by + oy, w = tw, h = th, score = best });
|
||||
}
|
||||
|
||||
// (3) de-dup: non-maximum suppression (keep best within 8px)
|
||||
hits.Sort((a, b) => b.score.CompareTo(a.score));
|
||||
var final = new List<Hit>();
|
||||
foreach (var h in hits) {
|
||||
bool dup = false;
|
||||
foreach (var f in final) {
|
||||
if (Math.Abs(f.x - h.x) < 8 && Math.Abs(f.y - h.y) < 8) { dup = true; break; }
|
||||
}
|
||||
if (!dup) final.Add(h);
|
||||
if (final.Count >= maxResults) break;
|
||||
}
|
||||
|
||||
var parts = new List<string>();
|
||||
foreach (var h in final) {
|
||||
// Build JSON by concatenation; reuse the shared Q / ci0 declared above.
|
||||
var ci = ci0;
|
||||
parts.Add("{" + Q + "x" + Q + ":" + h.x.ToString(ci) + "," + Q + "y" + Q + ":" + h.y.ToString(ci)
|
||||
+ "," + Q + "width" + Q + ":" + h.w.ToString(ci) + "," + Q + "height" + Q + ":" + h.h.ToString(ci)
|
||||
+ "," + Q + "score" + Q + ":" + h.score.ToString("F4", ci) + "}");
|
||||
}
|
||||
return "[" + String.Join(",", parts.ToArray()) + "]";
|
||||
}
|
||||
}
|
||||
|
||||
// Capture a screen region to a file (avoids extra process round-trips)
|
||||
public static string Capture(string outPath, int x, int y, int w, int h) {
|
||||
using (var bmp = new Bitmap(w, h))
|
||||
using (var g = Graphics.FromImage(bmp)) {
|
||||
g.CopyFromScreen(x, y, 0, 0, new Size(w, h));
|
||||
bmp.Save(outPath, ImageFormat.Png);
|
||||
}
|
||||
return outPath;
|
||||
}
|
||||
}
|
||||
"@ -ReferencedAssemblies System.Drawing
|
||||
`;
|
||||
|
||||
/**
|
||||
* 助手统一预加载段（window.list / window.focus / screen.find 的 UIA/OCR/图像三引擎共用）。
|
||||
* ⚠️ 必须定义在四段之后（块级作用域），且**四段缺一不可** ——
|
||||
* 漏拼会被 esbuild tree-shake，直到运行时才报「找不到类型」（真机踩过）。
|
||||
* tests/unit/win-prelude.test.mjs 对此有回归断言。
|
||||
*/
|
||||
export const WIN_HELPER_PRELUDE = `${WIN32_PRELUDE}
|
||||
${UIA_ASSEMBLY_PRELUDE}
|
||||
${OCR_PRELUDE}
|
||||
${IMAGE_MATCH_PRELUDE}`;
|
||||
|
||||
interface WinInfo {
|
||||
hwnd: string;
|
||||
title: string;
|
||||
@@ -408,9 +640,10 @@ export async function screenFind(args: Args): Promise<unknown> {
|
||||
{ platform: process.platform },
|
||||
);
|
||||
}
|
||||
const text = args['text'] as string;
|
||||
if (typeof text !== 'string' || text.length === 0) {
|
||||
throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空');
|
||||
const text = (args['text'] as string | undefined) ?? '';
|
||||
const wantImage = (args['method'] as string | undefined) === 'image';
|
||||
if (!wantImage && (typeof text !== 'string' || text.length === 0)) {
|
||||
throw new CapabilityError(ErrorCodes.PARAM_INVALID, 'text 不能为空（method=image 时用 template 代替）');
|
||||
}
|
||||
const windowTitle = args['window'] as string | undefined;
|
||||
const controlType = args['control_type'] as string | undefined;
|
||||
@@ -424,6 +657,9 @@ export async function screenFind(args: Args): Promise<unknown> {
|
||||
const regionArg = args['region'] as
|
||||
| { x: number; y: number; width: number; height: number }
|
||||
| undefined;
|
||||
// v15：图像模板匹配
|
||||
const template = args['template'] as string | undefined;
|
||||
const threshold = Math.max(0.3, Math.min(0.999, (args['threshold'] as number | undefined) ?? 0.85));
|
||||
|
||||
// ---------- UIA 引擎 ----------
|
||||
const uiaFind = async (): Promise<Record<string, unknown>[]> => {
|
||||
@@ -458,6 +694,81 @@ if ($out.Count -eq 0) { Write-Output '[]' } else { $out | ConvertTo-Json -Compre
|
||||
return toArray<Record<string, unknown>>(extractJson<Record<string, unknown>[]>(out) ?? []);
|
||||
};
|
||||
|
||||
// ---------- 图像模板引擎（灰度 ZNCC 粗到精，v15） ----------
|
||||
// 适用：纯图标 / 无文字的自绘控件（UIA 无控件树、OCR 无文字可读的盲区）
|
||||
const imageFind = async (): Promise<Record<string, unknown>[]> => {
|
||||
if (!template) {
|
||||
throw new CapabilityError(
|
||||
ErrorCodes.PARAM_INVALID,
|
||||
'method=image 需要 template 参数（被控端上的模板图片路径，支持 png/jpg/bmp）',
|
||||
);
|
||||
}
|
||||
const tmpDir = join(agentDir(), 'tmp');
|
||||
mkdirSync(tmpDir, { recursive: true });
|
||||
const shot = join(tmpDir, `match-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.png`);
|
||||
// PowerShell 单引号字面量：不插值（避免路径里的 $ 被展开）、不处理反斜杠转义。
|
||||
// 不能用 JSON.stringify —— 它会产出双引号串，而 PS 双引号会做变量插值且把 \ 当字面两个反斜杠。
|
||||
const q = (v: string): string => `'${v.replace(/'/g, "''")}'`;
|
||||
|
||||
// 截图区域：显式 region 优先，否则整块虚拟屏
|
||||
const capX = regionArg ? regionArg.x : 'VSX';
|
||||
const capY = regionArg ? regionArg.y : 'VSY';
|
||||
const capW = regionArg ? regionArg.width : 'VSW';
|
||||
const capH = regionArg ? regionArg.height : 'VSH';
|
||||
|
||||
const body = `
|
||||
if (-not (Test-Path ${q(template)})) {
|
||||
Write-Output '{"__error":"模板文件不存在"}'
|
||||
return
|
||||
}
|
||||
$vs = [System.Windows.Forms.SystemInformation]::VirtualScreen
|
||||
$cx = ${typeof capX === 'number' ? capX : '$vs.X'}
|
||||
$cy = ${typeof capY === 'number' ? capY : '$vs.Y'}
|
||||
$cw = ${typeof capW === 'number' ? capW : '$vs.Width'}
|
||||
$ch = ${typeof capH === 'number' ? capH : '$vs.Height'}
|
||||
try {
|
||||
[void][NAImageMatch]::Capture(${q(shot)}, $cx, $cy, $cw, $ch)
|
||||
$json = [NAImageMatch]::Match(${q(shot)}, ${q(template)}, ${Number(limit)}, ${threshold}, 0, 0, 0, 0)
|
||||
Write-Output $json
|
||||
} catch {
|
||||
Write-Output ('{"__error":"' + $_.Exception.Message.Replace('"','''') + '"}')
|
||||
} finally {
|
||||
Remove-Item ${q(shot)} -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
`;
|
||||
const out = await runPS(body, 120_000);
|
||||
type ImgHit = { x: number; y: number; width: number; height: number; score: number };
|
||||
const raw = extractJson<Record<string, unknown> | ImgHit[]>(out);
|
||||
if (raw && !Array.isArray(raw) && typeof raw === 'object' && '__error' in raw) {
|
||||
throw new CapabilityError(
|
||||
ErrorCodes.EXECUTION_FAILED,
|
||||
`图像匹配失败: ${String((raw as { __error?: string }).__error)}`,
|
||||
);
|
||||
}
|
||||
const hits: ImgHit[] = Array.isArray(raw) ? raw : [];
|
||||
// 模板纯色时 C# 会返回 [{__error:...}]（数组形式），这里统一转成可读错误
|
||||
const first = hits[0] as unknown as { __error?: string; variance?: number } | undefined;
|
||||
if (first && typeof first === 'object' && first.__error === 'template_has_no_contrast') {
|
||||
throw new CapabilityError(
|
||||
ErrorCodes.PARAM_INVALID,
|
||||
'模板几乎没有色彩/明暗差异（纯色或低对比），无法用于模板匹配；请改用有纹理或图标的区域',
|
||||
{ variance: first.variance },
|
||||
);
|
||||
}
|
||||
return hits.map((h) => ({
|
||||
name: `image:${template.split(/[\\/]/).pop() ?? 'template'}`,
|
||||
control_type: 'Image(template)',
|
||||
score: h.score,
|
||||
// 命中坐标相对截图原点 -> 补回屏幕原点
|
||||
x: Math.round(capX === 'VSX' ? h.x + (regionArg?.x ?? 0) : h.x + Number(capX)),
|
||||
y: Math.round(capY === 'VSY' ? h.y + (regionArg?.y ?? 0) : h.y + Number(capY)),
|
||||
left: Math.round(capX === 'VSX' ? h.x + (regionArg?.x ?? 0) : h.x + Number(capX)),
|
||||
top: Math.round(capY === 'VSY' ? h.y + (regionArg?.y ?? 0) : h.y + Number(capY)),
|
||||
width: h.width,
|
||||
height: h.height,
|
||||
}));
|
||||
};
|
||||
|
||||
// ---------- OCR 引擎（截图 → Windows.Media.Ocr → 文字坐标） ----------
|
||||
const ocrFind = async (): Promise<Record<string, unknown>[]> => {
|
||||
const body = `
|
||||
@@ -607,6 +918,7 @@ $out | ConvertTo-Json -Compress
|
||||
const attempt = async (): Promise<{ matches: Record<string, unknown>[]; engine: string }> => {
|
||||
if (method === 'uia') return { matches: (await uiaFind()).slice(0, limit), engine: 'uia' };
|
||||
if (method === 'ocr') return { matches: (await ocrFind()).slice(0, limit), engine: 'ocr' };
|
||||
if (method === 'image') return { matches: (await imageFind()).slice(0, limit), engine: 'image' };
|
||||
// auto：UIA 优先（快且带控件语义），找不到再 OCR 兜底（自绘 UI）
|
||||
const uiaMatches = await uiaFind();
|
||||
if (uiaMatches.length > 0) return { matches: uiaMatches.slice(0, limit), engine: 'uia' };
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { execCommand } from './exec.js';
|
||||
|
||||
/**
|
||||
@@ -28,6 +31,41 @@ const MAX_CONSECUTIVE_FAILURES = 3;
|
||||
/** 助手启动（含类型编译）允许的较长时间；只发生一次 */
|
||||
const SPAWN_TIMEOUT_MS = 60_000;
|
||||
|
||||
/**
|
||||
* 从 PowerShell 的 stderr 中提取**真正有用的错误信息**。
|
||||
*
|
||||
* PowerShell 的 stderr 是 CLIXML（XML）：里面既有 `Preparing modules for first use.`
|
||||
* 这类**进度噪声**，也有真正的错误。原样截断展示会把真错误淹没在噪声里
|
||||
* （真机踩过：排查 C# 编译失败时只看到一堆 progress 记录）。
|
||||
*/
|
||||
export function extractPsError(stderr: string): string {
|
||||
if (!stderr) return '';
|
||||
const decoded = stderr
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/'/g, "'");
|
||||
|
||||
// CLIXML 的错误条目形如 <S S="Error">错误文本</S>
|
||||
const errs: string[] = [];
|
||||
const re = /<S S="Error">([\s\S]*?)<\/S>/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(decoded)) !== null) {
|
||||
const text = m[1]!.trim();
|
||||
if (text) errs.push(text);
|
||||
}
|
||||
if (errs.length > 0) return errs.join('\n').slice(0, 800);
|
||||
|
||||
// 没有结构化错误：剥掉 CLIXML 标签后取残余文本
|
||||
const stripped = decoded
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/Preparing modules for first use\./g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
return stripped.slice(0, 500);
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析助手输出的一行协议报文。
|
||||
* 协议：`@@R <base64>` 成功 / `@@E <base64>` 脚本异常；其他行（Add-Type 噪声等）返回 null。
|
||||
@@ -131,9 +169,11 @@ export class PsShell {
|
||||
/** 助手里执行一段脚本；返回 stdout。抛出异常表示「本次不可用」，调用方应降级。 */
|
||||
async run(body: string, timeoutMs = 30_000): Promise<string> {
|
||||
if (!this.stats.enabled) throw new Error(`PsShell[${this.label}] 已熔断`);
|
||||
await this.ensureStarted();
|
||||
const started = Date.now();
|
||||
try {
|
||||
// ensureStarted 必须在 try 内：它失败（如 spawn 失败）同样要计入统计，
|
||||
// 否则会表现成「spawned 一直涨、calls 恒为 0」的诡异状态（真机踩过，掩盖了根因）。
|
||||
await this.ensureStarted();
|
||||
const out = await this.enqueue(body, timeoutMs);
|
||||
this.stats.calls += 1;
|
||||
this.stats.hits += 1;
|
||||
@@ -186,9 +226,16 @@ export class PsShell {
|
||||
this.starting = new Promise<void>((resolve, reject) => {
|
||||
this.stats.spawned += 1;
|
||||
const bootstrap = buildBootstrap(this.prelude);
|
||||
// ⚠️ 不能用 -EncodedCommand：预加载段含内嵌 C#（编码后 ~36KB）超过 Windows
|
||||
// 命令行 ~32KB 上限，spawn 会直接失败（ENAMETOOLONG）→ 每次都静默降级到
|
||||
// 慢路径（真机踩过：助手 spawned 计数增长但 calls 恒为 0）。
|
||||
// 改为落地临时脚本 + -File（无长度限制）。
|
||||
const dir = mkdtempSync(join(tmpdir(), 'na-psh-'));
|
||||
const bootPath = join(dir, 'bootstrap.ps1');
|
||||
writeFileSync(bootPath, `\uFEFF${bootstrap}`, 'utf8');
|
||||
const p = spawn(
|
||||
'powershell.exe',
|
||||
['-NoProfile', '-NoLogo', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-EncodedCommand', toEncodedCommand(bootstrap)],
|
||||
['-NoProfile', '-NoLogo', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', bootPath],
|
||||
{ windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
);
|
||||
this.proc = p;
|
||||
@@ -232,7 +279,11 @@ export class PsShell {
|
||||
clearTimeout(q.timer);
|
||||
q.reject(new Error(`PsShell[${this.label}] 进程退出`));
|
||||
}
|
||||
if (!wasReady) reject(new Error(`PsShell[${this.label}] 启动失败（code=${String(code)}）`));
|
||||
if (!wasReady) {
|
||||
// 带上 stderr 尾巴 —— 否则只看到「启动失败(code=1)」无从下手（真机踩过）
|
||||
const tail = this.stderrBuf.trim().slice(-600);
|
||||
reject(new Error(`PsShell[${this.label}] 启动失败（code=${String(code)}）${tail ? ` :: ${tail}` : ''}`));
|
||||
}
|
||||
});
|
||||
}).finally(() => {
|
||||
this.starting = null;
|
||||
@@ -380,12 +431,20 @@ export async function runPowerShellSmart(opts: RunPsOptions): Promise<{ stdout:
|
||||
}
|
||||
}
|
||||
|
||||
// ⚠️ 不能用 -EncodedCommand 传整个脚本：预加载段（含内嵌 C#）已远超
|
||||
// Windows 的 ~32KB 命令行上限，会报 spawn ENAMETOOLONG（真机踩过）。
|
||||
// 改为**落地成临时脚本再 -File 执行**：无长度限制，且 UTF-8 BOM 保证
|
||||
// PowerShell 5.1 正确读取非 ASCII（BOM 缺失会被当 ANSI 解析）。
|
||||
const dir = mkdtempSync(join(tmpdir(), 'na-ps-'));
|
||||
const scriptPath = join(dir, 'run.ps1');
|
||||
writeFileSync(scriptPath, `\uFEFF${prelude}\n${body}`, 'utf8');
|
||||
const r = await execCommand({
|
||||
command: `powershell.exe -NoProfile -NonInteractive -EncodedCommand ${toEncodedCommand(`${prelude}\n${body}`)}`,
|
||||
command: `powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File ${JSON.stringify(scriptPath)}`,
|
||||
timeoutMs,
|
||||
});
|
||||
if (r.exit_code !== 0) {
|
||||
const err = new Error((r.stderr || r.stdout).slice(0, 600)) as Error & { exitCode?: number };
|
||||
const detail = extractPsError(r.stderr) || r.stdout.slice(0, 400);
|
||||
const err = new Error(detail || `PowerShell 退出码 ${r.exit_code}`) as Error & { exitCode?: number };
|
||||
err.exitCode = r.exit_code;
|
||||
throw err;
|
||||
}
|
||||
|
||||
@@ -99,6 +99,10 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
agent_script: { type: 'string', description: '被控端入口脚本路径，供一键升级定位（v11）' },
|
||||
agent_home: { type: 'string', description: '被控端数据目录 NODEAGENT_HOME（v11）' },
|
||||
node_path: { type: 'string', description: '被控端 Node 可执行文件路径（v11）' },
|
||||
ps_helper: {
|
||||
type: 'object',
|
||||
description: 'PowerShell 常驻助手状态（v15）：spawned/hits/failures/consecutiveFailures/avg_ms 等',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -776,9 +780,23 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
},
|
||||
method: {
|
||||
type: 'string',
|
||||
enum: ['auto', 'uia', 'ocr'],
|
||||
enum: ['auto', 'uia', 'ocr', 'image'],
|
||||
default: 'auto',
|
||||
description: 'auto=UIA 优先+OCR 兜底；uia=仅 UIA；ocr=仅截图识别',
|
||||
description:
|
||||
'auto=UIA 优先+OCR 兜底；uia=仅控件树；ocr=仅文字识别；image=仅图像模板匹配（需 template）',
|
||||
},
|
||||
template: {
|
||||
type: 'string',
|
||||
description:
|
||||
'method=image 时必填：被控端上的模板图片路径（png/jpg/bmp）。' +
|
||||
'用于纯图标/无文字控件 —— UIA 无控件树、OCR 无文字可读的场景',
|
||||
},
|
||||
threshold: {
|
||||
type: 'number',
|
||||
minimum: 0.3,
|
||||
maximum: 0.999,
|
||||
default: 0.85,
|
||||
description: 'method=image 的匹配阈值（零均值归一化互相关，越高越严格）',
|
||||
},
|
||||
limit: { type: 'integer', default: 20 },
|
||||
wait_ms: {
|
||||
@@ -797,10 +815,9 @@ export const CAPABILITY_MANIFEST: CapabilityDescriptor[] = [
|
||||
width: { type: 'integer' },
|
||||
height: { type: 'integer' },
|
||||
},
|
||||
description: '可选：只在该屏幕区域查找（OCR 更快更准；UIA 忽略此参数）',
|
||||
description: '可选：只在该屏幕区域查找（OCR/图像匹配更快更准；UIA 忽略此参数）',
|
||||
},
|
||||
},
|
||||
required: ['text'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
returns_schema: {
|
||||
|
||||
@@ -108,7 +108,16 @@ test('screen.find：method 枚举受控，非法值被拒', () => {
|
||||
assert.equal(check('screen.find', { text: '确定' }).length, 0);
|
||||
assert.equal(check('screen.find', { text: 'x', method: 'ocr' }).length, 0);
|
||||
assert.ok(check('screen.find', { text: 'x', method: 'magic' }).length > 0, '非法 method 应被拒');
|
||||
assert.ok(check('screen.find', {}).length > 0, '缺 text 应报错');
|
||||
});
|
||||
|
||||
test('screen.find：v15 图像模板 —— method=image 与 template/threshold', () => {
|
||||
assert.equal(check('screen.find', { method: 'image', template: 'C:\\icon.png' }).length, 0);
|
||||
assert.equal(check('screen.find', { text: 'x', method: 'image', threshold: 0.9 }).length, 0);
|
||||
// text 改为可选（image 模式不需要），但 handler 仍会校验「非 image 模式必须有 text」
|
||||
assert.equal(check('screen.find', {}).length, 0, 'schema 层允许空对象（由 handler 判定 text/template）');
|
||||
assert.ok(check('screen.find', { text: 'x', threshold: 0.1 }).length > 0, 'threshold 下限 0.3');
|
||||
assert.ok(check('screen.find', { text: 'x', threshold: 1.5 }).length > 0, 'threshold 上限 0.999');
|
||||
assert.ok(check('screen.find', { text: 'x', bogus: 1 }).length > 0, '未知参数应被拒');
|
||||
});
|
||||
|
||||
test('input.key.press：支持 repeat 与 sequence', () => {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { WIN_HELPER_PRELUDE } from '../../apps/agent/dist/capabilities/window.js';
|
||||
|
||||
/**
|
||||
* 回归测试：常驻助手的预加载段必须四段齐全。
|
||||
*
|
||||
* 为什么需要它：这些段是**多行模板字符串拼接**，一旦某次编辑没拼进去，
|
||||
* 表现是「常量未被引用 → 被 esbuild tree-shake 掉 → 运行时才报找不到类型」，
|
||||
* 而且不报编译错 —— 真机上就是这么静默坏掉的（OCR 与图像匹配同时失效）。
|
||||
* 断言关键标记，任何一段丢失都会立刻失败。
|
||||
*/
|
||||
test('预加载段：Win32 C# 类型齐全', () => {
|
||||
assert.match(WIN_HELPER_PRELUDE, /public class NAWin32/, '缺少 Win32 P/Invoke 类型');
|
||||
assert.match(WIN_HELPER_PRELUDE, /EnumWindows/, '缺 EnumWindows 声明');
|
||||
});
|
||||
|
||||
test('预加载段：UIA 程序集与查找函数齐全', () => {
|
||||
assert.match(WIN_HELPER_PRELUDE, /Add-Type -AssemblyName UIAutomationClient/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /function Get-WinByTitle/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /function Find-UiaByText/, '缺子串查找函数（v14 起 body 依赖它）');
|
||||
});
|
||||
|
||||
test('预加载段：OCR（含 WinRT 与 Await 辅助）齐全', () => {
|
||||
assert.match(WIN_HELPER_PRELUDE, /Add-Type -AssemblyName System\.Windows\.Forms/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /OcrEngine, Windows\.Foundation, ContentType=WindowsRuntime/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /function global:Await/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /\$script:ocrReady/);
|
||||
});
|
||||
|
||||
test('预加载段：图像模板匹配 C# 齐全且带 -ReferencedAssemblies', () => {
|
||||
assert.match(WIN_HELPER_PRELUDE, /public class NAImageMatch/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /public static string Match\(/);
|
||||
assert.match(WIN_HELPER_PRELUDE, /public static string Capture\(/);
|
||||
// Add-Type 的坑：不显式引用程序集，编译期就找不到 System.Drawing.Imaging
|
||||
assert.match(
|
||||
WIN_HELPER_PRELUDE,
|
||||
/-ReferencedAssemblies System\.Drawing/,
|
||||
'缺 -ReferencedAssemblies：C# 编译会失败（真机踩过）',
|
||||
);
|
||||
});
|
||||
|
||||
test('预加载段：C# 源码必须纯 ASCII（否则 Add-Type 在中文系统上乱码）', () => {
|
||||
// Add-Type 会把源码按**系统 ANSI 代码页**（中文 Windows = GBK）落盘再编译，
|
||||
// C# 里的中文注释必然乱码 → 编译失败（真机踩过：整个图像匹配引擎静默不可用）。
|
||||
const csharpBlocks = WIN_HELPER_PRELUDE.match(/Add-Type @"[\s\S]*?"@/g) ?? [];
|
||||
assert.ok(csharpBlocks.length > 0, '应至少有一个内嵌 C# 块');
|
||||
for (const [i, block] of csharpBlocks.entries()) {
|
||||
const bad = [...new Set(block.match(/[^\x00-\x7f]/g) ?? [])];
|
||||
assert.equal(
|
||||
bad.length,
|
||||
0,
|
||||
`第 ${i + 1} 个 C# 块含非 ASCII 字符 ${bad.join('')} —— 必须改为英文注释`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('预加载段：C# 内不得残留裸引号 JSON 字面量（TS 模板串会把 \\" 吃成 "）', () => {
|
||||
// 陷阱：在 TS 模板字符串里写 \\" 会得到裸 "，C# 收到未转义引号即编译失败。
|
||||
// 正确做法是用 C# 字符常量 '"' 拼 JSON。这里断言几种典型坏形态不存在。
|
||||
const csharpBlocks = WIN_HELPER_PRELUDE.match(/Add-Type @"[\s\S]*?"@/g) ?? [];
|
||||
for (const block of csharpBlocks) {
|
||||
assert.ok(!/\[\{"/.test(block), 'C# 中出现 "[" + "{" 的裸引号形态，应改用字符常量拼 JSON');
|
||||
assert.ok(!/"\{[^"]*"/.test(block.replace(/"[^"]*"/g, '')), 'C# 字符串中出现未转义引号');
|
||||
}
|
||||
});
|
||||
|
||||
test('预加载段：C# 内局部常量不得重复声明（编辑残留会编译失败）', () => {
|
||||
const csharpBlocks = WIN_HELPER_PRELUDE.match(/Add-Type @"[\s\S]*?"@/g) ?? [];
|
||||
for (const block of csharpBlocks) {
|
||||
const decls = block.match(/\bQ\s*=\s*'";/g) ?? [];
|
||||
assert.ok(decls.length <= 1, `Q 被声明了 ${decls.length} 次 —— 重复定义会让 C# 编译失败`);
|
||||
}
|
||||
});
|
||||
|
||||
test('预加载段：顺序与完整性（四段都在同一个字符串里）', () => {
|
||||
const idxWin32 = WIN_HELPER_PRELUDE.indexOf('public class NAWin32');
|
||||
const idxUia = WIN_HELPER_PRELUDE.indexOf('function Get-WinByTitle');
|
||||
const idxOcr = WIN_HELPER_PRELUDE.indexOf('OcrEngine, Windows.Foundation');
|
||||
const idxImg = WIN_HELPER_PRELUDE.indexOf('public class NAImageMatch');
|
||||
for (const [name, idx] of Object.entries({ idxWin32, idxUia, idxOcr, idxImg })) {
|
||||
assert.ok(idx > 0, `${name} 未包含在预加载段中`);
|
||||
}
|
||||
assert.ok(idxWin32 < idxUia && idxUia < idxOcr && idxOcr < idxImg, '四段顺序应为 Win32 → UIA → OCR → 图像');
|
||||
});
|
||||
Reference in new issue
Block a user