test: 新增自动化真机验证体系

- tests/e2e/windows.mjs: Windows 专属能力测试(service.list / app.list / winget 装软件 / cmd 外壳)
- tests/e2e/run.mjs: service.list 断言改为平台感知(Windows 期望成功)
- scripts/verify.mjs: 远程验收脚本,对齐 PRD FR-01~FR-08 逐条验收 + Markdown 报告
- package.json: 新增 test:windows / verify 脚本
- README: 补充自动化真机验证三层体系说明

注: .github/workflows/windows-e2e.yml 需 workflow scope,待授权后单独提交
This commit is contained in:
jkooe committed 2026-09-27 01:24:52 +08:00
1 parent 77c024703a
commit f1506b3fd0
5 files changed
+546 -4

No files matched your search

+20
View File
@@ -87,8 +87,28 @@ MCP 工具:`na_status` · `na_system_info` · `na_exec` · `na_install` · `na
pnpm build # 构建全部包
pnpm typecheck # 类型检查
pnpm test:e2e # 端到端测试(本机起 Agent,跑通握手 + 能力调用 + 异常路径)
pnpm test:windows # Windows 专属能力测试(服务 / 软件 / winget 装软件)
pnpm verify # 对本机配置的被控端跑全套验收并输出报告
```
## 自动化真机验证
三层验证体系,无需手工点测:
| 层次 | 方式 | 覆盖 |
|---|---|---|
| **CI(推荐)** | GitHub Actions `windows-latest` | 真实 Windows 上跑协议 E2E、Windows 专属能力(`Get-Service`/注册表/winget 装软件)、`install.ps1` 安装链路、CLI 连入、MCP 工具列表 |
| **远程验收** | `node scripts/verify.mjs --host <IP> --key <密钥> [--insecure]` | 对齐 PRD FR-01~FR-08 的逐条验收,输出终端报告 + `--report report.md` |
| **本地测试** | `pnpm test:e2e` / `pnpm test:windows` | 单机自举:起一个 Agent 当被控端,自测协议与能力 |
CI 工作流见 [`.github/workflows/windows-e2e.yml`](./.github/workflows/windows-e2e.yml),每次 push 到 `main` 或手动触发即运行。
```bash
# 对皇上自己的 Windows 机器验收(含真实装软件)
node scripts/verify.mjs --host 192.168.1.100 --port 8765 --key <密钥> --insecure --with-install jqlang.jq --report verify-report.md
```
### 工程结构
```
+3 -1
View File
@@ -14,7 +14,9 @@
"agent": "pnpm --filter @nodeagent/agent start",
"cli": "pnpm --filter @nodeagent/cli start",
"mcp": "pnpm --filter @nodeagent/mcp start",
"test:e2e": "node tests/e2e/run.mjs"
"test:e2e": "node tests/e2e/run.mjs",
"test:windows": "node tests/e2e/windows.mjs",
"verify": "node scripts/verify.mjs"
},
"devDependencies": {
"@types/node": "^22.10.2",
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env node
/**
* nodeagent 远程验收脚本
*
* 对一台可达的 Windows 被控端执行全套验收(对齐 PRD 的 FR-01 ~ FR-08),输出报告。
*
* 用法:
* node scripts/verify.mjs # 使用本机已保存的配置
* node scripts/verify.mjs --host 192.168.1.100 --port 8765 --key <key> [--insecure]
* node scripts/verify.mjs --with-install jqlang.jq # 额外执行装软件验收(会真实安装)
* node scripts/verify.mjs --report report.md # 同时写出 Markdown 报告
*/
import { writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { NodeAgentClient, loadConfig, toWsUrl } from '../packages/client/dist/index.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
// ---------- 参数 ----------
function parseArgs(argv) {
const out = { flags: new Set(), values: {} };
for (let i = 0; i < argv.length; i += 1) {
const a = argv[i];
if (!a.startsWith('--')) continue;
const key = a.slice(2);
const next = argv[i + 1];
if (next && !next.startsWith('--')) {
out.values[key] = next;
i += 1;
} else {
out.flags.add(key);
}
}
return out;
}
const { flags, values } = parseArgs(process.argv.slice(2));
let conn;
if (values['host']) {
conn = {
host: values['host'],
port: Number(values['port'] ?? 8765),
key: values['key'] ?? '',
client_id: values['id'] ?? 'verify',
tls: !flags.has('no-tls'),
insecure: flags.has('insecure') || flags.has('no-tls'),
};
if (!conn.key) {
console.error('✗ 使用 --host 时必须同时提供 --key <预共享密钥>');
process.exit(1);
}
} else {
const saved = loadConfig();
if (!saved) {
console.error('✗ 未找到本机配置。请先执行: nodeagent connect <host> --port 8765 --key <密钥>');
process.exit(1);
}
conn = saved;
}
// ---------- 结果收集 ----------
const results = [];
async function check(id, name, fn) {
const t0 = Date.now();
try {
const detail = await fn();
if (detail === 'skip') {
results.push({ id, name, status: 'skip', detail: '不适用' });
console.log(` - ${id} ${name} [跳过]`);
} else {
results.push({ id, name, status: 'pass', detail: detail ?? '' , ms: Date.now() - t0 });
console.log(` ✓ ${id} ${name}${detail ? ` — ${detail}` : ''}`);
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
results.push({ id, name, status: 'fail', detail: msg });
console.log(` ✗ ${id} ${name}\n ${msg}`);
}
}
async function connect(key = conn.key) {
const c = new NodeAgentClient({
url: toWsUrl(conn),
key,
clientId: conn.client_id ?? 'verify',
insecure: conn.insecure,
});
await c.connect();
return c;
}
// ---------- 验收主体 ----------
async function main() {
const startedAt = new Date();
console.log('\nnodeagent 验收报告');
console.log('─'.repeat(52));
console.log(` 目标 : ${toWsUrl(conn)}`);
console.log(` 时间 : ${startedAt.toLocaleString('zh-CN')}`);
console.log('─'.repeat(52));
// 探测被控端平台
let platform = 'unknown';
{
const c = await connect();
const r = await c.invoke('system.info');
if (r.status === 'ok') platform = String(r.data.os ?? 'unknown');
c.close();
}
const isWindows = /windows/i.test(platform);
console.log(` 被控端 : ${platform}${isWindows ? '' : ' ⚠️ 非 Windows,部分验收将跳过'}\n`);
console.log('[FR-01] 连接与鉴权');
await check('FR-01.1', '握手成功并获取能力清单', async () => {
const c = await connect();
const n = c.listCapabilities().length;
c.close();
if (n === 0) throw new Error('能力清单为空');
return `${n} 项能力`;
});
await check('FR-01.2', '错误密钥被拒绝', async () => {
try {
const c = await connect('definitely-wrong-key');
c.close();
throw new Error('错误密钥竟然通过了鉴权');
} catch (err) {
if (err.message === '错误密钥竟然通过了鉴权') throw err;
if (err.code !== -32401 && err.name !== 'E_AUTH_FAILED') {
throw new Error(`期望 E_AUTH_FAILED,实际 ${err.name}: ${err.message}`);
}
return 'E_AUTH_FAILED';
}
});
console.log('\n[FR-04] 状态查询');
await check('FR-04.1', 'system.info 系统信息', async () => {
const c = await connect();
const r = await c.invoke('system.info');
c.close();
assertOk(r, 'system.info');
return `${r.data.hostname} · ${r.data.os} · ${r.data.cpu_cores} 核`;
});
await check('FR-04.2', 'system.status 资源状态', async () => {
const c = await connect();
const r = await c.invoke('system.status');
c.close();
assertOk(r, 'system.status');
if (typeof r.data.cpu_pct !== 'number') throw new Error('缺少 cpu_pct');
if (!(r.data.memory_total > 0)) throw new Error('memory_total 无效');
if (!Array.isArray(r.data.disks) || r.data.disks.length === 0) throw new Error('磁盘列表为空');
return `CPU ${r.data.cpu_pct}% · 内存 ${r.data.memory_pct}% · ${r.data.disks.length} 个磁盘`;
});
await check('FR-04.3', 'system.process.list 进程列表', async () => {
const c = await connect();
const r = await c.invoke('system.process.list', { limit: 10 });
c.close();
assertOk(r, 'system.process.list');
if (r.data.processes.length === 0) throw new Error('进程列表为空');
return `${r.data.processes.length} 个进程`;
});
await check('FR-04.4', 'system.service.list 服务列表', async () => {
if (!isWindows) return 'skip';
const c = await connect();
const r = await c.invoke('system.service.list', { limit: 20 });
c.close();
assertOk(r, 'system.service.list');
if (r.data.services.length === 0) throw new Error('服务列表为空');
return `${r.data.services.length} 个服务`;
});
console.log('\n[FR-02] 命令执行');
await check('FR-02.1', '执行命令并返回退出码', async () => {
const c = await connect();
const cmd = isWindows ? 'Write-Output verify-ok' : 'echo verify-ok';
const r = await c.invoke('system.shell.exec', { command: cmd });
c.close();
assertOk(r, 'system.shell.exec');
if (r.data.exit_code !== 0) throw new Error(`退出码 ${r.data.exit_code}`);
if (!/verify-ok/.test(r.data.stdout)) throw new Error(`输出异常: ${r.data.stdout}`);
return `${r.data.duration_ms}ms`;
});
await check('FR-02.2', '非零退出码正确回传', async () => {
const c = await connect();
const r = await c.invoke('system.shell.exec', { command: 'exit 7' });
c.close();
assertOk(r, 'system.shell.exec');
if (r.data.exit_code !== 7) throw new Error(`期望 7,实际 ${r.data.exit_code}`);
return 'exit_code=7';
});
await check('FR-02.3', '超时命令被强制终止', async () => {
const c = await connect();
const r = await c.invoke('system.shell.exec', { command: 'sleep 60', timeout_ms: 1500 }, 30_000);
c.close();
assertOk(r, 'system.shell.exec');
if (r.data.killed !== true) throw new Error('未标记 killed=true');
return `killed=true, exit=${r.data.exit_code}`;
});
console.log('\n[FR-05] 错误反馈');
await check('FR-05.1', '未知能力返回明确错误码', async () => {
const c = await connect();
try {
await c.invoke('no.such.capability');
c.close();
throw new Error('未知能力未报错');
} catch (err) {
c.close();
if (err.code !== -32403) throw new Error(`期望 -32403,实际 ${err.code}`);
return 'E_CAPABILITY_NOT_FOUND';
}
});
await check('FR-05.2', '参数校验拦截缺参调用', async () => {
const c = await connect();
try {
await c.invoke('system.shell.exec', {});
c.close();
throw new Error('缺参未报错');
} catch (err) {
c.close();
if (err.code !== -32602) throw new Error(`期望 -32602,实际 ${err.code}`);
return 'E_PARAM_INVALID';
}
});
console.log('\n[FR-03] 软件管理');
await check('FR-03.1', 'app.list 列出已安装软件', async () => {
if (!isWindows) return 'skip';
const c = await connect();
const r = await c.invoke('app.list');
c.close();
assertOk(r, 'app.list');
return `${r.data.apps.length} 款软件`;
});
const installPkg = values['with-install'];
if (installPkg) {
await check('FR-03.2', `app.install 安装 ${installPkg}`, async () => {
const c = await connect();
const r = await c.invoke('app.install', { package: installPkg, timeout_ms: 600_000 }, 660_000);
c.close();
assertOk(r, 'app.install');
if (!r.data.installed) throw new Error(`安装失败:\n${r.data.detail}`);
return `${r.data.name} ${r.data.version || ''}`.trim();
});
} else {
results.push({ id: 'FR-03.2', name: 'app.install 安装软件', status: 'skip', detail: '需 --with-install <包名>' });
console.log(' - FR-03.2 app.install 安装软件 [跳过:需 --with-install <包名>]');
}
// ---------- 汇总 ----------
const pass = results.filter((r) => r.status === 'pass').length;
const fail = results.filter((r) => r.status === 'fail').length;
const skip = results.filter((r) => r.status === 'skip').length;
const finishedAt = new Date();
console.log('\n' + '─'.repeat(52));
console.log(` 汇总: ${pass} 通过 / ${fail} 失败 / ${skip} 跳过 (耗时 ${((finishedAt - startedAt) / 1000).toFixed(1)}s)`);
console.log('─'.repeat(52) + '\n');
const reportPath = values['report'];
if (reportPath) {
const md = buildReport(conn, platform, results, startedAt, finishedAt);
writeFileSync(reportPath, md);
console.log(`报告已写入: ${reportPath}\n`);
}
process.exit(fail > 0 ? 1 : 0);
}
function assertOk(r, cap) {
if (r.status !== 'ok') {
throw new Error(`${cap} 执行失败: ${r.error?.name}: ${r.error?.message}`);
}
}
function buildReport(conn, platform, results, startedAt, finishedAt) {
const pass = results.filter((r) => r.status === 'pass').length;
const fail = results.filter((r) => r.status === 'fail').length;
const skip = results.filter((r) => r.status === 'skip').length;
const icon = (s) => (s === 'pass' ? '✅' : s === 'fail' ? '❌' : '⏭️');
return [
'# nodeagent 验收报告',
'',
'| 项 | 值 |',
'|---|---|',
`| 被控端 | \`${toWsUrl(conn)}\` |`,
`| 平台 | ${platform} |`,
`| 开始 | ${startedAt.toLocaleString('zh-CN')} |`,
`| 结束 | ${finishedAt.toLocaleString('zh-CN')} |`,
`| 结果 | **${pass} 通过 / ${fail} 失败 / ${skip} 跳过** |`,
'',
'## 明细',
'',
'| # | 验收项 | 结果 | 说明 |',
'|---|---|---|---|',
...results.map((r) => `| ${r.id} | ${r.name} | ${icon(r.status)} ${r.status} | ${r.detail.replace(/\n/g, ' ')} |`),
'',
'> 本报告由 `scripts/verify.mjs` 自动生成,仅用于功能验收,不构成任何安全承诺。',
'',
].join('\n');
}
main().catch((err) => {
console.error(`\n✗ 验收中断: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
});
+9 -3
View File
@@ -204,12 +204,18 @@ async function main() {
c.close();
});
await test('平台不支持的能力返回 status=failed + E_UNSUPPORTED_PLATFORM', async () => {
await test('平台相关能力:Windows 成功 / 非 Windows 报 E_UNSUPPORTED_PLATFORM', async () => {
const c = await connect();
const r = await c.invoke('system.service.list');
c.close();
assert.equal(r.status, 'failed');
assert.equal(r.error.name, 'E_UNSUPPORTED_PLATFORM');
if (process.platform === 'win32') {
assert.equal(r.status, 'ok', 'Windows 上 service.list 应成功');
assert.ok(Array.isArray(r.data.services), 'services 应为数组');
assert.ok(r.data.services.length > 0, 'Windows 上应有服务');
} else {
assert.equal(r.status, 'failed');
assert.equal(r.error.name, 'E_UNSUPPORTED_PLATFORM');
}
});
await test('未认证调用返回 E_AUTH_REQUIRED', async () => {
+202
View File
@@ -0,0 +1,202 @@
/**
* nodeagent Windows 专属能力 E2E
*
* 仅在 Windows 上运行:验证 system.service.list / app.list / app.install(winget)等
* 只在 Windows 被控端可用的能力。
*
* 运行:node tests/e2e/windows.mjs
*/
import { spawn } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import assert from 'node:assert/strict';
import { NodeAgentClient } from '../../packages/client/dist/index.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const root = join(__dirname, '../..');
const PORT = 18768;
const KEY = 'windows-e2e-key-0123456789abcdef0123456789abcdef';
const URL = `ws://127.0.0.1:${PORT}`;
let passed = 0;
let failed = 0;
let skipped = 0;
async function test(name, fn) {
try {
const r = await fn();
if (r === 'skip') {
console.log(` - ${name} [skip]`);
skipped += 1;
} else {
console.log(` \u2713 ${name}`);
passed += 1;
}
} catch (err) {
console.log(` \u2717 ${name}`);
console.log(` ${err instanceof Error ? err.message : String(err)}`);
failed += 1;
}
}
function startAgent() {
const home = mkdtempSync(join(tmpdir(), 'nodeagent-win-e2e-'));
mkdirSync(join(home, '.nodeagent'), { recursive: true });
writeFileSync(
join(home, '.nodeagent', 'agent.json'),
JSON.stringify({ node_id: 'win_e2e', host: '127.0.0.1', port: PORT, tls: false, key: KEY, log_level: 'warn' }),
);
const child = spawn(process.execPath, [join(root, 'apps/agent/dist/index.js')], {
// Windows 上 os.homedir() 读 USERPROFILE
env: { ...process.env, HOME: home, USERPROFILE: home },
stdio: ['ignore', 'pipe', 'pipe'],
});
child.stderr.on('data', (d) => process.stderr.write(`[agent] ${d}`));
return child;
}
async function waitReady(timeoutMs = 20_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
try {
const c = new NodeAgentClient({ url: URL, key: KEY, clientId: 'probe' });
await c.connect();
c.close();
return;
} catch {
await new Promise((r) => setTimeout(r, 300));
}
}
throw new Error('被控端未在超时内就绪');
}
async function connect() {
const c = new NodeAgentClient({ url: URL, key: KEY, clientId: 'win_e2e_mac' });
await c.connect();
return c;
}
async function main() {
if (process.platform !== 'win32') {
console.log('\n非 Windows 平台,跳过 Windows 专属测试。');
process.exit(0);
}
console.log('\nnodeagent Windows 专属能力 E2E\n');
const agent = startAgent();
try {
await waitReady();
console.log(` 被控端已就绪 ${URL}\n`);
await test('system.info 报告 Windows 平台', async () => {
const c = await connect();
const r = await c.invoke('system.info');
c.close();
assert.equal(r.status, 'ok');
assert.match(String(r.data.os), /Windows/i, `os 字段异常: ${r.data.os}`);
assert.ok(r.data.cpu_cores > 0);
assert.ok(r.data.memory_total > 0);
});
await test('system.status 返回 Windows 磁盘(盘符)', async () => {
const c = await connect();
const r = await c.invoke('system.status');
c.close();
assert.equal(r.status, 'ok');
assert.ok(Array.isArray(r.data.disks) && r.data.disks.length > 0, 'disks 为空');
assert.match(String(r.data.disks[0].drive), /^[A-Z]:/i, `盘符格式异常: ${r.data.disks[0].drive}`);
});
await test('system.service.list 返回 Windows 服务', async () => {
const c = await connect();
const r = await c.invoke('system.service.list', { limit: 50 });
c.close();
assert.equal(r.status, 'ok');
assert.ok(r.data.services.length > 0, '服务列表为空');
const s = r.data.services[0];
assert.ok(s.name && s.state, `服务字段异常: ${JSON.stringify(s)}`);
});
await test('system.service.list 支持按名称筛选', async () => {
const c = await connect();
const r = await c.invoke('system.service.list', { filter: { name_pattern: 'win' } });
c.close();
assert.equal(r.status, 'ok');
assert.ok(r.data.services.every((s) => /win/i.test(s.name) || /win/i.test(s.display_name)));
});
await test('system.process.list 返回进程(含 CPU 百分比)', async () => {
const c = await connect();
const r = await c.invoke('system.process.list', { limit: 10 });
c.close();
assert.equal(r.status, 'ok');
assert.ok(r.data.processes.length > 0, '进程列表为空');
assert.equal(typeof r.data.processes[0].cpu_pct, 'number');
});
await test('system.shell.exec 执行 PowerShell 命令', async () => {
const c = await connect();
const r = await c.invoke('system.shell.exec', { command: 'Get-Date -Format "yyyy-MM-dd"', shell: 'powershell' });
c.close();
assert.equal(r.status, 'ok');
assert.equal(r.data.exit_code, 0);
assert.match(r.data.stdout, /\d{4}-\d{2}-\d{2}/);
});
await test('system.shell.exec 支持 cmd 外壳', async () => {
const c = await connect();
const r = await c.invoke('system.shell.exec', { command: 'echo cmd-ok', shell: 'cmd' });
c.close();
assert.equal(r.status, 'ok');
assert.equal(r.data.exit_code, 0);
assert.match(r.data.stdout, /cmd-ok/);
});
await test('app.list 返回已安装软件(注册表)', async () => {
const c = await connect();
const r = await c.invoke('app.list');
c.close();
assert.equal(r.status, 'ok');
assert.ok(Array.isArray(r.data.apps), 'apps 应为数组');
assert.ok(r.data.apps.length > 0, '未读到任何已安装软件');
});
await test('app.install 用 winget 安装软件', async () => {
const c = await connect();
const probe = await c.invoke('system.shell.exec', { command: 'winget --version' });
if (probe.status !== 'ok' || probe.data.exit_code !== 0) {
c.close();
return 'skip'; // CI 环境无 winget
}
const r = await c.invoke('app.install', { package: 'jqlang.jq', timeout_ms: 600_000 }, 660_000);
c.close();
assert.equal(r.status, 'ok', `app.install 协议层失败: ${JSON.stringify(r.error)}`);
assert.equal(r.data.installed, true, `winget 安装失败:\n${r.data.detail}`);
});
await test('安装后可在 app.list 中检索到', async () => {
const c = await connect();
const r = await c.invoke('app.list', { filter: { name_pattern: 'jq' } });
c.close();
assert.equal(r.status, 'ok');
// jq 由 winget 通过 MSI 安装,注册表中应可见;找不到则视为跳过(安装方式差异)
if (!r.data.apps.some((a) => /jq/i.test(a.name))) return 'skip';
assert.ok(true);
});
} finally {
agent.kill();
await new Promise((r) => setTimeout(r, 300));
}
console.log(`\n结果: ${passed} 通过, ${failed} 失败, ${skipped} 跳过\n`);
process.exit(failed > 0 ? 1 : 0);
}
main().catch((err) => {
console.error('测试异常:', err);
process.exit(1);
});