Compare commits

...
26 changed files with 1858 additions and 707 deletions

No files matched your search

+42 -1
View File
@@ -1,6 +1,7 @@
import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { inflateRawSync } from "node:zlib";
import { DiagnosticLogger } from "@one-memory/runtime-control";
import { afterEach, expect, it, vi } from "vitest";
@@ -29,6 +30,27 @@ async function fixture() {
return dataDir;
}
function readZipEntry(archive: Buffer, expected: string): string {
let offset = 0;
while (archive.readUInt32LE(offset) === 0x04034b50) {
const method = archive.readUInt16LE(offset + 8);
const compressedSize = archive.readUInt32LE(offset + 18);
const nameLength = archive.readUInt16LE(offset + 26);
const extraLength = archive.readUInt16LE(offset + 28);
const nameStart = offset + 30;
const dataStart = nameStart + nameLength + extraLength;
const name = archive.subarray(nameStart, nameStart + nameLength).toString();
const compressed = archive.subarray(dataStart, dataStart + compressedSize);
if (name === expected) {
if (method === 0) return compressed.toString();
if (method === 8) return inflateRawSync(compressed).toString();
throw new Error("unsupported zip compression");
}
offset = dataStart + compressedSize;
}
throw new Error(`missing ${expected}`);
}
it("prints matching diagnostic events as NDJSON without starting Runtime", async () => {
const dataDir = await fixture();
let stdout = "";
@@ -74,6 +96,25 @@ it("exports one support zip to the requested downloads directory", async () => {
});
expect(receipt.path).toMatch(/MemHub-日志-3800-\d{8}-\d{4}\.zip$/);
expect(await readdir(downloads)).toEqual([path.basename(receipt.path)]);
const archive = await readFile(receipt.path);
expect(JSON.parse(readZipEntry(archive, "summary.json")).commit).toBeNull();
});
it("fails closed for an explicitly supplied invalid commit", async () => {
const dataDir = await fixture();
const downloads = await mkdtemp(path.join(os.tmpdir(), "om-cli-downloads-"));
roots.push(downloads);
vi.spyOn(process.stdout, "write").mockImplementation(() => true);
await expect(
run(["logs", "export", "--json"], {
ONE_MEMORY_DATA_DIR: dataDir,
ONE_MEMORY_EXPORT_DIR: downloads,
ONE_MEMORY_BUILD_NUMBER: "3800",
ONE_MEMORY_COMMIT: "unknown",
}),
).rejects.toThrow("diagnostic_commit_invalid");
expect(await readdir(downloads)).toEqual([]);
});
it("never emits a secret from a malformed manually written event", async () => {
+11 -5
View File
@@ -1048,15 +1048,21 @@ async function runCommand(
entry: "cli",
}));
const target = path.join(downloads, diagnosticZipFilename(build));
const environmentCommit = environment.ONE_MEMORY_COMMIT?.trim();
const embeddedCommit =
typeof __ONE_MEMORY_COMMIT__ === "string"
? __ONE_MEMORY_COMMIT__.trim()
: "";
const commit =
environmentCommit ||
(embeddedCommit && embeddedCommit !== "unknown"
? embeddedCommit
: undefined);
const receipt = await exportDiagnosticZip({
dataDir: config.dataDir,
target,
productVersion: build,
commit:
environment.ONE_MEMORY_COMMIT?.trim() ||
(typeof __ONE_MEMORY_COMMIT__ === "string"
? __ONE_MEMORY_COMMIT__
: "unknown"),
...(commit === undefined ? {} : { commit }),
coreSha256: await sha256File(config.coreCommand).catch(() => undefined),
home: os.homedir(),
crashReportsDir: path.join(
+31 -1
View File
@@ -1,4 +1,4 @@
import { mkdtemp, rm } from "node:fs/promises";
import { mkdtemp, rm, symlink } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { DiagnosticLogger } from "@one-memory/runtime-control";
@@ -41,11 +41,15 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
session_id: "11111111-1111-4111-8111-111111111111",
details: {
count: 1,
reason: "/Users/me/private-query.txt",
status: "sk-live-secret-token",
source_id: "credential.private",
prompt: "secret-token-123",
authorization: "Bearer secret-token-123",
} as unknown as Record<string, string | number | boolean | null>,
safe_stack: [
"packages/runtime-control/src/example.ts:42:7",
"packages/runtime/sk-live-secret-token.ts:1:1",
"/Users/me/private.py:1",
"C:\\Users\\me\\private.py:1",
],
@@ -56,6 +60,8 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
"packages/runtime-control/src/example.ts:42:7",
]);
expect(JSON.stringify(redacted)).not.toContain("secret-token-123");
expect(JSON.stringify(redacted)).not.toContain("sk-live-secret-token");
expect(JSON.stringify(redacted)).not.toContain("credential.private");
expect(JSON.stringify(redacted)).not.toContain("/Users/me");
expect(JSON.stringify(redacted)).not.toContain("C:\\\\Users");
});
@@ -132,3 +138,27 @@ it("follows a newly appended event without stopping Runtime", async () => {
});
expect(seen).toEqual(["service.ready", "core.failed"]);
});
it("fails follow mode when the diagnostic directory is unsafe", async () => {
const dataDir = await mkdtemp(
path.join(os.tmpdir(), "om-cli-follow-unsafe-"),
);
const external = await mkdtemp(
path.join(os.tmpdir(), "om-cli-follow-external-"),
);
roots.push(dataDir, external);
await symlink(external, path.join(dataDir, "logs"));
const query = parseLogQuery(["--follow"]);
expect(query).not.toBeNull();
if (!query) throw new Error("query_missing");
await expect(
followLogs({
dataDir,
query,
signal: new AbortController().signal,
maxEvents: 1,
emit: () => undefined,
wait: async () => undefined,
}),
).rejects.toThrow("log_directory_unreadable");
});
+6 -102
View File
@@ -3,6 +3,7 @@ import path from "node:path";
import type { DiagnosticEventV1 } from "@one-memory/runtime-control";
import {
diagnosticLogFiles,
projectDiagnosticEventForSafeOutput,
readDiagnosticEvents,
} from "@one-memory/runtime-control";
@@ -24,54 +25,6 @@ export interface LogQuery {
const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
const NAME = /^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$/;
const SAFE_DETAIL_KEYS = new Set([
"action",
"attempt",
"count",
"method",
"mode",
"phase",
"platform",
"reason",
"result_count",
"source_id",
"spawned",
"status",
"version",
"parent_session_id",
"invalid_lines",
"truncated",
"runtime_state",
"core_state",
"broker_state",
"instance_id",
"open_file_count",
"memory_bytes",
"thread_count",
"database_error_count",
"file_count",
"sqlite_file_count",
"byte_file_count",
"total_bytes",
"copied_bytes",
"bytes_per_second",
"stored_count",
"revised_count",
"skipped_count",
"error_count",
"pending_count",
"prepared_count",
"queue_depth",
"in_flight_count",
"call_count",
"created_count",
"updated_count",
"superseded_count",
"deleted_count",
"operation_kind",
"file_kind",
]);
export function parseLogQuery(arguments_: readonly string[]): LogQuery | null {
if (arguments_.filter((item) => item === "--json").length > 1) return null;
const values: Partial<LogQuery> = { errorsOnly: false, follow: false };
@@ -237,63 +190,13 @@ export function renderLogText(
return `${lines.join("\n")}\n`;
}
/** Defence in depth for the CLI read surface. `DiagnosticLogger` already
* allow-lists keys, but a future writer, a manually edited log, or a malformed
* line must not turn `memhub logs` into an exfiltration path. */
/** Defence in depth for the CLI read surface. Local writers accept arbitrary
* bounded scalar keys, so every outward read surface must reuse the same
* field-and-value projection as the support-bundle exporter. */
export function redactDiagnosticEvent(
event: DiagnosticEventV1,
): DiagnosticEventV1 {
const redacted: DiagnosticEventV1 = {
schema_version: 1,
timestamp: event.timestamp,
level: event.level,
service: event.service,
component: event.component,
event: event.event,
outcome: event.outcome,
session_id: event.session_id,
...(event.trace_id === undefined ? {} : { trace_id: event.trace_id }),
...(event.incident_id === undefined
? {}
: { incident_id: event.incident_id }),
...(event.instance_id === undefined
? {}
: { instance_id: event.instance_id }),
...(event.operation_id === undefined
? {}
: { operation_id: event.operation_id }),
...(event.request_id === undefined ? {} : { request_id: event.request_id }),
...(event.job_id === undefined ? {} : { job_id: event.job_id }),
...(event.task_id === undefined ? {} : { task_id: event.task_id }),
...(event.duration_ms === undefined
? {}
: { duration_ms: event.duration_ms }),
...(event.state_before === undefined
? {}
: { state_before: event.state_before }),
...(event.state_after === undefined
? {}
: { state_after: event.state_after }),
...(event.error_code === undefined ? {} : { error_code: event.error_code }),
...(event.error_type === undefined ? {} : { error_type: event.error_type }),
...(event.safe_stack === undefined
? {}
: {
safe_stack: event.safe_stack.filter(
(frame) =>
!frame.startsWith("/") &&
!/^[A-Za-z]:\\/.test(frame) &&
!frame.includes(".."),
),
}),
};
const details = Object.fromEntries(
Object.entries(event.details ?? {}).filter(([key]) =>
SAFE_DETAIL_KEYS.has(key),
),
);
if (Object.keys(details).length > 0) redacted.details = details;
return redacted;
return projectDiagnosticEventForSafeOutput(event);
}
export async function followLogs(options: {
@@ -315,6 +218,7 @@ export async function followLogs(options: {
...options.query,
tail: 5_000,
});
if (report.failure) throw new Error(report.failure);
for (const event of report.events) {
const identity = JSON.stringify(event);
if (seen.has(identity)) continue;
+24 -38
View File
@@ -998,9 +998,7 @@ def create_app(
app.state.recall_target = GovernedRecallTarget(
app.state.memory_governance_store,
filter_memories=app.state.source_read_access.memories,
semantic_search=lambda query: _sdk_memory_runtime(
app
).semantic_recall(query),
semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
# Read per request rather than cached: an import that lands
# while the app is open should make its workspace reachable
# without a restart. Affordable only because the read is served
@@ -1102,9 +1100,7 @@ def create_app(
authorize_generation=lambda profile, question, grounding: (
_authorize_answer_generation(app, profile, question, grounding)
),
semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(
query
),
semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
resolve_workspace=lambda workspace_path: resolve_workspace_scopes(
workspace_path,
evidence_store.known_workspaces(user_id="local"),
@@ -1128,21 +1124,21 @@ def create_app(
profiles=app.state.model_profile_store,
secrets=app.state.secret_store,
data_dir=runtime_settings.data_dir,
transport_factory=lambda profile: lambda: AuditedModelTransport(
app.state.model_request_audit_store,
profile_id=profile.profile_id,
profile_revision=profile.revision,
provider_kind=str(profile.provider_kind),
model=profile.model,
inner=(
BudgetedModelTransport(model_request_budget)
if model_request_budget is not None
else None
),
transport_factory=lambda profile: (
lambda: AuditedModelTransport(
app.state.model_request_audit_store,
profile_id=profile.profile_id,
profile_revision=profile.revision,
provider_kind=str(profile.provider_kind),
model=profile.model,
inner=(
BudgetedModelTransport(model_request_budget)
if model_request_budget is not None
else None
),
)
),
sedimentation_enabled=lambda: _config_store(
app
).sedimentation_enabled(),
sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(),
)
app.state.sdk_memory_runtime = SdkMemoryRuntime(
coordinator=app.state.sdk_write_coordinator,
@@ -1165,10 +1161,8 @@ def create_app(
exclude_error_codes=NON_RETRYABLE_POST_WRITE_CODES,
)
for job in unknown:
receipt = (
await app.state.sdk_write_coordinator.inquire_write_outcome(
job.request_id
)
receipt = await app.state.sdk_write_coordinator.inquire_write_outcome(
job.request_id
)
if receipt is None:
continue
@@ -1608,9 +1602,7 @@ def create_app(
)
if getattr(
app.state, "memory_authority", MemoryAuthority.LEGACY
) is MemoryAuthority.SDK and request.url.path.startswith(
legacy_paths
):
) is MemoryAuthority.SDK and request.url.path.startswith(legacy_paths):
return JSONResponse(
status_code=409,
content={"detail": "sdk_memory_authority_required"},
@@ -1732,7 +1724,7 @@ def create_app(
trace_id=trace_id,
instance_id=runtime_settings.instance_id,
duration_ms=elapsed_ms(started),
details={**diagnostic_details, "status": str(response.status_code)},
details={**diagnostic_details, "status_code": response.status_code},
)
return response
except Exception as error:
@@ -3781,9 +3773,7 @@ def create_app(
all_memories, projected, nodes, edges = await asyncio.to_thread(read_governed_part)
try:
hy_graph = (
None
if revoked
else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
None if revoked else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
)
except HySdkError:
hy_graph = None
@@ -3892,9 +3882,7 @@ def create_app(
active_queue, in_flight_count = supervisor.active_depths()
planned, current = supervisor.manual_batch_progress()
sdk_counts = (
supervisor.status_counts()
if app.state.memory_authority is MemoryAuthority.SDK
else {}
supervisor.status_counts() if app.state.memory_authority is MemoryAuthority.SDK else {}
)
# `running` counts rows the store still marks running, which includes
# work a worker has already given up on. Reporting that as "what is
@@ -3915,9 +3903,7 @@ def create_app(
),
recent_completions=depths.recent_completions,
running=sdk_counts.get("running", running_rows),
stranded_running=(
0 if sdk_counts else max(0, running_rows - in_flight_count)
),
stranded_running=(0 if sdk_counts else max(0, running_rows - in_flight_count)),
unexecuted_policy_failures=store.unexecuted_policy_failures(),
waiting_for_model=sdk_counts.get(
"waiting_for_model",
@@ -5284,7 +5270,7 @@ def _report_visibility_failure(phase: str, error: BaseException) -> None:
event="memory_visibility.background_failed",
outcome="failed",
error_type=type(error).__name__,
details={"phase": phase, "reason": str(error)[:200]},
details={"phase": phase},
)
+217 -122
View File
@@ -20,8 +20,10 @@ from __future__ import annotations
import atexit
import json
import logging
import math
import os
import re
import stat
import sys
import threading
import time
@@ -72,80 +74,30 @@ class _Refused(OSError):
"""The path is not one we are willing to write to, and never will be."""
DiagnosticLevel = Literal["info", "warn", "error"]
DiagnosticLevel = Literal["debug", "info", "warn", "error"]
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
_EVENT = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$")
_DETAIL_KEYS = frozenset(
{
"action",
"attempt",
"count",
"method",
"mode",
"phase",
"platform",
"reason",
"result_count",
"source_id",
"spawned",
"status",
"version",
"parent_session_id",
"invalid_lines",
"truncated",
"open_file_count",
"memory_bytes",
"thread_count",
"database_error_count",
"route_path",
"sqlite_errorcode",
"sqlite_errorname",
"file_count",
"sqlite_file_count",
"byte_file_count",
"total_bytes",
"copied_bytes",
"bytes_per_second",
"stored_count",
"revised_count",
"skipped_count",
"error_count",
"pending_count",
"prepared_count",
"queue_depth",
"in_flight_count",
"call_count",
"created_count",
"updated_count",
"superseded_count",
"deleted_count",
"operation_kind",
"index_complete",
"lexical_hits",
"semantic_hits",
"generation",
"timed_out",
"failure_site",
"error_class",
"file_kind",
"file_size",
"file_mtime_ns",
"file_inode",
"cursor_new",
"cursor_old",
"cursor_read_to_end",
"cursor_rewind_branch",
"summed_ms",
"model_ms",
"accounted_ms",
"unaccounted_ms",
"conversation_hash",
# The sedimentation switch: a bounded boolean, the same shape as
# `spawned` and `index_complete`. Left out of this list it did not
# drop a log line, it turned the switch's own request into a 500.
"enabled",
}
_DIAGNOSTIC_LOG_NAME = re.compile(
r"^diagnostic-(runtime|core|host-bridge|desktop|cli)\.jsonl(?:\.\d+)?$"
)
_DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
_CODE_FRAME = re.compile(
r"^(?:(?:apps|packages|scripts)/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:\d+"
r"|one_memory/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:[A-Za-z_][A-Za-z0-9_]*"
r"|[A-Za-z0-9_][A-Za-z0-9_.-]*\.py:\d+:[A-Za-z_][A-Za-z0-9_]*)$"
)
_SHORT_CREDENTIAL_TOKEN = re.compile(
r"(?<![A-Za-z0-9])([A-Za-z0-9]{24,47})(?:[_-])?(?![A-Za-z0-9])"
)
_REFUSED_DETAIL_KEYS = frozenset({"__proto__", "prototype", "constructor"})
_MAX_DETAIL_KEYS = 32
_MAX_SAFE_INTEGER = (1 << 53) - 1
_EXPORT_SECRET_MARKER = re.compile(
r"(?:^|[._:-])(?:authorization|credential|password|prompt|query|secret|token)(?:$|[._:-])",
re.IGNORECASE,
)
_TRACE_STRING_DETAIL_KEYS = frozenset({"phase", "mode", "status", "reason", "failure_site"})
_TRACE_NUMBER_DETAIL_KEYS = frozenset({"count"})
_trace_id: ContextVar[str | None] = ContextVar("one_memory_trace_id", default=None)
_active: DiagnosticLog | None = None
_database_errors: Counter[tuple[str, str]] = Counter()
@@ -264,7 +216,12 @@ class DiagnosticLog:
}
value.update({key: _identifier(item) for key, item in identifiers.items() if item})
if duration_ms is not None:
if not isinstance(duration_ms, int) or duration_ms < 0:
if (
not isinstance(duration_ms, int)
or isinstance(duration_ms, bool)
or duration_ms < 0
or duration_ms > _MAX_SAFE_INTEGER
):
raise ValueError("diagnostic_duration_invalid")
value["duration_ms"] = duration_ms
for field_name, item in {
@@ -276,7 +233,9 @@ class DiagnosticLog:
if item is not None:
value[field_name] = _identifier(item)
if safe_stack:
value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack[:256]]
if len(safe_stack) > 16:
raise ValueError("diagnostic_stack_invalid")
value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack]
if details:
value["details"] = _details(details)
self._submit(_serialize(value))
@@ -289,6 +248,8 @@ class DiagnosticLog:
event: str,
outcome: str,
) -> dict[str, object]:
if level not in ("debug", "info", "warn", "error"):
raise ValueError("diagnostic_level_invalid")
return {
"schema_version": 1,
"timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
@@ -651,24 +612,17 @@ def _drain_diagnostics_at_exit() -> None:
def diagnostic_event(**values: object) -> None:
active = _active
if active is None:
return
if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
return
try:
active = _active
if active is None:
return
if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
return
active.write(**values) # type: ignore[arg-type]
except OSError:
# Diagnostics must not become a new product outage when a disk is full
# or the log directory is temporarily unavailable.
return
except ValueError:
# Same promise, same reason, for a rejected payload instead of a failed
# write. An unlisted `details` key once turned every external recall
# into a 500 and then killed the process, and the sedimentation switch
# into a permanent failure. The payload itself is still wrong and is
# still caught in CI by tests/test_diagnostic_payload_keys.py; what
# must not happen is the caller paying for it.
except Exception:
# Diagnostics are best-effort: validation, serialization, logging, and
# filesystem failures must never become a product outage. BaseException
# deliberately remains visible so process-control signals still work.
return
@@ -691,46 +645,95 @@ def read_trace_events(
if (request_id is None) == (trace_id is None):
raise ValueError("exactly_one_trace_identity_required")
key, expected = ("request_id", request_id) if request_id is not None else ("trace_id", trace_id)
if expected is None or not _IDENTIFIER.fullmatch(expected):
if expected is None or not _export_identifier(expected):
raise ValueError("invalid_trace_identity")
selected: list[dict[str, object]] = []
allowed_details = {"phase", "mode", "status", "reason", "failure_site", "count"}
files = sorted(
(data_dir / "logs").glob("diagnostic-*.jsonl*"),
key=lambda path: path.stat().st_mtime,
)
for path in files:
if path.is_symlink() or not path.is_file():
directory = data_dir / "logs"
try:
directory_info = directory.lstat()
except FileNotFoundError:
return ()
if stat.S_ISLNK(directory_info.st_mode) or not stat.S_ISDIR(directory_info.st_mode):
raise ValueError("diagnostic_directory_unsafe")
files: list[tuple[Path, float]] = []
for path in directory.iterdir():
if not _DIAGNOSTIC_LOG_NAME.fullmatch(path.name):
continue
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
try:
info = path.lstat()
except OSError:
continue
if stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode):
files.append((path, info.st_mtime))
for path, _mtime in sorted(files, key=lambda item: item[1]):
for line in _read_regular_file_no_follow(path).splitlines():
try:
event = json.loads(line)
except (TypeError, json.JSONDecodeError):
continue
if not isinstance(event, dict) or event.get(key) != expected:
continue
details = event.get("details")
projected = {
name: event[name]
for name in (
"timestamp",
"component",
"event",
"outcome",
"duration_ms",
"error_code",
"error_type",
)
if name in event
timestamp = event.get("timestamp")
if not isinstance(timestamp, str):
continue
try:
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
except ValueError:
continue
component = event.get("component")
event_name = event.get("event")
outcome = event.get("outcome")
projected: dict[str, object] = {
"timestamp": timestamp,
"component": (
component
if isinstance(component, str) and _export_identifier(component)
else "diagnostic.redacted"
),
"event": (
event_name
if isinstance(event_name, str) and _export_identifier(event_name)
else "diagnostic.redacted"
),
"outcome": (
outcome
if isinstance(outcome, str) and _export_identifier(outcome)
else "redacted"
),
}
duration_ms = event.get("duration_ms")
if (
isinstance(duration_ms, int)
and not isinstance(duration_ms, bool)
and 0 <= duration_ms <= _MAX_SAFE_INTEGER
):
projected["duration_ms"] = duration_ms
for name in ("error_code", "error_type"):
value = event.get(name)
if isinstance(value, str) and _export_identifier(value):
projected[name] = value
details = event.get("details")
if isinstance(details, dict):
projected["details"] = {
name: value for name, value in details.items() if name in allowed_details
}
safe_details = _trace_details(details)
if safe_details:
projected["details"] = safe_details
selected.append(projected)
return tuple(selected[-max(1, min(limit, 500)) :])
def _read_regular_file_no_follow(path: Path) -> str:
descriptor = os.open(path, os.O_RDONLY | _O_NOFOLLOW)
try:
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
raise ValueError("diagnostic_file_unsafe")
with os.fdopen(descriptor, encoding="utf-8", errors="replace") as handle:
descriptor = -1
return handle.read()
finally:
if descriptor >= 0:
os.close(descriptor)
def current_trace_id() -> str | None:
return _trace_id.get()
@@ -837,8 +840,77 @@ def diagnostic_exception(error: BaseException) -> dict[str, object]:
}
def _contains_credential_material(value: str) -> bool:
return bool(
re.search(
r"(?:^|[^A-Za-z0-9])(?:sk-|sk_|omhg_|ghp_|github_pat_)[A-Za-z0-9._-]+",
value,
re.IGNORECASE,
)
or re.search(r"(?:^|[^A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]+", value, re.IGNORECASE)
or re.search(r"(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])", value)
or re.search(r"(?:^|[^A-Za-z0-9])AKID[A-Za-z0-9]{12,}", value, re.IGNORECASE)
or re.search(r"(?:^|[^A-Za-z0-9])AIza[A-Za-z0-9_-]{20,}", value)
or re.search(r"(?:^|[^A-Za-z0-9])GOCSPX-[A-Za-z0-9_-]+", value)
or any(
_is_conservative_short_credential(match.group(1))
for match in _SHORT_CREDENTIAL_TOKEN.finditer(value)
)
or re.search(
r"\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b",
value,
)
or any(
len(candidate) >= 48
and any(character.isalpha() for character in candidate)
and any(character.isdigit() for character in candidate)
for candidate in re.findall(r"\b[A-Za-z0-9._-]{48,}\b", value)
)
)
def _is_conservative_short_credential(value: str) -> bool:
return bool(
re.search(r"[A-Z]", value)
and re.search(r"[a-z]", value)
and re.search(r"\d", value)
and not re.fullmatch(r"[A-Fa-f0-9]+", value)
)
def _export_identifier(value: str) -> bool:
return bool(
_IDENTIFIER.fullmatch(value)
and not _EXPORT_SECRET_MARKER.search(value)
and not _contains_credential_material(value)
)
def _trace_details(values: Mapping[object, object]) -> dict[str, str | int | float]:
result: dict[str, str | int | float] = {}
for key, value in values.items():
if (
key in _TRACE_STRING_DETAIL_KEYS
and isinstance(value, str)
and _export_identifier(value)
):
result[str(key)] = value
elif key in _TRACE_NUMBER_DETAIL_KEYS and isinstance(value, (int, float)):
if isinstance(value, bool):
continue
safe_integer = isinstance(value, int) and abs(value) <= _MAX_SAFE_INTEGER
safe_float = (
isinstance(value, float)
and math.isfinite(value)
and (not value.is_integer() or abs(value) <= _MAX_SAFE_INTEGER)
)
if safe_integer or safe_float:
result[str(key)] = value
return result
def _serialize(value: Mapping[str, object]) -> str:
return json.dumps(value, ensure_ascii=True, separators=(",", ":")) + "\n"
return json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")) + "\n"
def _uuid(value: str | None) -> str | None:
@@ -864,24 +936,47 @@ def _event_name(value: str) -> str:
def _safe_stack_line(value: str) -> str:
if not value or len(value) > 256 or "\n" in value or "\r" in value:
if (
not value
or len(value) > 256
or any(character in value for character in "\0\r\n")
or _contains_credential_material(value)
):
raise ValueError("diagnostic_stack_invalid")
if Path(value.split(":", 1)[0]).is_absolute() or "file://" in value:
raise ValueError("diagnostic_stack_invalid")
return value
if _CODE_FRAME.fullmatch(value):
return value
if value.startswith("thread|"):
frames = value.removeprefix("thread|").split(" > ")
if frames and all(_CODE_FRAME.fullmatch(frame) for frame in frames):
return value
raise ValueError("diagnostic_stack_invalid")
def _details(
values: Mapping[str, str | int | float | bool | None],
) -> dict[str, str | int | float | bool | None]:
if len(values) > _MAX_DETAIL_KEYS:
raise ValueError("diagnostic_detail_count_invalid")
result: dict[str, str | int | float | bool | None] = {}
for key, value in values.items():
if key not in _DETAIL_KEYS:
if (
not isinstance(key, str)
or not _DETAIL_KEY.fullmatch(key)
or key in _REFUSED_DETAIL_KEYS
):
raise ValueError("diagnostic_detail_key_invalid")
if isinstance(value, str):
if not value or len(value) > 256 or "\n" in value or "\r" in value:
if not value or len(value) > 256 or any(character in value for character in "\0\r\n"):
raise ValueError("diagnostic_detail_value_invalid")
elif value is not None and not isinstance(value, (int, float, bool)):
elif isinstance(value, float):
if not math.isfinite(value) or (value.is_integer() and abs(value) > _MAX_SAFE_INTEGER):
raise ValueError("diagnostic_detail_value_invalid")
elif isinstance(value, bool) or value is None:
pass
elif isinstance(value, int):
if abs(value) > _MAX_SAFE_INTEGER:
raise ValueError("diagnostic_detail_value_invalid")
else:
raise ValueError("diagnostic_detail_value_invalid")
result[key] = value
return result
@@ -3,8 +3,8 @@
Every site that catches `Exception` broadly must call `report_exception`
before (or instead of) continuing, so a failure never looks like success
and a stack is always on disk. Content (conversation text, keys, SQL
parameters) never reaches the log: only exception type, a truncated
message, and file:line:function frames.
parameters, and exception messages) never reaches the diagnostic log: only
exception type, a repeat count, and file:line:function frames.
"""
from __future__ import annotations
@@ -67,7 +67,7 @@ def report_exception(
event=event,
outcome="failed",
**payload,
details={"reason": _safe_message(error), "count": _RECENT[key]},
details={"count": _RECENT[key]},
)
if full:
_logger.error(
@@ -557,12 +557,10 @@ class PollingLiveSyncSupervisor:
)
def _note_internal_error(self, source_id: str, error: Exception) -> None:
# Type alone used to be all this said. It is not enough to act on:
# `OperationalError` covers a locked database, a missing table and a
# corrupt page alike, and without the message and the frames two dozen
# of these in a log could only be guessed at. The stack is the
# redacted, `one_memory`-relative one, so it names the call site
# without printing anyone's paths or query text.
# The ordinary local log keeps the exception message for debugging.
# The structured diagnostic event deliberately keeps only the stable
# error code, source id, type, and relative stack: exception messages
# can contain paths, query text, or credentials and are exportable data.
logging.getLogger(__name__).warning(
"live_sync_internal_error %s: %s", type(error).__name__, error
)
@@ -572,10 +570,7 @@ class PollingLiveSyncSupervisor:
event="live_sync.internal_error",
outcome="failed",
error_code="live_sync_internal_error",
# `reason` is the listed key that can carry the message; see
# `diagnostic_log._DETAIL_KEYS`. Truncated because a detail value
# longer than 256 characters is rejected.
details={"source_id": source_id, "reason": str(error)[:200]},
details={"source_id": source_id},
**diagnostic_exception(error),
)
if isinstance(error, sqlite3.Error):
@@ -867,10 +867,7 @@ class ImportRunner:
outcome="failed",
job_id=job_id,
error_code="import_heartbeat_failed",
# `reason` is the listed key for this; see
# `diagnostic_log._DETAIL_KEYS`. Truncated: a detail value
# over 256 characters is rejected.
details={"reason": str(error)[:200]},
details={"phase": "heartbeat"},
**diagnostic_exception(error),
)
@@ -14,6 +14,7 @@ stuck in it.
"""
import logging
import re
import sys
import threading
import time
@@ -23,6 +24,7 @@ from types import FrameType
logger = logging.getLogger("one_memory.loop_watchdog")
DEFAULT_STALL_SECONDS = 2.0
_SAFE_FALLBACK_FILE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$")
class LoopWatchdog:
@@ -120,22 +122,28 @@ def _safe_stack_for_main_thread() -> tuple[str, ...]:
def _safe_stacks_for_all_threads() -> tuple[str, ...]:
"""Return one content-free, bounded stack summary for every live thread."""
names = {thread.ident: thread.name for thread in threading.enumerate()}
"""Return bounded stack summaries without recording thread names."""
lines: list[str] = []
for ident, frame in sorted(sys._current_frames().items()):
for _ident, frame in sorted(sys._current_frames().items()):
stack: list[str] = []
# Most specific frame first so the 256-character privacy bound keeps
# the likely culprit rather than only the thread bootstrap frames.
# Most specific frame first; append only complete frames so truncation
# can never turn a valid location into free text.
for entry in reversed(traceback.extract_stack(frame)[-8:]):
parts = entry.filename.replace("\\", "/").split("/")
try:
root = parts.index("one_memory")
relative = "/".join(parts[root:])
except ValueError:
relative = parts[-1]
fallback = parts[-1]
relative = fallback if _SAFE_FALLBACK_FILE.fullmatch(fallback) else "unknown.py"
name = entry.name if entry.name.isidentifier() else "call"
stack.append(f"{relative}:{entry.lineno}:{name}")
thread_name = names.get(ident, "unknown").replace("\n", " ").replace("\r", " ")
lines.append(f"thread={thread_name[:48]}|{' > '.join(stack)}"[:256])
candidate = f"{relative}:{entry.lineno}:{name}"
separator = "" if not stack else " > "
if len("thread|") + len(" > ".join(stack)) + len(separator) + len(candidate) > 256:
break
stack.append(candidate)
if stack:
lines.append(f"thread|{' > '.join(stack)}")
if len(lines) == 16:
break
return tuple(lines)
+310 -18
View File
@@ -18,11 +18,13 @@ from one_memory.diagnostic_log import (
diagnostic_exception,
diagnostics_enabled,
diagnostics_requested,
read_trace_events,
record_database_error,
reset_diagnostics,
reset_trace_id,
resource_snapshot,
)
from one_memory.exception_reporting import report_exception
def test_debug_events_are_collected_only_at_debug_level(tmp_path, monkeypatch) -> None:
@@ -105,7 +107,31 @@ def test_structured_log_is_private_correlated_and_owner_only(tmp_path):
assert log.path.stat().st_mode & 0o777 == 0o600
def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
def test_arbitrary_valid_detail_keys_and_scalars_are_recorded(tmp_path):
log = DiagnosticLog(tmp_path)
details = {
"Custom.metric-v1": "ready",
"noneValue": None,
"boolValue": False,
"intValue": 7,
"floatValue": 1.5,
}
log.write(
level="info",
component="core.lifecycle",
event="core.started",
outcome="succeeded",
details=details,
)
assert log.flush()
assert json.loads(log.path.read_text())["details"] == details
@pytest.mark.parametrize(
"key",
["__proto__", "prototype", "constructor", "_private", "9startsWithDigit", "a" * 65],
)
def test_dangerous_or_malformed_detail_keys_are_rejected(tmp_path, key):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_key_invalid"):
log.write(
@@ -113,8 +139,76 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"prompt": "private"},
details={key: "value"},
)
@pytest.mark.parametrize("value", [{}, [], object()])
def test_complex_detail_values_are_rejected(tmp_path, value):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"payload": value},
)
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
def test_non_finite_detail_values_are_rejected(tmp_path, value):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"metric": value},
)
@pytest.mark.parametrize("value", [2**53, -(2**53), float(2**53), -float(2**53)])
def test_numeric_detail_values_outside_json_safe_range_are_rejected(tmp_path, value):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"count": value},
)
@pytest.mark.parametrize("value", ["", "nul\0byte", "carriage\rreturn", "line\nfeed", "x" * 257])
def test_invalid_detail_strings_are_rejected(tmp_path, value):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"message": value},
)
def test_more_than_32_detail_keys_are_rejected(tmp_path):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_count_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={f"key{index}": index for index in range(33)},
)
def test_absolute_or_credential_bearing_stack_is_rejected(tmp_path):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
log.write(
level="error",
@@ -123,16 +217,90 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
outcome="failed",
safe_stack=("/Users/me/private.py:1",),
)
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
safe_stack=("one_memory/sk-live-secret-token.py:1:call",),
)
def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) -> None:
"""Rejection protects the log; it must not become the caller's outage.
def test_safe_stack_uses_strict_shared_grammar_and_sixteen_line_limit(tmp_path):
log = DiagnosticLog(tmp_path)
for line in (
"arbitrary stack text",
"thread=worker|threading.py:12:wait",
"FILE:///Users/me/private.py:1:call",
"one_memory/worker.py:9:call /Users/private",
"one_memory/GOCSPX-privateCredential.py:9:call",
"packages/ASIA1234567890ABCDEF/file.ts:1:1",
"packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
):
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
safe_stack=(line,),
)
Two of our own nodes shipped with unlisted `details` keys. Both sat on a
request path, so the rejection surfaced as a 500 to the person using the
app — including the sedimentation switch, whose every request failed
because logging the toggle raised before the endpoint could answer.
"""
accepted = (
"apps/desktop/src/main.ts:80:1",
"one_memory/diagnostic_log.py:895:_safe_stack_line",
"threading.py:1045:_bootstrap_inner",
"thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
)
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
safe_stack=accepted,
)
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
log.write(
level="error",
component="hy.bridge",
event="hy.failed",
outcome="failed",
safe_stack=tuple(f"one_memory/worker.py:{index + 1}:call" for index in range(17)),
)
assert log.flush()
assert json.loads(log.path.read_text())["safe_stack"] == list(accepted)
def test_invalid_level_is_strict_but_diagnostic_event_remains_fail_open(
tmp_path, monkeypatch
) -> None:
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_level_invalid"):
log.write(
level="fatal", # type: ignore[arg-type]
component="core.lifecycle",
event="core.failed",
outcome="failed",
)
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
receipt = configure_diagnostics(tmp_path / "fail-open")
try:
diagnostic_event(
level="fatal",
component="core.lifecycle",
event="core.failed",
outcome="failed",
)
finally:
reset_diagnostics()
assert not receipt.path.exists()
def test_a_rejected_payload_does_not_break_the_caller_or_later_events(
tmp_path, monkeypatch
) -> None:
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
receipt = configure_diagnostics(tmp_path)
try:
@@ -141,20 +309,122 @@ def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) ->
component="memory.sedimentation",
event="memory_sedimentation.toggled",
outcome="succeeded",
details={"prompt": "private"},
details={"payload": ["not", "scalar"]},
)
diagnostic_event(
level="info",
component="memory.sedimentation",
event="memory_sedimentation.toggled",
outcome="succeeded",
details={"custom.metric": 1},
)
finally:
reset_diagnostics()
# Rejected, so nothing is written — but the call returned.
assert not receipt.path.exists() or receipt.path.read_text() == ""
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
assert [event["details"] for event in events] == [{"custom.metric": 1}]
def test_trace_projection_does_not_expose_arbitrary_local_detail_keys(tmp_path):
log = DiagnosticLog(tmp_path)
log.write(
level="info",
component="core.api",
event="memory.request.completed",
outcome="succeeded",
request_id="request-1",
details={
"phase": "recall",
"reason": "/Users/private/query.txt",
"status": "sk-live-secret-token",
"custom.private": "local-only",
},
)
assert log.flush()
events = read_trace_events(tmp_path, request_id="request-1")
assert events[0]["details"] == {"phase": "recall"}
def test_trace_projection_rejects_symlinked_directory_and_redacts_top_level_secrets(
tmp_path,
) -> None:
request_id = "request-1"
logs = tmp_path / "logs"
logs.mkdir()
event = {
"timestamp": "2026-10-23T01:00:00Z",
"request_id": request_id,
"component": "secret.token",
"event": "credential.failed",
"outcome": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
"error_code": "GOCSPX-clientSecret_A1",
"error_type": "ASIA1234567890ABCDEF",
"details": {
"phase": "recall",
"reason": "x_ASIA1234567890ABCDEF",
"mode": "x_GOCSPX-clientSecret_A1",
"status": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
"failure_site": "AIzaSyExampleCredential1234567890",
"count": 1,
},
}
(logs / "diagnostic-core.jsonl").write_text(json.dumps(event) + "\n")
projected = read_trace_events(tmp_path, request_id=request_id)
assert projected == (
{
"timestamp": "2026-10-23T01:00:00Z",
"component": "diagnostic.redacted",
"event": "diagnostic.redacted",
"outcome": "redacted",
"details": {"phase": "recall", "count": 1},
},
)
external = tmp_path / "external"
external.mkdir()
(logs / "diagnostic-core.jsonl").unlink()
logs.rmdir()
logs.symlink_to(external, target_is_directory=True)
with pytest.raises(ValueError, match="diagnostic_directory_unsafe"):
read_trace_events(tmp_path, request_id=request_id)
def test_credential_detection_is_conservative_and_matches_export_rules() -> None:
for value in (
"ASIA1234567890ABCDEF",
"GOCSPX-clientSecret_A1",
"Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
"Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8_",
):
assert diagnostic_log._contains_credential_material(value)
for value in (
"11111111-1111-4111-8111-111111111111",
"abcdef1234567890abcdef1234567890abcdef12",
"cursor",
):
assert not diagnostic_log._contains_credential_material(value)
def test_trace_reader_never_follows_a_diagnostic_file_symlink(tmp_path) -> None:
logs = tmp_path / "logs"
logs.mkdir()
outside = tmp_path / "outside.jsonl"
outside.write_text(
json.dumps(
{
"timestamp": "2026-10-23T01:00:00Z",
"request_id": "request-1",
"component": "core.api",
"event": "memory.request.completed",
"outcome": "succeeded",
}
)
+ "\n"
)
(logs / "diagnostic-core.jsonl").symlink_to(outside)
assert read_trace_events(tmp_path, request_id="request-1") == ()
def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
"""`enabled` is a bounded boolean and has to be a legal detail key.
Omitting it from the allow-list is what made the switch's own endpoint
answer 500 while still saving the new value.
"""
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
receipt = configure_diagnostics(tmp_path)
try:
@@ -181,7 +451,7 @@ def test_exception_diagnostic_keeps_code_location_not_message():
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"prompt": secret},
details={"prompt": f"{secret}\n"},
)
except ValueError as error:
safe = diagnostic_exception(error)
@@ -191,6 +461,28 @@ def test_exception_diagnostic_keeps_code_location_not_message():
assert str(__file__) not in json.dumps(safe)
def test_report_exception_never_writes_the_exception_message_to_diagnostics(
tmp_path, monkeypatch
) -> None:
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
receipt = configure_diagnostics(tmp_path)
secret = "sk-live-private-provider-response"
try:
report_exception(
"core.test",
"core.test.failed",
RuntimeError(f"provider failed with {secret} at /Users/private/query.txt"),
)
finally:
reset_diagnostics()
payload = receipt.path.read_text()
assert secret not in payload
assert "/Users/private" not in payload
event = json.loads(payload)
assert event["details"] == {"count": 1}
assert event["error_type"] == "RuntimeError"
def test_old_logs_are_removed_but_symlinks_are_never_followed(tmp_path):
log = DiagnosticLog(tmp_path)
log.path.parent.mkdir(parents=True)
@@ -449,7 +449,7 @@ def test_invalid_input_is_still_rejected_on_the_caller_side(tmp_path):
component="hy.bridge",
event="hy.failed",
outcome="failed",
details={"prompt": "private"},
details={"invalid key": "private"},
)
log.close()
@@ -1,22 +1,57 @@
"""An instrument must never be able to break the thing it measures.
`diagnostic_event` validates its payload: an unlisted `details` key, a missing
`level`, or a detail value over 256 characters all raise. Two of our own nodes
shipped with unlisted keys, and because they sat on a request path and on the
loop heartbeat, they turned every external recall into a 500 and then killed
the process. Both were caught only by running the built runtime, not by tests.
`diagnostic_event` validates its payload: a malformed `details` key, a missing
`level`, or an unsafe detail value is rejected. A diagnostic payload must never
be able to break the request path or loop heartbeat it is intended to measure.
So this checks the payloads at their source, by walking every literal call.
"""
import ast
import re
from pathlib import Path
import pytest
from one_memory.diagnostic_log import _DETAIL_KEYS
ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
REFUSED_DETAIL_KEYS = {"__proto__", "prototype", "constructor"}
EXPORTED_STRING_DETAIL_KEYS = {
"action",
"method",
"mode",
"phase",
"platform",
"reason",
"source_id",
"status",
"version",
"parent_session_id",
"runtime_state",
"core_state",
"broker_state",
"instance_id",
"operation_kind",
"file_kind",
"last_probe",
"signal",
}
PRIVATE_DETAIL_KEYS = {
"api_key",
"authorization",
"content",
"credential",
"memory",
"memory_text",
"message",
"messages",
"password",
"prompt",
"query",
"raw_text",
"secret",
"token",
}
#: Every module, not a hand-picked list. The two payloads that caused real
#: outages were in files nobody thought to add here, so the only shape of this
@@ -33,6 +68,19 @@ def _literal_calls(path: Path):
yield node
def _contains_free_text_conversion(node: ast.AST) -> bool:
for item in ast.walk(node):
if (
isinstance(item, ast.Call)
and isinstance(item.func, ast.Name)
and item.func.id in {"str", "repr", "_safe_message"}
):
return True
if isinstance(item, ast.Attribute) and item.attr in {"args", "message"}:
return True
return False
@pytest.mark.parametrize("relative", SOURCES)
def test_every_diagnostic_payload_is_valid(relative: str) -> None:
path = ROOT / relative
@@ -45,7 +93,22 @@ def test_every_diagnostic_payload_is_valid(relative: str) -> None:
for keyword in node.keywords:
if keyword.arg != "details" or not isinstance(keyword.value, ast.Dict):
continue
for key in keyword.value.keys:
if isinstance(key, ast.Constant) and key.value not in _DETAIL_KEYS:
problems.append(f"{relative}:{node.lineno}: unlisted details key {key.value!r}")
for key, expression in zip(keyword.value.keys, keyword.value.values, strict=True):
if not isinstance(key, ast.Constant):
continue
value = key.value
if (
not isinstance(value, str)
or not DETAIL_KEY.fullmatch(value)
or value in REFUSED_DETAIL_KEYS
):
problems.append(f"{relative}:{node.lineno}: invalid details key {value!r}")
elif value.lower() in PRIVATE_DETAIL_KEYS:
problems.append(f"{relative}:{node.lineno}: private details key {value!r}")
elif value in EXPORTED_STRING_DETAIL_KEYS and _contains_free_text_conversion(
expression
):
problems.append(
f"{relative}:{node.lineno}: exported detail {value!r} uses free text"
)
assert not problems, "\n".join(problems)
+4 -2
View File
@@ -102,6 +102,8 @@ def test_stall_captures_a_safe_stack_for_every_thread() -> None:
assert watchdog.stall_stacks
captured = watchdog.stall_stacks[0]
assert any(line.startswith("thread=MainThread|") for line in captured)
assert any(line.startswith("thread=source-reader-test|") for line in captured)
assert captured
assert len(captured) <= 16
assert all(line.startswith("thread|") for line in captured)
assert all("source-reader-test" not in line and "MainThread" not in line for line in captured)
assert all("/Users/" not in line and "\n" not in line for line in captured)
@@ -94,12 +94,14 @@ it("returns only diagnostic events written during this desktop launch", async ()
await mkdir(logs, { recursive: true });
const target = path.join(logs, "diagnostic-core.jsonl");
const old = {
schema_version: 1,
timestamp: "2026-10-14T09:00:00.000Z",
level: "info",
service: "core",
component: "core.startup",
event: "core.old",
outcome: "succeeded",
session_id: "11111111-1111-4111-8111-111111111111",
};
await writeFile(target, `${JSON.stringify(old)}\n`);
const cursor = await captureDebugLogCursor(dataDir);
@@ -109,6 +111,11 @@ it("returns only diagnostic events written during this desktop launch", async ()
timestamp: "2026-10-14T10:01:00.000Z",
event: "core.current",
level: "warn",
details: {
count: 1,
"custom.private": "/Users/private/query.txt",
credential_hint: "ghp_private_token",
},
};
await appendFile(target, `${JSON.stringify(current)}\nnot-json\n`);
@@ -118,8 +125,11 @@ it("returns only diagnostic events written during this desktop launch", async ()
expect(snapshot.entries[0]).toMatchObject({
event: "core.current",
level: "warn",
details: { count: 1 },
source: "diagnostic",
});
expect(JSON.stringify(snapshot)).not.toContain("/Users/private");
expect(JSON.stringify(snapshot)).not.toContain("ghp_private_token");
});
it("tracks diagnostic JSONL across rotation without rereading the old prefix", async () => {
@@ -130,12 +140,14 @@ it("tracks diagnostic JSONL across rotation without rereading the old prefix", a
const target = path.join(logs, "diagnostic-core.jsonl");
const event = (name: string, timestamp: string) =>
JSON.stringify({
schema_version: 1,
timestamp,
level: "info",
service: "core",
component: "core.test",
event: name,
outcome: "succeeded",
session_id: "22222222-2222-4222-8222-222222222222",
});
await writeFile(target, `${event("before", "2026-10-14T09:00:00.000Z")}\n`);
const cursor = await captureDebugLogCursor(dataDir);
+51 -51
View File
@@ -1,10 +1,13 @@
import { lstat, open, readdir } from "node:fs/promises";
import path from "node:path";
import {
type DiagnosticEventV1,
type DiagnosticLogger,
diagnosticEvent,
diagnosticZipFilename,
exportDiagnosticZip,
inspectBackgroundHealth,
projectDiagnosticEventForSafeOutput,
recordDiagnosticMarker,
} from "@one-memory/runtime-control";
import type { BrowserWindow, IpcMainInvokeEvent } from "electron";
@@ -370,42 +373,58 @@ function debugLogEntry(
const timestamp = typeof value.timestamp === "string" ? value.timestamp : "";
const timestampMs = Date.parse(timestamp);
if (!Number.isFinite(timestampMs) || timestampMs < startedAt) return null;
const level = value.level;
const service = value.service;
const component = value.component;
const event = value.event;
const outcome = value.outcome;
if (
!["info", "warn", "error"].includes(String(level)) ||
!["runtime", "core", "host-bridge", "desktop", "cli"].includes(
String(service),
) ||
typeof component !== "string" ||
typeof event !== "string" ||
typeof outcome !== "string"
) {
let projected: DiagnosticEventV1;
try {
projected = projectDiagnosticEventForSafeOutput(
diagnosticEvent({
timestamp,
level: value.level as never,
service: value.service as never,
component: value.component as never,
event: value.event as never,
outcome: value.outcome as never,
session_id: value.session_id as never,
...(typeof value.trace_id === "string"
? { trace_id: value.trace_id }
: {}),
...(typeof value.duration_ms === "number"
? { duration_ms: value.duration_ms }
: {}),
...(typeof value.error_code === "string"
? { error_code: value.error_code }
: {}),
...(typeof value.error_type === "string"
? { error_type: value.error_type }
: {}),
...(value.details && typeof value.details === "object"
? { details: value.details as never }
: {}),
}),
);
} catch {
return null;
}
const details = safeDebugDetails(value.details);
return {
id: `${filename}:${lineNumber}:${timestamp}`,
timestamp,
level: level as DebugLogEntry["level"],
service: service as DebugLogEntry["service"],
component,
event,
outcome,
...(typeof value.duration_ms === "number"
? { duration_ms: value.duration_ms }
: {}),
...(typeof value.error_code === "string"
? { error_code: value.error_code }
: {}),
...(typeof value.error_type === "string"
? { error_type: value.error_type }
: {}),
...(typeof value.trace_id === "string" ? { trace_id: value.trace_id } : {}),
...(details === undefined ? {} : { details }),
timestamp: projected.timestamp,
level: projected.level,
service: projected.service,
component: projected.component,
event: projected.event,
outcome: projected.outcome,
...(projected.duration_ms === undefined
? {}
: { duration_ms: projected.duration_ms }),
...(projected.error_code === undefined
? {}
: { error_code: projected.error_code }),
...(projected.error_type === undefined
? {}
: { error_type: projected.error_type }),
...(projected.trace_id === undefined
? {}
: { trace_id: projected.trace_id }),
...(projected.details === undefined ? {} : { details: projected.details }),
source: "diagnostic",
};
}
@@ -523,25 +542,6 @@ function logLineTimestamp(
return new Date(Math.max(startedAt, fallback)).toISOString();
}
function safeDebugDetails(
value: unknown,
): Record<string, string | number | boolean | null> | undefined {
if (!value || typeof value !== "object" || Array.isArray(value))
return undefined;
const details: Record<string, string | number | boolean | null> = {};
for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
if (
item === null ||
typeof item === "string" ||
typeof item === "number" ||
typeof item === "boolean"
) {
details[key] = item;
}
}
return Object.keys(details).length ? details : undefined;
}
async function pipelineLogFiles(root: string): Promise<PipelineLogCursor[]> {
const rootInfo = await lstat(root).catch(() => null);
if (!rootInfo?.isDirectory() || rootInfo.isSymbolicLink()) return [];
+1 -1
View File
@@ -1098,7 +1098,7 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
? "blocked"
: "skipped",
...(migration.code === undefined ? {} : { error_code: migration.code }),
details: { state: migration.state },
details: { runtime_state: migration.state },
});
requireCompletedStartupSdkMigration(migration);
};
+9 -5
View File
@@ -6,10 +6,13 @@
2. 记忆沉淀有没有持续运行、产出了多少条;
3. 如果停住,停在导入、模型调用还是最终写入。
默认 `info` 下的结构化诊断日志不包含会话正文、模型输入输出或 API Key。`debug`
会额外启用 HY pipeline trace,其中可能包含完整 prompt/response 和获授权的对话正文。
排查时优先通过 App 的“导出日志”提供文件,不要直接发送整个 `~/.one-memory` 数据目录;
debug 期间产生的 pipeline 日志在分享前必须由开发人员确认并脱敏。
默认 `info` 下的官方结构化事件不应写入会话正文、模型输入输出或 API Key;但本地原始日志允许
新增受限标量标签,仍应按私人数据处理,不能直接对外发送。`debug` 会额外启用 HY pipeline
trace,其中可能包含完整 prompt/response 和获授权的对话正文。
排查时只通过 App 的“导出日志”提供文件:导出过程会按安全白名单重新生成诊断事件,不会打包
原始日志。不要直接发送 `diagnostic-*.jsonl`、整个 `~/.one-memory` 数据目录或 debug pipeline
日志;后者如确需分享,必须由开发人员另行确认并脱敏。
## 1. Log level 设什么
@@ -39,7 +42,8 @@ debug 可能把完整模型 prompt/response 写入本机调试日志,不得长
## 2. 日志在哪里
App 内推荐直接使用“运行诊断 → 导出日志”。开发人员远程指导时,优先使用
`memhub logs`,它会再次过滤不安全字段。需要现场查看原始结构化日志时,Core 日志默认位于:
`memhub logs`,它会再次过滤不安全字段。原始结构化日志只用于本机现场查看,不得作为附件发送。
Core 日志默认位于:
```text
~/.one-memory/logs/diagnostic-core.jsonl
+24 -8
View File
@@ -15,9 +15,17 @@ Desktop -> Runtime -> Core -> HY -> Host 的实际执行链。日志不是用户
可选关联字段为 `trace_id`、`incident_id`、`instance_id`、`operation_id`、`request_id`、
`job_id`、`task_id`、`duration_ms`、状态、稳定错误码、错误类型和清理后的仓库相对栈帧。
`details` 只接受代码声明的标量白名单。禁止记录对话原文、查询、回答、模型输入输出、
请求体、响应体、token、API Key、secret ref、完整用户路径或异常 message/cause。未知字段和
不合规标识拒绝写入。日志写入失败不能阻断产品主链。
`details` 的本地写入合同只约束结构,不维护业务字段白名单:最多 32 个 key;key 必须是
1–64 字符的字母、数字、点、下划线或连字符并以字母开头;value 只接受 `null`、布尔值、
跨 Python/JavaScript 可无损表示的有限数字(整数限于 ±(2^53-1))或不超过 256 字符且不含
控制换行的短字符串,不接受对象和数组。这样新增诊断标签
不需要先修改中央白名单。诊断日志的校验、序列化、排队或落盘发生任何异常时都只能丢弃该条
日志,不能阻断产品主链。
本地写入放宽不代表允许官方代码记录用户内容。对话原文、查询、回答、模型输入输出、请求体、
响应体、token、API Key、secret ref、完整用户路径和异常 message/cause 仍禁止进入诊断事件,
由类型、静态检查和隐私测试共同防止误用;确需保存完整用户输入的调试能力必须使用独立的本地
trace/session 文件,不得混入可导出的诊断日志。
## 关联规则
@@ -34,9 +42,16 @@ Desktop -> Runtime -> Core -> HY -> Host 的实际执行链。日志不是用户
host-bridge、desktop 和 cli。单文件 4 MiB,保留 3 份轮转;额外清理 14 天以前的文件并将
总量限制在 128 MiB。POSIX 目录为 0700、文件为 0600。
读取器只枚举上述固定文件名的 regular file,不跟随符号链接,不接受任意路径。损坏行跳过
并计数,不让一行损坏遮蔽其余证据。诊断导出是单个脱敏 JSON,包含版本、平台、架构、
非秘密状态和最多 5000 条已验证事件,不包含数据库或配置文件。
读取器只枚举上述固定文件名的 regular file,不跟随符号链接,不接受任意路径。日常本地查询
遇到损坏行时跳过并计数,不让一行损坏遮蔽其余证据。
导出是独立的安全边界:它重新读取并校验全部候选事件,只把明确列入导出白名单的字段投影为
新的 `diagnostics/events.jsonl`;details 不只检查 key,还按字段约束 value 类型,并拒绝自由文本、
路径和凭证形态的字符串。绝不复制原始 `diagnostic-*.jsonl`、普通运行日志或崩溃报告。
导出包还可以包含固定 schema 的系统摘要和问题时间点,但不接受调用方传入的任意对象展开。
任何事件、marker、安全投影或日志目录读取失败时,导出必须 fail-closed:删除临时文件、不发布
诊断包,更不能回退为打包原始日志。发布使用已设好权限的临时文件原子创建最终目标,已存在的
目标不得覆盖,发布后也不存在再 chmod 失败留下伪成功文件的窗口。
## 覆盖与节流
@@ -56,7 +71,8 @@ host-bridge、desktop 和 cli。单文件 4 MiB,保留 3 份轮转;额外清
## 验收
隐私测试必须先植入一条可检测的秘密作为正向对照,再证明真实链路日志和导出均不含该秘密。
至少用隔离数据制造 Core 启动失败、HY 失败和宿主连接失败,并仅凭日志还原失败组件、时间、
隐私测试必须先向本地日志植入带未知 key 的可检测秘密作为正向对照,再证明本地读取仍保留该
证据、导出安全投影必定剔除它;另需证明损坏行会阻止导出且不会留下伪成功文件。至少用隔离
数据制造 Core 启动失败、HY 失败和宿主连接失败,并仅凭允许导出的字段还原失败组件、时间、
关联 ID、稳定错误码和代码位置。自动测试通过只能标记 verified,正式安装 App 的真实体验仍需
单独验收。
+1 -1
View File
@@ -2,7 +2,7 @@
"name": "one-memory",
"version": "1.0.5",
"release": {
"build_number": 14,
"build_number": 16,
"macos_min_version": "13.0",
"app_update": "manual",
"runtime_manifest_schema": 1,
@@ -192,16 +192,12 @@ describe("RecallBroker", () => {
level: "error",
component: "host.broker",
outcome: "failed",
details: expect.objectContaining({
action: "host",
code: "connection_control_unavailable",
error_type: "Error",
}),
error_code: "connection_control_unavailable",
error_type: "Error",
details: { action: "host" },
});
expect(typeof exception?.details).toBe("object");
const details = exception?.details as Record<string, unknown>;
expect(typeof details.stack_origin).toBe("string");
expect(details.stack_origin).toContain("recall-broker");
expect(exception?.details).not.toHaveProperty("message");
expect(exception?.details).not.toHaveProperty("stack_origin");
});
it.each(["recall", "ask"])(
"answers %s without waiting for a slow diagnostic write",
@@ -702,20 +702,10 @@ export class RecallBroker {
event: "connection_control.exception",
outcome: "failed",
instance_id: this.options.instanceId,
details: {
action: action ?? "invalid",
code,
error_type:
error instanceof Error ? error.constructor.name : typeof error,
message:
error instanceof Error
? error.message.slice(0, 256)
: String(error).slice(0, 256),
stack_origin:
error instanceof Error && error.stack
? (error.stack.split("\n").at(1)?.trim() ?? "").slice(0, 256)
: null,
},
error_code: code,
error_type:
error instanceof Error ? error.constructor.name : typeof error,
details: { action: action ?? "invalid" },
});
}
const fingerprint = `error:${code}`;
@@ -680,7 +680,7 @@ export class CoreSupervisor {
outcome: "failed",
instance_id: this.activeInstanceId,
error_code: "core_restart_circuit_open",
details: { streak: this.restartStreak },
details: { restart_streak: this.restartStreak },
});
return;
}
@@ -765,7 +765,7 @@ export class CoreSupervisor {
outcome: "failed",
instance_id: this.activeInstanceId,
error_code: "core_restart_exhausted",
details: { attempts: budget, streak: this.restartStreak },
details: { attempts: budget, restart_streak: this.restartStreak },
});
await this.options.diagnostics?.log({
level: "error",
@@ -774,7 +774,7 @@ export class CoreSupervisor {
outcome: "failed",
instance_id: this.activeInstanceId,
error_code: "core_restart_circuit_open",
details: { streak: this.restartStreak },
details: { restart_streak: this.restartStreak },
});
}
} finally {
@@ -1,4 +1,5 @@
import {
chmod,
lstat,
mkdir,
mkdtemp,
@@ -21,8 +22,10 @@ import {
diagnosticLogFiles,
exportDiagnosticBundle,
exportDiagnosticZip,
projectDiagnosticEventForSafeOutput,
pruneDiagnosticLogs,
readDiagnosticEvents,
recordDiagnosticMarker,
writeDiagnosticEventSync,
} from "./diagnostic-log.js";
@@ -66,6 +69,30 @@ describe("diagnostic log", () => {
});
});
it("keeps the synchronous writer fail-open for invalid input", async () => {
const dataDir = await root();
expect(() =>
writeDiagnosticEventSync(
{
dataDir,
service: "desktop",
sessionId: "55555555-5555-4555-8555-555555555555",
},
{
level: "error",
component: "desktop.lifecycle",
event: "desktop.early_start_failed",
outcome: "failed",
details: { nested: ["private"] } as never,
},
),
).not.toThrow();
expect(await readDiagnosticEvents(dataDir)).toEqual({
events: [],
invalid_lines: 0,
});
});
it("writes structured, private events and reads them by correlation identity", async () => {
const dataDir = await root();
const session = "11111111-1111-4111-8111-111111111111";
@@ -89,6 +116,8 @@ describe("diagnostic log", () => {
open_file_count: 7,
memory_bytes: 4096,
thread_count: 4,
"cache.hit-rate": 0.75,
custom_label: "warm",
},
});
const result = await readDiagnosticEvents(dataDir, { traceId: trace });
@@ -101,6 +130,8 @@ describe("diagnostic log", () => {
event: "runtime.ready",
details: expect.objectContaining({
open_file_count: 7,
"cache.hit-rate": 0.75,
custom_label: "warm",
}),
});
const file = (await diagnosticLogFiles(dataDir))[0];
@@ -184,7 +215,7 @@ describe("diagnostic log", () => {
}
});
it("rejects arbitrary detail fields and absolute stack paths", () => {
it("strictly rejects complex detail values, dangerous keys and absolute stacks", () => {
const base = {
timestamp: new Date().toISOString(),
level: "error" as const,
@@ -195,11 +226,142 @@ describe("diagnostic log", () => {
session_id: "11111111-1111-4111-8111-111111111111",
};
expect(() =>
diagnosticEvent({ ...base, details: { prompt: "private" } }),
diagnosticEvent({
...base,
details: { nested: { secret: "private" } } as never,
}),
).toThrow("diagnostic_detail_value_invalid");
expect(() =>
diagnosticEvent({ ...base, details: { constructor: "private" } }),
).toThrow("diagnostic_detail_key_invalid");
expect(() =>
diagnosticEvent({ ...base, details: { "9invalid": "private" } }),
).toThrow("diagnostic_detail_key_invalid");
expect(() =>
diagnosticEvent({
...base,
details: Object.fromEntries(
Array.from({ length: 33 }, (_, index) => [`key_${index}`, index]),
),
}),
).toThrow("diagnostic_detail_count_invalid");
expect(() =>
diagnosticEvent({ ...base, details: { message: "line1\nline2" } }),
).toThrow("diagnostic_text_invalid");
expect(() =>
diagnosticEvent({ ...base, details: { metric: Number.NaN } }),
).toThrow("diagnostic_detail_value_invalid");
expect(() =>
diagnosticEvent({
...base,
details: { count: Number.MAX_SAFE_INTEGER + 1 },
}),
).toThrow("diagnostic_detail_value_invalid");
expect(() =>
diagnosticEvent({ ...base, details: { list: ["private"] } as never }),
).toThrow("diagnostic_detail_value_invalid");
expect(() =>
diagnosticEvent({ ...base, safe_stack: ["/Users/me/private.py:1"] }),
).toThrow("diagnostic_stack_invalid");
expect(() =>
diagnosticEvent({
...base,
safe_stack: ["packages/runtime/sk-live-secret-token.ts:1:1"],
}),
).toThrow("diagnostic_stack_invalid");
for (const line of [
"arbitrary stack text",
"thread=worker|threading.py:12:wait",
"FILE:///Users/me/private.py:1:call",
"one_memory/worker.py:9:call /Users/private",
"one_memory/GOCSPX-privateCredential.py:9:call",
"packages/ASIA1234567890ABCDEF/file.ts:1:1",
"packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
]) {
expect(() => diagnosticEvent({ ...base, safe_stack: [line] })).toThrow(
"diagnostic_stack_invalid",
);
}
expect(() =>
diagnosticEvent({
...base,
safe_stack: Array.from(
{ length: 17 },
(_, index) => `one_memory/worker.py:${index + 1}:call`,
),
}),
).toThrow("diagnostic_stack_invalid");
const accepted = diagnosticEvent({
...base,
safe_stack: [
"apps/desktop/src/main.ts:80:1",
"one_memory/diagnostic_log.py:895:_safe_stack_line",
"threading.py:1045:_bootstrap_inner",
"thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
],
});
expect(accepted.safe_stack).toHaveLength(4);
const projected = projectDiagnosticEventForSafeOutput({
...accepted,
safe_stack: [
"free text",
"thread=worker|one_memory/worker.py:9:call",
"packages/runtime-control/src/diagnostic-log.ts:42:7",
"thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
],
});
expect(projected.safe_stack).toEqual([
"packages/runtime-control/src/diagnostic-log.ts:42:7",
"thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
]);
const conservative = projectDiagnosticEventForSafeOutput(
diagnosticEvent({
...base,
details: {
source_id: "cursor",
reason: "abcdef1234567890abcdef1234567890abcdef12",
status: "11111111-1111-4111-8111-111111111111",
},
}),
);
expect(conservative.details).toEqual({
source_id: "cursor",
reason: "abcdef1234567890abcdef1234567890abcdef12",
status: "11111111-1111-4111-8111-111111111111",
});
});
it("drops malformed logger events without interrupting later writes", async () => {
const dataDir = await root();
const logger = new DiagnosticLogger({
dataDir,
service: "desktop",
sessionId: "11111111-1111-4111-8111-111111111111",
});
await expect(
logger.log({
level: "info",
component: "desktop.sdk_migration",
event: "sdk_migration.startup",
outcome: "succeeded",
details: { nested: { secret: "private" } } as never,
}),
).resolves.toBeUndefined();
await logger.log({
level: "info",
component: "desktop.sdk_migration",
event: "sdk_migration.startup",
outcome: "succeeded",
details: { arbitrary_state: "cutover" },
});
const result = await readDiagnosticEvents(dataDir);
expect(result.invalid_lines).toBe(0);
expect(result.events).toHaveLength(1);
expect(result.events[0].details).toEqual({ arbitrary_state: "cutover" });
});
it("keeps code locations but strips exception messages and absolute prefixes", () => {
@@ -215,29 +377,162 @@ describe("diagnostic log", () => {
expect(JSON.stringify(safe)).not.toContain("/Users/me");
});
it("marks incidents and exports only validated events", async () => {
it("exports events through an independent detail whitelist", async () => {
const dataDir = await root();
const logger = new DiagnosticLogger({ dataDir, service: "desktop" });
const incident = await logger.markIncident({
component: "desktop.diagnostics",
status: "degraded",
});
await writeFile(
path.join(dataDir, "logs", "diagnostic-core.jsonl"),
"not-json\n",
{ mode: 0o600 },
);
await logger.log({
level: "info",
component: "desktop.diagnostics",
event: "diagnostics.checked",
outcome: "succeeded",
details: {
count: 1,
reason: "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signaturevalue",
status: "ASIA1234567890ABCDEF",
source_id: "GOCSPX-clientSecret_A1",
mode: "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
action: "AIzaSyExampleCredential1234567890",
platform: "x_ASIA1234567890ABCDEF",
phase: "x_GOCSPX-clientSecret_A1",
local_only_secret: "must-not-export",
},
});
await logger.log({
level: "error",
component: "credential.private",
event: "token.failed",
outcome: "ghp_private_token",
trace_id: "ghp_private_token",
instance_id: "ASIA1234567890ABCDEF",
request_id: "GOCSPX-clientSecret_A1",
task_id: "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
error_code: "secret.token",
});
const target = path.join(await root(), "bundle.json");
const result = await exportDiagnosticBundle({
dataDir,
target,
productVersion: "1.2.3",
status: { status: "degraded" },
status: { status: "degraded", local_only_secret: "must-not-export" },
});
expect(result).toEqual({ event_count: 1, invalid_lines: 1 });
expect(result).toEqual({ event_count: 3, invalid_lines: 0 });
const payload = JSON.parse(await readFile(target, "utf8"));
expect(payload.events[0].incident_id).toBe(incident.incident_id);
expect(JSON.stringify(payload)).not.toContain(dataDir);
expect(payload.events[1].details).toEqual({ count: 1 });
expect(payload.events[2]).toMatchObject({
component: "diagnostic.redacted",
event: "diagnostic.redacted",
outcome: "redacted",
});
expect(payload.events[2].trace_id).toBeUndefined();
expect(payload.events[2].instance_id).toBeUndefined();
expect(payload.events[2].request_id).toBeUndefined();
expect(payload.events[2].task_id).toBeUndefined();
expect(payload.events[2].error_code).toBeUndefined();
expect(JSON.stringify(payload)).not.toContain("local_only_secret");
expect(JSON.stringify(payload)).not.toContain("must-not-export");
expect(JSON.stringify(payload)).not.toContain("eyJhbGciOiJIUzI1NiJ9");
expect(JSON.stringify(payload)).not.toContain("ASIA1234567890ABCDEF");
expect(JSON.stringify(payload)).not.toContain("GOCSPX-clientSecret_A1");
expect(JSON.stringify(payload)).not.toContain("x_ASIA1234567890ABCDEF");
expect(JSON.stringify(payload)).not.toContain("x_GOCSPX-clientSecret_A1");
expect(JSON.stringify(payload)).not.toContain("Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8");
expect(JSON.stringify(payload)).not.toContain(
"AIzaSyExampleCredential1234567890",
);
expect(JSON.stringify(payload)).not.toContain("ghp_private_token");
});
it("fails closed when exported build identity has a credential shape", async () => {
const dataDir = await root();
const outputDir = await root();
await expect(
exportDiagnosticBundle({
dataDir,
target: path.join(outputDir, "bundle.json"),
productVersion: "xoxb-123456-abcdef",
}),
).rejects.toThrow("diagnostic_product_version_invalid");
await expect(
exportDiagnosticZip({
dataDir,
target: path.join(outputDir, "commit.zip"),
productVersion: "1.2.3",
commit: "ghp_private_token",
}),
).rejects.toThrow("diagnostic_commit_invalid");
await expect(
exportDiagnosticZip({
dataDir,
target: path.join(outputDir, "sha.zip"),
productVersion: "1.2.3",
coreSha256: "AKIDExampleSecretId1234567890",
}),
).rejects.toThrow("diagnostic_sha256_invalid");
expect(await readdir(outputDir)).toEqual([]);
});
it("fails closed without publishing a JSON bundle when any line is invalid", async () => {
const dataDir = await root();
await mkdir(path.join(dataDir, "logs"), { recursive: true });
await writeFile(
path.join(dataDir, "logs", "diagnostic-core.jsonl"),
"not-json\n",
{ mode: 0o600 },
);
const outputDir = await root();
const target = path.join(outputDir, "bundle.json");
await expect(
exportDiagnosticBundle({
dataDir,
target,
productVersion: "1.2.3",
}),
).rejects.toThrow("diagnostic_export_invalid_lines");
expect(await readdir(outputDir)).toEqual([]);
});
it("fails closed when the diagnostic directory cannot be read", async () => {
if (process.platform === "win32") return;
const dataDir = await root();
const logs = path.join(dataDir, "logs");
await mkdir(logs, { recursive: true, mode: 0o700 });
await writeFile(path.join(logs, "diagnostic-core.jsonl"), "{}\n");
const outputDir = await root();
const target = path.join(outputDir, "bundle.json");
await chmod(logs, 0o000);
try {
await expect(
exportDiagnosticBundle({
dataDir,
target,
productVersion: "1.2.3",
}),
).rejects.toMatchObject({ code: "EACCES" });
} finally {
await chmod(logs, 0o700);
}
expect(await readdir(outputDir)).toEqual([]);
});
it("never overwrites an existing export target or leaves temporary files", async () => {
const dataDir = await root();
const outputDir = await root();
const target = path.join(outputDir, "bundle.json");
await writeFile(target, "existing-artifact", { mode: 0o600 });
await expect(
exportDiagnosticBundle({
dataDir,
target,
productVersion: "1.2.3",
}),
).rejects.toMatchObject({ code: "EEXIST" });
expect(await readFile(target, "utf8")).toBe("existing-artifact");
expect(await readdir(outputDir)).toEqual(["bundle.json"]);
});
it("preserves microsecond ordering across service log files", async () => {
@@ -280,7 +575,7 @@ describe("diagnostic log", () => {
expect(result.events[1].timestamp).toBe("2026-10-14T00:00:00.123999Z");
});
it("exports a private seven-day zip with summary, logs, crashes and markers", async () => {
it("exports only reprojected events, a strict summary and README", async () => {
const dataDir = await root();
const outputDir = await root();
const crashDir = await root();
@@ -326,39 +621,178 @@ describe("diagnostic log", () => {
markers: [
{
timestamp: "2026-10-05T01:30:00.000Z",
description: "蓝莓没出来",
description:
"蓝莓没出来 /Users/private/query.txt ASIA1234567890ABCDEF GOCSPX-clientSecret_A1 Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8 token=ghp_private_token xoxb-123456-abcdef AKIDExampleSecretId1234567890 AIzaSyExampleCredential1234567890 eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signaturevalue",
},
],
summary: {
sources: [{ source_id: "workbuddy", imported_messages: 12 }],
memory_tasks: { unknown: 1, last_error_codes: { model_failed: 1 } },
summary_secret: conversation,
background: {
status: "exported",
runtime_state: "ready",
core_state: "healthy",
broker_state: "ready",
instance_id: "instance-1",
secret: conversation,
},
},
});
expect(result.filename).toBe("MemHub.zip");
expect(result).toEqual({ filename: "MemHub.zip", file_count: 3 });
const files = unzipSync(await readFile(target));
expect(Object.keys(files)).toEqual(
expect.arrayContaining([
"README.txt",
"summary.json",
"logs/diagnostic-core.jsonl",
"logs/core.log",
"crashes/MemHub-2026-10-05.ips",
]),
);
const summary = files["summary.json"];
expect(Object.keys(files).sort()).toEqual([
"README.txt",
"diagnostics/events.jsonl",
"summary.json",
]);
const eventFile = files["diagnostics/events.jsonl"];
const summaryFile = files["summary.json"];
const readme = files["README.txt"];
if (!summary || !readme) throw new Error("missing support bundle metadata");
expect(strFromU8(summary)).toContain("model_failed");
expect(strFromU8(readme)).toContain("蓝莓没出来");
if (!eventFile || !summaryFile || !readme)
throw new Error("missing support bundle metadata");
const exportedEvent = JSON.parse(strFromU8(eventFile).trim());
expect(exportedEvent.details).toEqual({ count: 1 });
const summary = JSON.parse(strFromU8(summaryFile));
expect(summary.background).toEqual({
status: "exported",
runtime_state: "ready",
core_state: "healthy",
broker_state: "ready",
instance_id: "instance-1",
});
expect(summary.sources).toBeUndefined();
expect(summary.summary_secret).toBeUndefined();
expect(summary.problem_markers).toEqual([
{
timestamp: "2026-10-05T01:30:00.000Z",
description: "user_marked_incident",
},
]);
expect(strFromU8(readme)).toContain("user_marked_incident");
expect(strFromU8(readme)).not.toContain("蓝莓没出来");
const all = Object.values(files)
.map((value) => strFromU8(value))
.join("\n");
expect(all).not.toContain(secret);
expect(all).not.toContain("ghp_private_token");
expect(all).not.toContain("xoxb-123456-abcdef");
expect(all).not.toContain("AKIDExampleSecretId1234567890");
expect(all).not.toContain("ASIA1234567890ABCDEF");
expect(all).not.toContain("GOCSPX-clientSecret_A1");
expect(all).not.toContain("Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8");
expect(all).not.toContain("AIzaSyExampleCredential1234567890");
expect(all).not.toContain("eyJhbGciOiJIUzI1NiJ9");
expect(all).not.toContain(conversation);
expect(all).not.toContain(home);
expect(all).not.toContain("/Users/");
expect(all).toContain("~/MemHub");
});
it("fails closed without publishing a ZIP when a diagnostic line is invalid", async () => {
const dataDir = await root();
await mkdir(path.join(dataDir, "logs"), { recursive: true });
await writeFile(
path.join(dataDir, "logs", "diagnostic-core.jsonl"),
"not-json\n",
);
const outputDir = await root();
const target = path.join(outputDir, "MemHub.zip");
await expect(
exportDiagnosticZip({
dataDir,
target,
productVersion: "3800",
}),
).rejects.toThrow("diagnostic_export_invalid_lines");
expect(await readdir(outputDir)).toEqual([]);
});
it("stores and exports only the fixed marker description", async () => {
const dataDir = await root();
const markerFile = path.join(dataDir, "logs", "problem-markers.jsonl");
await recordDiagnosticMarker(
dataDir,
{
timestamp: "2026-10-05T01:00:00.000Z",
description:
"蓝莓 /Users/private/query.txt ASIA1234567890ABCDEF GOCSPX-clientSecret_A1 Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
},
"/Users/private",
);
expect(await readFile(markerFile, "utf8")).toBe(
`${JSON.stringify({ timestamp: "2026-10-05T01:00:00.000Z", description: "user_marked_incident" })}\n`,
);
await writeFile(
markerFile,
`${await readFile(markerFile, "utf8")}${JSON.stringify({ timestamp: "2026-10-05T01:30:00.000Z", description: "legacy arbitrary text" })}\n`,
{ mode: 0o600 },
);
const target = path.join(await root(), "markers.zip");
await exportDiagnosticZip({
dataDir,
target,
productVersion: "3800",
now: new Date("2026-10-05T02:00:00.000Z"),
});
const files = unzipSync(await readFile(target));
const summaryFile = files["summary.json"];
if (!summaryFile) throw new Error("missing summary.json");
const payload = JSON.stringify(JSON.parse(strFromU8(summaryFile)));
expect(payload.match(/user_marked_incident/g)).toHaveLength(2);
expect(payload).not.toContain("legacy arbitrary text");
expect(payload).not.toContain("蓝莓");
expect(payload).not.toContain("/Users/");
});
it("never follows a problem marker symlink for reads or writes", async () => {
const dataDir = await root();
const logs = path.join(dataDir, "logs");
await mkdir(logs, { recursive: true });
const outside = path.join(await root(), "outside-marker");
await writeFile(outside, "private");
await symlink(outside, path.join(logs, "problem-markers.jsonl"));
await expect(
recordDiagnosticMarker(dataDir, {
timestamp: "2026-10-05T01:00:00.000Z",
description: "anything",
}),
).rejects.toThrow("diagnostic_marker_file_unsafe");
await expect(
exportDiagnosticZip({
dataDir,
target: path.join(await root(), "markers.zip"),
productVersion: "3800",
}),
).rejects.toThrow("diagnostic_marker_file_unsafe");
expect(await readFile(outside, "utf8")).toBe("private");
});
it("rejects invalid explicit and persisted markers without publishing a ZIP", async () => {
const dataDir = await root();
const logs = path.join(dataDir, "logs");
await mkdir(logs, { recursive: true });
const outputDir = await root();
const explicitTarget = path.join(outputDir, "explicit.zip");
await expect(
exportDiagnosticZip({
dataDir,
target: explicitTarget,
productVersion: "3800",
markers: [{ timestamp: "invalid", description: "marker" }],
}),
).rejects.toThrow("diagnostic_marker_invalid");
await writeFile(path.join(logs, "problem-markers.jsonl"), "not-json\n");
const persistedTarget = path.join(outputDir, "persisted.zip");
await expect(
exportDiagnosticZip({
dataDir,
target: persistedTarget,
productVersion: "3800",
}),
).rejects.toThrow("diagnostic_marker_invalid");
expect(await readdir(outputDir)).toEqual([]);
});
it("an incident retrieves its surrounding session rather than only its marker", async () => {
File diff suppressed because it is too large. Load diff
@@ -59,6 +59,7 @@ export {
diagnosticZipFilename,
exportDiagnosticBundle,
exportDiagnosticZip,
projectDiagnosticEventForSafeOutput,
pruneDiagnosticLogs,
readDiagnosticEvents,
recordDiagnosticMarker,