Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
754555e3c3 | ||
|
|
b43c967baa |
No files matched your search
@@ -3,6 +3,7 @@ import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
claimRuntimeServiceForApp,
|
||||
clearSdkMigrationBlock,
|
||||
readRuntimeServiceOwner,
|
||||
recordSdkMigrationBlock,
|
||||
resolveRuntimeCliConfig,
|
||||
@@ -922,6 +923,38 @@ describe("public One Memory CLI", () => {
|
||||
target_path: "/tmp/install/current/bin/one-memory",
|
||||
};
|
||||
|
||||
it("says the data was restored when a failed migration put it back", async () => {
|
||||
vi.clearAllMocks();
|
||||
runtimeMocks.planCliLink.mockResolvedValueOnce(presentCliLink);
|
||||
runtimeMocks.runSdkMigration.mockResolvedValueOnce({
|
||||
state: "deferred",
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
dataRestored: true,
|
||||
} as never);
|
||||
const streams = captureStreams();
|
||||
let code: number;
|
||||
try {
|
||||
code = await run(
|
||||
upgradeArguments,
|
||||
{},
|
||||
{ waitForRuntimeReady: async () => true },
|
||||
);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(code!).toBe(1);
|
||||
const failure = JSON.parse(streams.stderr());
|
||||
expect(failure).toMatchObject({
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
data_restored: true,
|
||||
log_file: "/tmp/custom data/logs/sdk-migration.log",
|
||||
});
|
||||
expect(failure.message).toContain("记忆数据已恢复到升级前的状态。");
|
||||
expect(failure.message).toContain(
|
||||
"/tmp/custom data/logs/sdk-migration.log",
|
||||
);
|
||||
});
|
||||
|
||||
it("migrates the home with the new Core before restarting the Runtime", async () => {
|
||||
vi.clearAllMocks();
|
||||
runtimeMocks.planCliLink.mockResolvedValueOnce(presentCliLink);
|
||||
@@ -2134,6 +2167,94 @@ describe("lifecycle commands report a migration block", () => {
|
||||
expect(output.value()).not.toContain("runtime_not_ready_within_budget");
|
||||
});
|
||||
|
||||
it("restart on a frozen home retries the migration in the foreground first", async () => {
|
||||
const dataDir = await mkdtemp(path.join(tmpdir(), "memhub-frozen-"));
|
||||
await recordSdkMigrationBlock(
|
||||
dataDir,
|
||||
"sdk_migration_memory_unreadable",
|
||||
"desktop",
|
||||
{ frozen: { service_was_enabled: true }, data_restored: true },
|
||||
);
|
||||
const controls = {
|
||||
...blockedControls(),
|
||||
migrateSdkData: vi.fn(async () => ({
|
||||
state: "deferred" as const,
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
dataRestored: true,
|
||||
})),
|
||||
};
|
||||
const output = captureStdout();
|
||||
let code: number;
|
||||
try {
|
||||
code = await run(
|
||||
["restart", "--json"],
|
||||
{ ONE_MEMORY_DATA_DIR: dataDir },
|
||||
controls,
|
||||
);
|
||||
} finally {
|
||||
output.restore();
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
expect(code!).toBe(1);
|
||||
expect(controls.migrateSdkData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ dataDir, caller: "cli-restart" }),
|
||||
);
|
||||
// Still frozen: the service is not started to retry on its own.
|
||||
expect(controls.restartManagedRuntime).not.toHaveBeenCalled();
|
||||
expect(controls.ensureRuntime).not.toHaveBeenCalled();
|
||||
expect(JSON.parse(output.value())).toMatchObject({
|
||||
command: "restart",
|
||||
state: "blocked",
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
data_restored: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("restart on a frozen home restarts the Runtime once the retry succeeds", async () => {
|
||||
const dataDir = await mkdtemp(path.join(tmpdir(), "memhub-frozen-"));
|
||||
await recordSdkMigrationBlock(
|
||||
dataDir,
|
||||
"sdk_migration_memory_unreadable",
|
||||
"desktop",
|
||||
{ frozen: { service_was_enabled: false } },
|
||||
);
|
||||
const order: string[] = [];
|
||||
const migrateSdkData = vi.fn(async () => {
|
||||
order.push("migrate");
|
||||
await clearSdkMigrationBlock(dataDir);
|
||||
return { state: "cutover" as const };
|
||||
});
|
||||
const output = captureStreams();
|
||||
let code: number;
|
||||
try {
|
||||
code = await run(
|
||||
["restart", "--json"],
|
||||
{ ONE_MEMORY_DATA_DIR: dataDir },
|
||||
{
|
||||
migrateSdkData,
|
||||
waitForRuntimeReady: vi.fn(async () => true),
|
||||
restartManagedRuntime: vi.fn(async () => {
|
||||
order.push("restart");
|
||||
return "restarted" as const;
|
||||
}),
|
||||
stopRuntime: vi.fn(async () => "stopped" as const),
|
||||
ensureRuntime: vi.fn(async () => {
|
||||
order.push("restart");
|
||||
return { pid: 1 } as never;
|
||||
}),
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
output.restore();
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
expect(code!).toBe(0);
|
||||
expect(migrateSdkData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ caller: "cli-restart" }),
|
||||
);
|
||||
expect(order).toEqual(["migrate", "restart"]);
|
||||
});
|
||||
|
||||
it("restart in text mode names the log file and GitHub issues", async () => {
|
||||
const dataDir = await blockedHome();
|
||||
const output = captureStdout();
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
processIsAlive,
|
||||
type ResolvedRuntimeRelease,
|
||||
RuntimeDiscoveryError,
|
||||
readInstalledRuntimeSource,
|
||||
readSdkMigrationBlock,
|
||||
removeRuntimeServiceOwner,
|
||||
resolveRuntimeCliConfig,
|
||||
@@ -43,6 +44,7 @@ import {
|
||||
runSdkMigration,
|
||||
runtimeInstallLayout,
|
||||
type SdkMigrationResult,
|
||||
type SdkMigrationSource,
|
||||
sdkMigrationLogPath,
|
||||
stopRuntimeProcess,
|
||||
} from "@one-memory/runtime-control";
|
||||
@@ -1166,6 +1168,41 @@ async function runCommand(
|
||||
// declared command, and lifecycle is exactly where scripts need it.
|
||||
if (arguments_.length !== (asJson ? 2 : 1)) return usage();
|
||||
const config = resolveRuntimeCliConfig(environment, process.argv[1]);
|
||||
// A failed upgrade migration froze this home: its service exits without
|
||||
// migrating. `start`/`restart` is an explicit retry, run here in the
|
||||
// foreground; success lifts the freeze and turns the service back on.
|
||||
const frozen = await readSdkMigrationBlock(config.dataDir).catch(
|
||||
() => null,
|
||||
);
|
||||
if (frozen?.frozen !== undefined) {
|
||||
const migration = await (
|
||||
controls.migrateSdkData ?? migrateSdkDataBeforeStart
|
||||
)({
|
||||
coreCommand: config.coreCommand,
|
||||
dataDir: config.dataDir,
|
||||
runtimeBuild:
|
||||
config.buildIdentity?.buildNumber ??
|
||||
config.buildIdentity?.version ??
|
||||
"unknown",
|
||||
asJson,
|
||||
caller: command === "restart" ? "cli-restart" : "cli-start",
|
||||
});
|
||||
if (migration.state === "deferred") {
|
||||
const failure = sdkMigrationFailure(
|
||||
command,
|
||||
migration.code ?? "sdk_migration_incomplete",
|
||||
config.dataDir,
|
||||
"blocked",
|
||||
migration.dataRestored === true,
|
||||
);
|
||||
process.stdout.write(
|
||||
asJson
|
||||
? `${JSON.stringify({ ...failure, state: "blocked" })}\n`
|
||||
: `${failure.message}\n`,
|
||||
);
|
||||
return EXIT_FAILED;
|
||||
}
|
||||
}
|
||||
const enableSnapshot = await setManagedServiceEnabled(config.dataDir, true);
|
||||
const restoreEnableState = () =>
|
||||
restoreManagedServiceEnabled(config.dataDir, enableSnapshot);
|
||||
@@ -3125,6 +3162,7 @@ async function runCommand(
|
||||
// `null` until the migration is handed this home; "failed" if it threw.
|
||||
let migrationState: SdkMigrationResult["state"] | "failed" | null = null;
|
||||
let deferredCode: string | null = null;
|
||||
let deferredRestored = false;
|
||||
let previousOwner:
|
||||
| Awaited<ReturnType<typeof claimCliServiceOwnership>>
|
||||
| undefined;
|
||||
@@ -3146,6 +3184,10 @@ async function runCommand(
|
||||
runtimeBuild: String(plan.build_number ?? plan.version),
|
||||
asJson: true,
|
||||
caller: "cli-install",
|
||||
sourceRuntime: await readInstalledRuntimeSource(
|
||||
plan.install_root,
|
||||
receipt.previous_current_target,
|
||||
),
|
||||
});
|
||||
migrationState = migration.state;
|
||||
// A deferred migration keeps this install: the new Runtime service is
|
||||
@@ -3154,6 +3196,7 @@ async function runCommand(
|
||||
// Runtime, which has no gate and would serve the legacy data.
|
||||
if (migration.state === "deferred") {
|
||||
deferredCode = migration.code ?? "sdk_migration_incomplete";
|
||||
deferredRestored = migration.dataRestored === true;
|
||||
}
|
||||
migrated =
|
||||
migration.state === "cutover" || migration.state === "complete";
|
||||
@@ -3233,7 +3276,13 @@ async function runCommand(
|
||||
process.stderr.write(
|
||||
`${JSON.stringify(
|
||||
deferredCode !== null
|
||||
? sdkMigrationFailure("install", deferredCode, layout.dataDir)
|
||||
? sdkMigrationFailure(
|
||||
"install",
|
||||
deferredCode,
|
||||
layout.dataDir,
|
||||
"upgrade",
|
||||
deferredRestored,
|
||||
)
|
||||
: migrated
|
||||
? notReadyAfterMigration("install")
|
||||
: migrationState === "failed"
|
||||
@@ -3672,6 +3721,10 @@ async function runCommand(
|
||||
runtimeBuild: String(plan.build_number ?? plan.version),
|
||||
asJson,
|
||||
caller: "cli-upgrade",
|
||||
sourceRuntime: await readInstalledRuntimeSource(
|
||||
plan.install_root,
|
||||
receipt.previous_current_target,
|
||||
),
|
||||
});
|
||||
migrationState = migration.state;
|
||||
if (migration.state === "deferred") {
|
||||
@@ -3680,7 +3733,7 @@ async function runCommand(
|
||||
// rolling back would restart the old Runtime, which has no such gate.
|
||||
await keepUpgradedRuntimeOffLegacyHome();
|
||||
process.stderr.write(
|
||||
`${JSON.stringify(sdkMigrationFailure("upgrade", migration.code ?? "sdk_migration_incomplete", parsed.dataDir ?? layout.dataDir), null, 2)}\n`,
|
||||
`${JSON.stringify(sdkMigrationFailure("upgrade", migration.code ?? "sdk_migration_incomplete", parsed.dataDir ?? layout.dataDir, "upgrade", migration.dataRestored === true), null, 2)}\n`,
|
||||
);
|
||||
return EXIT_FAILED;
|
||||
}
|
||||
@@ -4135,13 +4188,17 @@ async function migrateSdkDataBeforeStart(options: {
|
||||
dataDir: string;
|
||||
runtimeBuild: string;
|
||||
asJson: boolean;
|
||||
caller?: "cli-install" | "cli-upgrade";
|
||||
caller?: "cli-install" | "cli-upgrade" | "cli-start" | "cli-restart";
|
||||
sourceRuntime?: SdkMigrationSource | null;
|
||||
}): Promise<SdkMigrationResult> {
|
||||
return runSdkMigration({
|
||||
dataDir: options.dataDir,
|
||||
core: { command: options.coreCommand },
|
||||
runtimeBuild: options.runtimeBuild,
|
||||
...(options.caller === undefined ? {} : { caller: options.caller }),
|
||||
...(options.sourceRuntime === undefined
|
||||
? {}
|
||||
: { sourceRuntime: options.sourceRuntime }),
|
||||
onProgress: (step) => {
|
||||
process.stderr.write(
|
||||
options.asJson
|
||||
@@ -4220,8 +4277,12 @@ function sdkMigrationFailure(
|
||||
code: string,
|
||||
dataDir: string,
|
||||
context: "upgrade" | "blocked" = "upgrade",
|
||||
dataRestored = false,
|
||||
) {
|
||||
const logFile = sdkMigrationLogPath(dataDir);
|
||||
// Core put the home back exactly as it was before the upgrade, so the next
|
||||
// (fixed) version migrates from the original data.
|
||||
const restored = dataRestored ? "记忆数据已恢复到升级前的状态。" : "";
|
||||
const reason =
|
||||
SDK_MIGRATION_FAILURES[code] ??
|
||||
"记忆数据迁移没有完成。可以运行 memhub restart 重试。";
|
||||
@@ -4235,9 +4296,10 @@ function sdkMigrationFailure(
|
||||
command,
|
||||
status: "error",
|
||||
code,
|
||||
message: `${lead}原有记忆数据不会丢失。迁移日志:${logFile}。请在 GitHub 提交反馈并附上这个日志文件:${MIGRATION_ISSUES_URL}`,
|
||||
message: `${lead}原有记忆数据不会丢失。${restored}迁移日志:${logFile}。请在 GitHub 提交反馈并附上这个日志文件:${MIGRATION_ISSUES_URL}`,
|
||||
log_file: logFile,
|
||||
issues_url: MIGRATION_ISSUES_URL,
|
||||
...(dataRestored ? { data_restored: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4260,7 +4322,13 @@ async function runtimeMigrationBlock(
|
||||
): Promise<ReturnType<typeof sdkMigrationFailure> | null> {
|
||||
const block = await readSdkMigrationBlock(dataDir).catch(() => null);
|
||||
if (block === null) return null;
|
||||
return sdkMigrationFailure(command, block.code, dataDir, "blocked");
|
||||
return sdkMigrationFailure(
|
||||
command,
|
||||
block.code,
|
||||
dataDir,
|
||||
"blocked",
|
||||
block.data_restored === true,
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForRuntimeReady(
|
||||
@@ -4279,8 +4347,14 @@ async function waitForRuntimeReady(
|
||||
// The service deferred its migration during this wait and will not spawn
|
||||
// Core: stop waiting. An older note may predate a retry still running, so
|
||||
// only a note written after the wait began ends it early.
|
||||
// A frozen home: its service exits without migrating, so nothing will
|
||||
// come up until a foreground migration lifts the freeze.
|
||||
const blocked = await readSdkMigrationBlock(dataDir).catch(() => null);
|
||||
if (blocked !== null && Date.parse(blocked.at) >= started) return false;
|
||||
if (
|
||||
blocked !== null &&
|
||||
(blocked.frozen !== undefined || Date.parse(blocked.at) >= started)
|
||||
)
|
||||
return false;
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
return false;
|
||||
|
||||
@@ -314,6 +314,10 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
details={
|
||||
"runtime_build": runtime_build,
|
||||
"confirmed": confirmed,
|
||||
# The Runtime the upgrade replaced, as the orchestrator read it
|
||||
# from that Runtime's install manifest; never guessed.
|
||||
"source_version": _migration_source("ONE_MEMORY_MIGRATION_SOURCE_VERSION"),
|
||||
"source_build": _migration_source("ONE_MEMORY_MIGRATION_SOURCE_BUILD"),
|
||||
**authority_snapshot(database_path),
|
||||
},
|
||||
)
|
||||
@@ -386,6 +390,7 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
stream.write(f"{step_names.get(step, step)}\n")
|
||||
stream.flush()
|
||||
|
||||
restore: dict[str, object] = {}
|
||||
try:
|
||||
with _migration_lock(settings.data_dir):
|
||||
result = run_startup_migration(
|
||||
@@ -394,6 +399,7 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
sdk_version=HY_MEMORY_VERSION,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
progress=progress,
|
||||
restore_report=restore,
|
||||
)
|
||||
except Exception as error:
|
||||
from one_memory.sdk_migration_verify import MemoryReadbackFailed
|
||||
@@ -402,6 +408,10 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
failed: dict[str, object] = {"schema_version": 1, "state": "failed", "code": code}
|
||||
if isinstance(error, MemoryReadbackFailed):
|
||||
failed["readback"] = error.report.as_document()
|
||||
# The migration's own code stays the result; the restore is secondary.
|
||||
failed["data_restored"] = restore.get("state") == "restored"
|
||||
if restore.get("state") == "failed":
|
||||
failed["restore_code"] = restore.get("code")
|
||||
write_migration_log(
|
||||
settings.data_dir,
|
||||
writer="core-sdk-migrate",
|
||||
@@ -417,10 +427,13 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
"code_is_fallback": code == "sdk_migration_preflight_failed"
|
||||
and not _names_code(error, code),
|
||||
**({"readback": failed["readback"]} if "readback" in failed else {}),
|
||||
"restore": restore.get("state", "not_attempted"),
|
||||
**({"restore_code": restore["code"]} if "code" in restore else {}),
|
||||
**authority_snapshot(database_path),
|
||||
},
|
||||
)
|
||||
_write_restore(stream, as_json, failed, f"迁移未完成:{code}")
|
||||
restored_text = ";数据已恢复到升级前的状态" if failed["data_restored"] else ""
|
||||
_write_restore(stream, as_json, failed, f"迁移未完成:{code}{restored_text}")
|
||||
return 1
|
||||
document = result.as_document()
|
||||
write_migration_log(
|
||||
@@ -446,6 +459,14 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
_SAFE_SOURCE = re.compile(r"^[0-9A-Za-z][0-9A-Za-z.+\-]{0,63}$")
|
||||
|
||||
|
||||
def _migration_source(name: str) -> str:
|
||||
value = os.environ.get(name, "").strip()
|
||||
return value if _SAFE_SOURCE.fullmatch(value) else "unknown"
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _migration_lock(data_dir: Path) -> Iterator[None]:
|
||||
"""One `sdk-migrate` per home at a time.
|
||||
@@ -545,8 +566,18 @@ def _sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
|
||||
as_json = "--json" in options
|
||||
settings = load_settings()
|
||||
database_path = settings.data_dir / "one-memory.sqlite3"
|
||||
if not database_path.is_file():
|
||||
from one_memory.sdk_migration_restore import restore_pending
|
||||
|
||||
if restore_pending(settings.data_dir):
|
||||
# A restore was killed between its renames; the database may be
|
||||
# elsewhere. Only `sdk-migrate` may finish it.
|
||||
document: dict[str, object] = {
|
||||
"schema_version": 1,
|
||||
"authority": "legacy",
|
||||
"state": "required",
|
||||
}
|
||||
elif not database_path.is_file():
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
"authority": "sdk",
|
||||
"state": "not_required",
|
||||
@@ -619,8 +650,22 @@ def _refuse_unmigrated_home(data_dir: Path) -> None:
|
||||
"""
|
||||
from one_memory.sdk_data_plane_migration import MemoryAuthority, SdkDataPlaneMigrationStore
|
||||
from one_memory.sdk_migration_log import authority_snapshot, write_migration_log
|
||||
from one_memory.sdk_migration_restore import restore_pending
|
||||
|
||||
database_path = data_dir / "one-memory.sqlite3"
|
||||
if restore_pending(data_dir):
|
||||
# A killed restore may have moved the database aside: serving now
|
||||
# would create an empty home over the user's data.
|
||||
write_migration_log(
|
||||
data_dir,
|
||||
writer="core-startup",
|
||||
event="core.refused_legacy",
|
||||
outcome="refused",
|
||||
code="sdk_migration_required",
|
||||
details={"stage": "serve_entry", "restore_pending": True},
|
||||
)
|
||||
print("sdk_migration_required", file=sys.stderr, flush=True)
|
||||
raise SystemExit(SDK_MIGRATION_REQUIRED_EXIT)
|
||||
if not database_path.is_file():
|
||||
return
|
||||
try:
|
||||
|
||||
@@ -105,10 +105,17 @@ def capture_stdio(source_id: str, stdin: TextIO, output: TextIO) -> int:
|
||||
|
||||
|
||||
def sdk_migration_capture_fenced(data_dir: Path) -> bool:
|
||||
"""Ignore stale fences; only a live Desktop migration pauses hook writes."""
|
||||
"""Ignore stale fences; a live migration or a failed one pauses hook writes.
|
||||
|
||||
A failed upgrade migration leaves the fence `frozen` (no pid): the home was
|
||||
put back as it was before the upgrade and takes no writes until the next
|
||||
migration attempt, which replaces the fence and lifts it on success.
|
||||
"""
|
||||
marker = data_dir / SDK_MIGRATION_CAPTURE_FENCE
|
||||
try:
|
||||
document = json.loads(marker.read_text(encoding="utf-8"))
|
||||
if isinstance(document, dict) and document.get("frozen") is True:
|
||||
return True
|
||||
pid = int(document["pid"])
|
||||
if pid < 1:
|
||||
raise ValueError("invalid migration fence pid")
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
"""Put a home back exactly as it was when a failed `sdk-migrate` started.
|
||||
|
||||
A migration that fails before SDK authority is committed has already changed
|
||||
the home: schema migrations on `one-memory.sqlite3`, legacy work moved into the
|
||||
durable queue, renamed Chroma collections under `hy-memory/governed-v2`, ledger
|
||||
rows. The fixed release that follows must start from the original data, not
|
||||
from that half-way state, so the runner restores the verified full-home bundle
|
||||
it took before the first change (`restore point`).
|
||||
|
||||
The restore is staged and then swapped in with renames:
|
||||
|
||||
1. the bundle is verified (every file hashed against its manifest);
|
||||
2. its database, config and SDK root are copied next to their live places and
|
||||
verified again;
|
||||
3. a journal lists every rename, then the live items (database with its
|
||||
`-wal`/`-shm`, `config.json`, `hy-memory/governed-v2`) are renamed into
|
||||
`backups/sdk-migration-failed/` and the staged copies are renamed in.
|
||||
|
||||
A rename either happened or did not, and each step is idempotent, so a process
|
||||
killed during step 3 is finished by the next run (`finish_interrupted_restore`)
|
||||
before anything else reads the home. Core refuses to serve and the status
|
||||
probe reports `required` while a journal exists.
|
||||
|
||||
After SDK authority is committed nothing is restored: the home already belongs
|
||||
to the SDK and the legacy tables may be gone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from one_memory.database_access import close_main_databases
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
|
||||
from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
|
||||
|
||||
RESTORE_POINT = ".sdk-migration-restore-point.json"
|
||||
RESTORE_STAGING = ".sdk-migration-restore"
|
||||
JOURNAL = "journal.json"
|
||||
FAILED_STATE = "sdk-migration-failed"
|
||||
RESTORED_MARK = "restored.json"
|
||||
DATABASE = "one-memory.sqlite3"
|
||||
CONFIG = "config.json"
|
||||
_DATABASE_SIDECARS = ("-wal", "-shm", "-journal")
|
||||
_WRITER = "core-sdk-migrate"
|
||||
|
||||
|
||||
class MigrationRestoreFailed(DataPlaneMigrationError):
|
||||
"""The pre-migration data could not be put back; the home stays blocked."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RestoreResult:
|
||||
bundle_id: str
|
||||
files: int
|
||||
bytes: int
|
||||
duration_ms: float
|
||||
|
||||
def as_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"bundle_id": self.bundle_id,
|
||||
"files": self.files,
|
||||
"bytes": self.bytes,
|
||||
"duration_ms": self.duration_ms,
|
||||
}
|
||||
|
||||
|
||||
def _home(data_dir: Path) -> Path:
|
||||
return data_dir.resolve()
|
||||
|
||||
|
||||
def _staging(home: Path) -> Path:
|
||||
return home / RESTORE_STAGING
|
||||
|
||||
|
||||
def _journal(home: Path) -> Path:
|
||||
return _staging(home) / JOURNAL
|
||||
|
||||
|
||||
def _sdk_staging(home: Path) -> Path:
|
||||
return home / "hy-memory" / f".{HY_MEMORY_NAMESPACE}.restore-staging"
|
||||
|
||||
|
||||
def failed_state_path(data_dir: Path) -> Path:
|
||||
return _home(data_dir) / "backups" / FAILED_STATE
|
||||
|
||||
|
||||
def _write_json(path: Path, document: dict[str, object]) -> None:
|
||||
temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")
|
||||
temporary.write_text(json.dumps(document, sort_keys=True) + "\n", encoding="utf-8")
|
||||
os.chmod(temporary, 0o600)
|
||||
_fsync(temporary)
|
||||
temporary.replace(path)
|
||||
_fsync_directory(path.parent)
|
||||
|
||||
|
||||
def _fsync(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
# Windows cannot open a directory; its renames are durable on return.
|
||||
try:
|
||||
_fsync(path)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
|
||||
def _log(
|
||||
data_dir: Path,
|
||||
event: str,
|
||||
outcome: str,
|
||||
*,
|
||||
migration_id: str | None,
|
||||
code: str | None = None,
|
||||
error: BaseException | None = None,
|
||||
details: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
from one_memory.sdk_migration_log import write_migration_log
|
||||
|
||||
write_migration_log(
|
||||
data_dir,
|
||||
writer=_WRITER,
|
||||
event=event,
|
||||
outcome=outcome,
|
||||
migration_id=migration_id,
|
||||
code=code,
|
||||
error=error,
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
# --- restore point -----------------------------------------------------------
|
||||
|
||||
|
||||
def record_restore_point(data_dir: Path, *, migration_id: str, bundle_id: str) -> None:
|
||||
"""Name the verified bundle that holds this home as the migration found it.
|
||||
|
||||
Written before the first change, so a run killed part-way is rolled back
|
||||
by the next one instead of resumed.
|
||||
"""
|
||||
_write_json(
|
||||
_home(data_dir) / RESTORE_POINT,
|
||||
{
|
||||
"schema_version": 1,
|
||||
"migration_id": migration_id,
|
||||
"bundle_id": bundle_id,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def read_restore_point(data_dir: Path) -> dict[str, str] | None:
|
||||
try:
|
||||
document = json.loads((_home(data_dir) / RESTORE_POINT).read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (OSError, ValueError) as error:
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_point_invalid") from error
|
||||
if (
|
||||
not isinstance(document, dict)
|
||||
or document.get("schema_version") != 1
|
||||
or not isinstance(document.get("migration_id"), str)
|
||||
or not isinstance(document.get("bundle_id"), str)
|
||||
):
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_point_invalid")
|
||||
return {"migration_id": document["migration_id"], "bundle_id": document["bundle_id"]}
|
||||
|
||||
|
||||
def clear_restore_point(data_dir: Path) -> None:
|
||||
(_home(data_dir) / RESTORE_POINT).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def restore_pending(data_dir: Path) -> bool:
|
||||
"""A restore was interrupted between its renames; nothing may read the home."""
|
||||
return _journal(_home(data_dir)).is_file()
|
||||
|
||||
|
||||
# --- restore -----------------------------------------------------------------
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _release_handles() -> None:
|
||||
"""Close what this process holds on the home; Windows cannot rename open files."""
|
||||
close_main_databases()
|
||||
chromadb = sys.modules.get("chromadb")
|
||||
if chromadb is None:
|
||||
return
|
||||
try:
|
||||
from chromadb.api.shared_system_client import SharedSystemClient
|
||||
|
||||
SharedSystemClient.clear_system_cache()
|
||||
except Exception: # noqa: BLE001 - best effort; a held file fails the rename instead
|
||||
return
|
||||
|
||||
|
||||
def _verify_copy(copy: Path, expected: list[dict[str, object]], prefix: str) -> None:
|
||||
"""The staged tree holds exactly the bundle's files under `prefix`."""
|
||||
wanted = {
|
||||
str(item["path"])[len(prefix) :]: item
|
||||
for item in expected
|
||||
if str(item["path"]).startswith(prefix)
|
||||
}
|
||||
present = (
|
||||
{path.relative_to(copy).as_posix() for path in copy.rglob("*") if path.is_file()}
|
||||
if copy.is_dir()
|
||||
else set()
|
||||
)
|
||||
if present != set(wanted):
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_verify_failed")
|
||||
for relative, item in wanted.items():
|
||||
path = copy / relative
|
||||
if path.stat().st_size != int(str(item["bytes"])) or _sha256(path) != item["sha256"]:
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_verify_failed")
|
||||
|
||||
|
||||
def _plan(home: Path, *, config_staged: bool) -> list[tuple[str, str]]:
|
||||
"""Every rename of the swap, as paths relative to the home, in order."""
|
||||
failed = Path("backups") / FAILED_STATE
|
||||
staging = Path(RESTORE_STAGING)
|
||||
sdk_live = Path("hy-memory") / HY_MEMORY_NAMESPACE
|
||||
steps: list[tuple[Path, Path]] = [
|
||||
(Path(f"{DATABASE}{suffix}"), failed / "management" / f"{DATABASE}{suffix}")
|
||||
for suffix in _DATABASE_SIDECARS
|
||||
]
|
||||
steps += [
|
||||
(Path(DATABASE), failed / "management" / DATABASE),
|
||||
(Path(CONFIG), failed / "management" / CONFIG),
|
||||
(sdk_live, failed / "sdk" / HY_MEMORY_NAMESPACE),
|
||||
(staging / "management" / DATABASE, Path(DATABASE)),
|
||||
(_sdk_staging(home).relative_to(home), sdk_live),
|
||||
]
|
||||
if config_staged:
|
||||
steps.append((staging / "management" / CONFIG, Path(CONFIG)))
|
||||
return [(source.as_posix(), target.as_posix()) for source, target in steps]
|
||||
|
||||
|
||||
def _apply(
|
||||
home: Path,
|
||||
steps: list[list[str]] | list[tuple[str, str]],
|
||||
done: list[tuple[Path, Path]],
|
||||
) -> None:
|
||||
"""Run each rename once; a step whose source is already gone is skipped.
|
||||
|
||||
`done` collects the renames this call made, so a caller can undo them.
|
||||
"""
|
||||
for source_name, target_name in steps:
|
||||
source, target = home / source_name, home / target_name
|
||||
source_exists = source.exists() or source.is_symlink()
|
||||
target_exists = target.exists() or target.is_symlink()
|
||||
if not source_exists:
|
||||
continue
|
||||
if target_exists:
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_conflict")
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
source.replace(target)
|
||||
done.append((source, target))
|
||||
for directory in (home, home / "hy-memory"):
|
||||
_fsync_directory(directory)
|
||||
|
||||
|
||||
def _finish(home: Path, bundle_id: str) -> None:
|
||||
# The restore point goes first: a kill after it leaves only the journal,
|
||||
# whose steps are all done, and the next run just tidies up.
|
||||
clear_restore_point(home)
|
||||
shutil.rmtree(_sdk_staging(home), ignore_errors=True)
|
||||
_journal(home).unlink(missing_ok=True)
|
||||
shutil.rmtree(_staging(home), ignore_errors=True)
|
||||
bundle = home / "backups" / f"{bundle_id}.sdk-bundle"
|
||||
try:
|
||||
# A later verified bundle supersedes this one (`prune_restored_bundles`).
|
||||
_write_json(
|
||||
bundle / RESTORED_MARK,
|
||||
{"schema_version": 1, "restored_at": datetime.now(UTC).isoformat()},
|
||||
)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
|
||||
def restore_pre_migration_state(
|
||||
data_dir: Path,
|
||||
*,
|
||||
bundle_id: str,
|
||||
migration_id: str | None,
|
||||
reason: str,
|
||||
) -> RestoreResult:
|
||||
"""Put the restore-point bundle back in place of the half-migrated home.
|
||||
|
||||
The caller holds the migration lock with Runtime stopped and host capture
|
||||
fenced. Raises `MigrationRestoreFailed`; before the swap the home is left as
|
||||
the failure left it, and the restore point stays for the next run.
|
||||
"""
|
||||
home = _home(data_dir)
|
||||
started = time.monotonic()
|
||||
_log(
|
||||
home,
|
||||
"restore.started",
|
||||
"started",
|
||||
migration_id=migration_id,
|
||||
details={"bundle_id": bundle_id, "reason": reason},
|
||||
)
|
||||
try:
|
||||
result = _restore(home, bundle_id=bundle_id, started=started)
|
||||
except BaseException as error:
|
||||
code = getattr(error, "code", None)
|
||||
if not (isinstance(code, str) and code.startswith("sdk_migration_restore")):
|
||||
code = (
|
||||
"sdk_migration_restore_disk_space_insufficient"
|
||||
if isinstance(error, OSError) and error.errno in {errno.ENOSPC, errno.EDQUOT}
|
||||
else "sdk_migration_restore_failed"
|
||||
)
|
||||
_log(
|
||||
home,
|
||||
"restore.failed",
|
||||
"failed",
|
||||
migration_id=migration_id,
|
||||
code=code,
|
||||
error=error,
|
||||
details={
|
||||
"bundle_id": bundle_id,
|
||||
"reason": reason,
|
||||
"duration_ms": (time.monotonic() - started) * 1000,
|
||||
"swap_pending": restore_pending(home),
|
||||
},
|
||||
)
|
||||
if not isinstance(error, Exception) or (
|
||||
isinstance(error, MigrationRestoreFailed) and error.code == code
|
||||
):
|
||||
raise
|
||||
raise MigrationRestoreFailed(code) from error
|
||||
_log(
|
||||
home,
|
||||
"restore.finished",
|
||||
"succeeded",
|
||||
migration_id=migration_id,
|
||||
details={**result.as_document(), "reason": reason},
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _restore(home: Path, *, bundle_id: str, started: float) -> RestoreResult:
|
||||
from one_memory.sdk_migration_bundle import inspect_sdk_migration_bundle
|
||||
|
||||
if restore_pending(home):
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_pending")
|
||||
manifest = inspect_sdk_migration_bundle(data_dir=home, bundle_id=bundle_id)
|
||||
bundle = home / "backups" / f"{bundle_id}.sdk-bundle"
|
||||
listed = manifest.get("files")
|
||||
files = [item for item in listed if isinstance(item, dict)] if isinstance(listed, list) else []
|
||||
total = sum(int(str(item["bytes"])) for item in files)
|
||||
if shutil.disk_usage(home).free < total:
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_disk_space_insufficient")
|
||||
_release_handles()
|
||||
|
||||
staging = _staging(home)
|
||||
sdk_staging = _sdk_staging(home)
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
shutil.rmtree(sdk_staging, ignore_errors=True)
|
||||
(staging / "management").mkdir(parents=True, mode=0o700)
|
||||
try:
|
||||
shutil.copy2(bundle / "management" / DATABASE, staging / "management" / DATABASE)
|
||||
config_staged = manifest.get("config") is not None
|
||||
if config_staged:
|
||||
shutil.copy2(bundle / "management" / CONFIG, staging / "management" / CONFIG)
|
||||
_verify_copy(staging / "management", files, "management/")
|
||||
sdk_source = bundle / "sdk" / HY_MEMORY_NAMESPACE
|
||||
if not sdk_source.is_dir() or sdk_source.is_symlink():
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_bundle_incomplete")
|
||||
sdk_staging.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(sdk_source, sdk_staging, symlinks=True)
|
||||
_verify_copy(sdk_staging, files, f"sdk/{HY_MEMORY_NAMESPACE}/")
|
||||
for path in (staging / "management").iterdir():
|
||||
_fsync(path)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
shutil.rmtree(sdk_staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
failed = failed_state_path(home)
|
||||
# One failed state is kept for diagnosis; the newest replaces the last.
|
||||
shutil.rmtree(failed, ignore_errors=True)
|
||||
if failed.exists():
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_failed_state_busy")
|
||||
steps = _plan(home, config_staged=config_staged)
|
||||
_write_json(
|
||||
_journal(home),
|
||||
{"schema_version": 1, "bundle_id": bundle_id, "steps": [list(step) for step in steps]},
|
||||
)
|
||||
done: list[tuple[Path, Path]] = []
|
||||
try:
|
||||
_apply(home, steps, done)
|
||||
except BaseException as error:
|
||||
if not isinstance(error, Exception):
|
||||
# Interrupted (a signal tearing the process down): leave the
|
||||
# journal; the next run finishes the swap forward.
|
||||
raise
|
||||
try:
|
||||
for source, target in reversed(done):
|
||||
target.replace(source)
|
||||
except OSError:
|
||||
# The journal stays: the next run finishes the swap forward.
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_swap_interrupted") from error
|
||||
_journal(home).unlink(missing_ok=True)
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
shutil.rmtree(sdk_staging, ignore_errors=True)
|
||||
raise
|
||||
_finish(home, bundle_id)
|
||||
return RestoreResult(
|
||||
bundle_id=bundle_id,
|
||||
files=len(files),
|
||||
bytes=total,
|
||||
duration_ms=(time.monotonic() - started) * 1000,
|
||||
)
|
||||
|
||||
|
||||
def finish_interrupted_restore(data_dir: Path, *, migration_id: str | None = None) -> bool:
|
||||
"""Complete a swap a killed process left between its renames. True if it did."""
|
||||
home = _home(data_dir)
|
||||
journal = _journal(home)
|
||||
if not journal.is_file():
|
||||
return False
|
||||
started = time.monotonic()
|
||||
try:
|
||||
document = json.loads(journal.read_text(encoding="utf-8"))
|
||||
steps = document["steps"]
|
||||
bundle_id = str(document["bundle_id"])
|
||||
if not isinstance(steps, list) or not all(
|
||||
isinstance(step, list) and len(step) == 2 for step in steps
|
||||
):
|
||||
raise ValueError("journal steps invalid")
|
||||
_release_handles()
|
||||
_apply(home, steps, [])
|
||||
_finish(home, bundle_id)
|
||||
except Exception as error:
|
||||
code = (
|
||||
error.code
|
||||
if isinstance(error, MigrationRestoreFailed)
|
||||
else "sdk_migration_restore_resume_failed"
|
||||
)
|
||||
_log(
|
||||
home,
|
||||
"restore.failed",
|
||||
"failed",
|
||||
migration_id=migration_id,
|
||||
code=code,
|
||||
error=error,
|
||||
details={"reason": "interrupted_swap", "swap_pending": True},
|
||||
)
|
||||
if isinstance(error, MigrationRestoreFailed):
|
||||
raise
|
||||
raise MigrationRestoreFailed(code) from error
|
||||
_log(
|
||||
home,
|
||||
"restore.finished",
|
||||
"succeeded",
|
||||
migration_id=migration_id,
|
||||
details={
|
||||
"bundle_id": bundle_id,
|
||||
"reason": "interrupted_swap",
|
||||
"duration_ms": (time.monotonic() - started) * 1000,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def prune_restored_bundles(data_dir: Path, *, keep: set[str]) -> int:
|
||||
"""Drop bundles already restored, once a newer verified bundle exists.
|
||||
|
||||
Each holds the same pre-migration home a newer bundle now holds (capture
|
||||
stayed fenced since), so it is only a second copy of it.
|
||||
"""
|
||||
backups = _home(data_dir) / "backups"
|
||||
removed = 0
|
||||
for mark in backups.glob(f"*.sdk-bundle/{RESTORED_MARK}"):
|
||||
bundle = mark.parent
|
||||
if bundle.name.removesuffix(".sdk-bundle") in keep:
|
||||
continue
|
||||
shutil.rmtree(bundle, ignore_errors=True)
|
||||
removed += 0 if bundle.exists() else 1
|
||||
return removed
|
||||
|
||||
|
||||
def discard_failed_state(data_dir: Path) -> None:
|
||||
"""The failed half-migrated copy is only useful until a migration succeeds."""
|
||||
shutil.rmtree(failed_state_path(data_dir), ignore_errors=True)
|
||||
@@ -18,6 +18,7 @@ import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -483,30 +484,50 @@ def _resumable_identity(database_path: Path) -> tuple[str, str]:
|
||||
return migration_id, f"{migration_id}-pre-cutover"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_fence(data_dir: Path) -> Iterator[None]:
|
||||
"""Host capture hooks return silently while this process is alive."""
|
||||
def _write_capture_fence(data_dir: Path, document: dict[str, object]) -> None:
|
||||
marker = data_dir / CAPTURE_FENCE
|
||||
temporary = marker.with_name(f"{marker.name}.{os.getpid()}.tmp")
|
||||
temporary.write_text(
|
||||
json.dumps({"schema_version": 1, "pid": os.getpid()}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.write_text(json.dumps(document) + "\n", encoding="utf-8")
|
||||
os.chmod(temporary, 0o600)
|
||||
temporary.replace(marker)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_fence(data_dir: Path) -> Iterator[None]:
|
||||
"""Host capture hooks return silently while this process is alive.
|
||||
|
||||
A failed run leaves the fence frozen instead of lifting it: the home was
|
||||
put back (or is half-migrated) and must not take new writes until the next
|
||||
migration attempt replaces the fence with its own and succeeds.
|
||||
"""
|
||||
marker = data_dir / CAPTURE_FENCE
|
||||
_write_capture_fence(data_dir, {"schema_version": 1, "pid": os.getpid()})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
except BaseException:
|
||||
try:
|
||||
_write_capture_fence(
|
||||
data_dir, {"schema_version": 1, "frozen": True, "reason": "sdk_migration_failed"}
|
||||
)
|
||||
except OSError:
|
||||
marker.unlink(missing_ok=True)
|
||||
raise
|
||||
else:
|
||||
marker.unlink(missing_ok=True)
|
||||
temporary.unlink(missing_ok=True)
|
||||
finally:
|
||||
marker.with_name(f"{marker.name}.{os.getpid()}.tmp").unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _ensure_bundle(*, data_dir: Path, bundle_id: str, runtime_build: str) -> dict[str, object]:
|
||||
def _ensure_bundle(
|
||||
*, data_dir: Path, bundle_id: str, runtime_build: str
|
||||
) -> tuple[dict[str, object], bool]:
|
||||
"""The verified bundle, and whether this call created it."""
|
||||
from one_memory.sdk_migration_bundle import (
|
||||
MigrationBundleRejected,
|
||||
create_sdk_migration_bundle,
|
||||
)
|
||||
|
||||
created = True
|
||||
try:
|
||||
create_sdk_migration_bundle(
|
||||
data_dir=data_dir,
|
||||
@@ -517,7 +538,67 @@ def _ensure_bundle(*, data_dir: Path, bundle_id: str, runtime_build: str) -> dic
|
||||
except MigrationBundleRejected as error:
|
||||
if error.code != "sdk_migration_bundle_exists":
|
||||
raise
|
||||
return inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=bundle_id)
|
||||
created = False
|
||||
return inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=bundle_id), created
|
||||
|
||||
|
||||
def _restore_point_bundle(
|
||||
*, data_dir: Path, migration_id: str, bundle_id: str, created: bool, runtime_build: str
|
||||
) -> str:
|
||||
"""The bundle that holds this home exactly as this run found it.
|
||||
|
||||
The cutover bundle qualifies when this run created it. One left by an
|
||||
earlier run (an older build's blocked ledger row) may predate later writes,
|
||||
so restoring it could drop them; a fresh restore-point bundle is taken.
|
||||
"""
|
||||
if created:
|
||||
return bundle_id
|
||||
from one_memory.sdk_migration_bundle import create_sdk_migration_bundle
|
||||
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ")
|
||||
restore_id = f"{migration_id}-restore-point-{stamp}"
|
||||
create_sdk_migration_bundle(
|
||||
data_dir=data_dir,
|
||||
bundle_id=restore_id,
|
||||
runtime_build=runtime_build,
|
||||
runtime_stopped=True,
|
||||
)
|
||||
inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=restore_id)
|
||||
return restore_id
|
||||
|
||||
|
||||
def _recover_interrupted_attempt(data_dir: Path, database_path: Path, log: _StepLog) -> None:
|
||||
"""Roll back what a killed earlier run left, before anything reads the home."""
|
||||
from one_memory.sdk_migration_restore import (
|
||||
MigrationRestoreFailed,
|
||||
clear_restore_point,
|
||||
finish_interrupted_restore,
|
||||
read_restore_point,
|
||||
restore_pre_migration_state,
|
||||
)
|
||||
|
||||
finish_interrupted_restore(data_dir)
|
||||
point = read_restore_point(data_dir)
|
||||
if point is None:
|
||||
return
|
||||
with log.step("preflight.interrupted_attempt") as step:
|
||||
try:
|
||||
authority = SdkDataPlaneMigrationStore(database_path).authority()
|
||||
except Exception as error:
|
||||
# Never restore over a home that may already be on SDK authority.
|
||||
raise MigrationRestoreFailed("sdk_migration_restore_authority_unknown") from error
|
||||
step["authority"] = authority.value
|
||||
if authority is MemoryAuthority.SDK:
|
||||
clear_restore_point(data_dir)
|
||||
step["restored"] = False
|
||||
return
|
||||
restore_pre_migration_state(
|
||||
data_dir,
|
||||
bundle_id=point["bundle_id"],
|
||||
migration_id=point["migration_id"],
|
||||
reason="interrupted_attempt",
|
||||
)
|
||||
step["restored"] = True
|
||||
|
||||
|
||||
def _vacuum(database_path: Path) -> None:
|
||||
@@ -537,14 +618,28 @@ def run_startup_migration(
|
||||
namespace: str,
|
||||
progress: Callable[[str], None] = lambda _step: None,
|
||||
open_client: Callable[[ModelProfileSnapshot], AbstractContextManager[Any]] | None = None,
|
||||
restore_report: dict[str, object] | None = None,
|
||||
) -> StartupMigrationResult:
|
||||
"""Bring one stopped home to SDK authority. Safe to repeat after any failure.
|
||||
|
||||
The caller must stop Runtime and keep its service from restarting.
|
||||
`open_client` yields a public SDK client configured as the Runtime will be;
|
||||
tests replace it.
|
||||
|
||||
A failure before SDK authority is committed puts the home back exactly as
|
||||
this run found it (`sdk_migration_restore`), so the next run, possibly of a
|
||||
fixed release, starts from the original data. The original exception is
|
||||
re-raised either way; `restore_report` learns what the restore did
|
||||
(`state`: `restored`, `failed` with its own `code`, or `skipped`).
|
||||
"""
|
||||
from one_memory.sdk_migration_restore import read_restore_point, restore_pending
|
||||
|
||||
database_path = data_dir / "one-memory.sqlite3"
|
||||
log = _StepLog(data_dir)
|
||||
# Before anything reads the home: a run killed part-way is rolled back.
|
||||
if restore_pending(data_dir) or read_restore_point(data_dir) is not None:
|
||||
with capture_fence(data_dir):
|
||||
_recover_interrupted_attempt(data_dir, database_path, log)
|
||||
if not database_path.is_file():
|
||||
return StartupMigrationResult(state="not_required")
|
||||
|
||||
@@ -553,7 +648,6 @@ def run_startup_migration(
|
||||
return open_client(profile)
|
||||
return runtime_sdk_client(database_path, data_dir, profile)
|
||||
|
||||
log = _StepLog(data_dir)
|
||||
store = SdkDataPlaneMigrationStore(database_path)
|
||||
with log.step("preflight.authority_read") as step:
|
||||
authority, authority_migration = store.authority_state()
|
||||
@@ -613,6 +707,12 @@ def run_startup_migration(
|
||||
from one_memory.database_access import close_main_databases
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
|
||||
from one_memory.memory_server.migrations import MIGRATIONS
|
||||
from one_memory.sdk_migration_restore import (
|
||||
clear_restore_point,
|
||||
discard_failed_state,
|
||||
prune_restored_bundles,
|
||||
record_restore_point,
|
||||
)
|
||||
|
||||
with log.step("preflight.resumable_identity") as step:
|
||||
migration_id, bundle_id = _resumable_identity(database_path)
|
||||
@@ -630,66 +730,105 @@ def run_startup_migration(
|
||||
raise DataPlaneMigrationError("sdk_migration_model_profile_required")
|
||||
log.check("preflight.model_profile_check", None)
|
||||
with capture_fence(data_dir):
|
||||
progress("bundle")
|
||||
with log.step("bundle.ensure") as step:
|
||||
manifest = _ensure_bundle(
|
||||
data_dir=data_dir,
|
||||
bundle_id=bundle_id,
|
||||
runtime_build=runtime_build,
|
||||
)
|
||||
step["bundle_id"] = bundle_id
|
||||
progress("prepare")
|
||||
with log.step("prepare.schema_migrate") as step:
|
||||
asyncio.run(migrate(SqliteMigrationStore(database_path), MIGRATIONS))
|
||||
close_main_databases()
|
||||
step["schema_version"] = MIGRATIONS[-1].version
|
||||
# Builds through 19 left a staged SDK copy beside the original store.
|
||||
# Settling it before the bundle means the bundle holds the original
|
||||
# store, and a restore never brings the staged copy back.
|
||||
with log.step("prepare.recover_abandoned_staging"):
|
||||
recover_abandoned_staging(
|
||||
data_dir=data_dir,
|
||||
database_path=database_path,
|
||||
namespace=namespace,
|
||||
)
|
||||
with log.step("prepare.ledger_start"):
|
||||
store.start(
|
||||
migration_id=migration_id,
|
||||
source_runtime_build=str(manifest.get("runtime_build") or ""),
|
||||
target_sdk_version=sdk_version,
|
||||
source_schema_version=MIGRATIONS[-1].version,
|
||||
namespace_fingerprint=hashlib.sha256(namespace.encode()).hexdigest(),
|
||||
backend_fingerprint=hashlib.sha256(b"chroma:kuzu:sqlite").hexdigest(),
|
||||
backup_bundle_id=bundle_id,
|
||||
progress("bundle")
|
||||
with log.step("bundle.ensure") as step:
|
||||
manifest, created = _ensure_bundle(
|
||||
data_dir=data_dir,
|
||||
bundle_id=bundle_id,
|
||||
runtime_build=runtime_build,
|
||||
)
|
||||
with log.step("prepare.legacy_work") as step:
|
||||
preparation = prepare_legacy_work_for_cutover(database_path, step)
|
||||
step.update(preparation.as_document())
|
||||
reattached = ()
|
||||
readback = None
|
||||
if profile is not None:
|
||||
progress("vectors")
|
||||
with log.step("vectors.reattach") as step:
|
||||
reattached = reattach_legacy_vectors(data_dir, profile)
|
||||
step["collections"] = len(reattached)
|
||||
if not legacy.empty:
|
||||
assert profile is not None
|
||||
progress("verify")
|
||||
with log.step("verify.readback") as step:
|
||||
with client_for(profile) as client:
|
||||
readback = verify_readback(client, legacy)
|
||||
step.update(readback.as_document())
|
||||
if not readback.passed:
|
||||
store.set_phase(migration_id, phase="readback_failed", state="blocked")
|
||||
raise MemoryReadbackFailed(readback)
|
||||
with log.step("prepare.mark_prepared"):
|
||||
store.set_phase(migration_id, phase=PREPARED_PHASE, state="ready")
|
||||
progress("cutover")
|
||||
with log.step("cutover.commit"):
|
||||
commit_sdk_cutover(
|
||||
step["bundle_id"] = bundle_id
|
||||
step["created"] = created
|
||||
with log.step("bundle.restore_point") as step:
|
||||
restore_bundle = _restore_point_bundle(
|
||||
data_dir=data_dir,
|
||||
migration_id=migration_id,
|
||||
bundle_id=bundle_id,
|
||||
created=created,
|
||||
runtime_build=runtime_build,
|
||||
)
|
||||
record_restore_point(data_dir, migration_id=migration_id, bundle_id=restore_bundle)
|
||||
step["bundle_id"] = restore_bundle
|
||||
step["pruned"] = prune_restored_bundles(data_dir, keep={bundle_id, restore_bundle})
|
||||
# From here until SDK authority is committed, a failure puts the
|
||||
# restore point back. Only `cutover.commit` can fail after its own
|
||||
# commit; the database decides then.
|
||||
cutover_started = False
|
||||
try:
|
||||
progress("prepare")
|
||||
with log.step("prepare.schema_migrate") as step:
|
||||
asyncio.run(migrate(SqliteMigrationStore(database_path), MIGRATIONS))
|
||||
close_main_databases()
|
||||
step["schema_version"] = MIGRATIONS[-1].version
|
||||
with log.step("prepare.ledger_start"):
|
||||
store.start(
|
||||
migration_id=migration_id,
|
||||
source_runtime_build=str(manifest.get("runtime_build") or ""),
|
||||
target_sdk_version=sdk_version,
|
||||
source_schema_version=MIGRATIONS[-1].version,
|
||||
namespace_fingerprint=hashlib.sha256(namespace.encode()).hexdigest(),
|
||||
backend_fingerprint=hashlib.sha256(b"chroma:kuzu:sqlite").hexdigest(),
|
||||
backup_bundle_id=bundle_id,
|
||||
)
|
||||
with log.step("prepare.legacy_work") as step:
|
||||
preparation = prepare_legacy_work_for_cutover(database_path, step)
|
||||
step.update(preparation.as_document())
|
||||
reattached = ()
|
||||
readback = None
|
||||
if profile is not None:
|
||||
progress("vectors")
|
||||
with log.step("vectors.reattach") as step:
|
||||
reattached = reattach_legacy_vectors(data_dir, profile)
|
||||
step["collections"] = len(reattached)
|
||||
if not legacy.empty:
|
||||
assert profile is not None
|
||||
progress("verify")
|
||||
with log.step("verify.readback") as step:
|
||||
with client_for(profile) as client:
|
||||
readback = verify_readback(client, legacy)
|
||||
step.update(readback.as_document())
|
||||
if not readback.passed:
|
||||
store.set_phase(migration_id, phase="readback_failed", state="blocked")
|
||||
raise MemoryReadbackFailed(readback)
|
||||
with log.step("prepare.mark_prepared"):
|
||||
store.set_phase(migration_id, phase=PREPARED_PHASE, state="ready")
|
||||
progress("cutover")
|
||||
cutover_started = True
|
||||
with log.step("cutover.commit"):
|
||||
commit_sdk_cutover(
|
||||
data_dir=data_dir,
|
||||
database_path=database_path,
|
||||
migration_id=migration_id,
|
||||
bundle_id=bundle_id,
|
||||
sdk_version=sdk_version,
|
||||
)
|
||||
except Exception:
|
||||
_restore_after_failure(
|
||||
data_dir=data_dir,
|
||||
database_path=database_path,
|
||||
migration_id=migration_id,
|
||||
bundle_id=bundle_id,
|
||||
sdk_version=sdk_version,
|
||||
bundle_id=restore_bundle,
|
||||
cutover_started=cutover_started,
|
||||
report=restore_report,
|
||||
)
|
||||
raise
|
||||
# SDK authority is committed: from here on the home is never restored.
|
||||
clear_restore_point(data_dir)
|
||||
if restore_bundle != bundle_id:
|
||||
shutil.rmtree(
|
||||
data_dir.resolve() / "backups" / f"{restore_bundle}.sdk-bundle",
|
||||
ignore_errors=True,
|
||||
)
|
||||
discard_failed_state(data_dir)
|
||||
progress("cleanup")
|
||||
with log.step("cleanup.drop_legacy_tables") as step:
|
||||
dropped = resume_legacy_cleanup(data_dir=data_dir, database_path=database_path)
|
||||
@@ -706,3 +845,52 @@ def run_startup_migration(
|
||||
reattached=reattached,
|
||||
readback=readback,
|
||||
)
|
||||
|
||||
|
||||
def _restore_after_failure(
|
||||
*,
|
||||
data_dir: Path,
|
||||
database_path: Path,
|
||||
migration_id: str,
|
||||
bundle_id: str,
|
||||
cutover_started: bool,
|
||||
report: dict[str, object] | None,
|
||||
) -> None:
|
||||
"""Put the restore point back after a pre-cutover failure. Never raises.
|
||||
|
||||
The caller re-raises the migration's own error: the restore is secondary,
|
||||
so its outcome only goes to the log and to `report`.
|
||||
"""
|
||||
from one_memory.sdk_migration_log import write_migration_log
|
||||
from one_memory.sdk_migration_restore import restore_pre_migration_state
|
||||
|
||||
report = {} if report is None else report
|
||||
if cutover_started:
|
||||
try:
|
||||
committed = SdkDataPlaneMigrationStore(database_path).authority() is MemoryAuthority.SDK
|
||||
except Exception: # noqa: BLE001 - never restore over an unknown authority
|
||||
committed = True
|
||||
if committed:
|
||||
write_migration_log(
|
||||
data_dir,
|
||||
writer="core-sdk-migrate",
|
||||
event="restore.skipped",
|
||||
outcome="succeeded",
|
||||
migration_id=migration_id,
|
||||
details={"reason": "authority_committed_or_unknown"},
|
||||
)
|
||||
report["state"] = "skipped"
|
||||
return
|
||||
try:
|
||||
restore_pre_migration_state(
|
||||
data_dir,
|
||||
bundle_id=bundle_id,
|
||||
migration_id=migration_id,
|
||||
reason="migration_failed",
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - the migration's own failure is raised
|
||||
code = getattr(error, "code", None)
|
||||
report["state"] = "failed"
|
||||
report["code"] = code if isinstance(code, str) else "sdk_migration_restore_failed"
|
||||
return
|
||||
report["state"] = "restored"
|
||||
@@ -249,6 +249,7 @@
|
||||
| `test_sdk_memory_qa.py` | `contract` | 覆盖 sdk ask propagates sdk search failure without legacy fallback、sdk ask revalidates memory before returning extract、sdk ask uses native sdk memory without governance fields 场景。 |
|
||||
| `test_sdk_migration_bundle.py` | `contract` | SDK-authority cutover requires a verified management + SDK data-plane bundle. |
|
||||
| `test_sdk_migration_log.py` | `integration` | `sdk-migration.log` explains an unfinished upgrade, and a legacy home is never served. |
|
||||
| `test_sdk_migration_restore.py` | `contract` | A failed upgrade migration puts the home back exactly as it found it. |
|
||||
| `test_sdk_verbatim_fallback.py` | `contract` | D98: the SDK's verbatim fallback must stay off. |
|
||||
| `test_search_without_profile.py` | `contract` | What 'semantic_search' should do once the profile that filled the store is gone. |
|
||||
| `test_secondary_source_adapters.py` | `contract` | 覆盖 pi and qwenwork keep tool and sidechain content out of the persons words、qwen code reads only the transcript beside its bookkeeping、qwen native parts preserve visible text not reasoning or tools 等 22 个场景。 |
|
||||
@@ -306,4 +307,4 @@
|
||||
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
|
||||
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
|
||||
|
||||
共 301 个测试文件。
|
||||
共 302 个测试文件。
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
|
||||
EXPECTED_TEST_FILE_COUNT = 301
|
||||
EXPECTED_TEST_FILES_SHA256 = "1e5f3a85f124a26ed67861e07fba7c670512a82139db64c1b7ba767ca37ed563"
|
||||
EXPECTED_TEST_FILE_COUNT = 302
|
||||
EXPECTED_TEST_FILES_SHA256 = "800de24f074be47f27f36b32f744df8def491e7c1e0de0d1eddfb736306daf4f"
|
||||
|
||||
# Pure transformations, validation models, parity checks and state machines.
|
||||
# This list is intentionally small: uncertain files default to contract.
|
||||
|
||||
@@ -200,6 +200,7 @@ def test_sdk_migrate_failure_is_one_named_code(tmp_path, monkeypatch) -> None:
|
||||
"schema_version": 1,
|
||||
"state": "failed",
|
||||
"code": "sdk_migration_disk_space_insufficient",
|
||||
"data_restored": False,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -301,16 +301,20 @@ def test_the_runner_names_the_step_and_sub_step_that_failed(tmp_path, monkeypatc
|
||||
for line in written
|
||||
if "step" in line.get("details", {})
|
||||
]
|
||||
assert steps[:6] == [
|
||||
assert steps[:8] == [
|
||||
("step.finished", "preflight.authority_read"),
|
||||
("step.finished", "preflight.profile_read"),
|
||||
("step.finished", "preflight.resumable_identity"),
|
||||
("step.finished", "preflight.legacy_ids_scan"),
|
||||
("step.finished", "preflight.model_profile_check"),
|
||||
("step.finished", "prepare.recover_abandoned_staging"),
|
||||
("step.finished", "bundle.ensure"),
|
||||
("step.finished", "bundle.restore_point"),
|
||||
]
|
||||
assert steps[-1] == ("step.failed", "prepare.legacy_work")
|
||||
failed = written[-1]
|
||||
# The failure put the home back; the restore says so after the failed step.
|
||||
assert [line["event"] for line in written[-2:]] == ["restore.started", "restore.finished"]
|
||||
failed = next(line for line in reversed(written) if line["event"] == "step.failed")
|
||||
assert failed["migration_id"].startswith("sdk-authority-v2-")
|
||||
assert failed["details"]["sub_step"] == "write_queue"
|
||||
assert failed["error"]["chain"][0]["type"] == "ValueError"
|
||||
|
||||
@@ -0,0 +1,483 @@
|
||||
"""A failed upgrade migration puts the home back exactly as it found it.
|
||||
|
||||
The fixed release that follows must migrate from the original 1.0.4 data, not
|
||||
from a half-migrated home: every pre-cutover failure restores the database,
|
||||
the SDK store and `config.json`, and a later run succeeds from there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import closing, contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory import cli
|
||||
from one_memory.config import Settings
|
||||
from one_memory.database_access import close_main_databases, main_database
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.live_capture import sdk_migration_capture_fenced
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
|
||||
from one_memory.memory_server.migrations import MIGRATIONS
|
||||
from one_memory.runtime_client import RuntimeStatus
|
||||
from one_memory.sdk_data_plane_migration import (
|
||||
DataPlaneMigrationError,
|
||||
MemoryAuthority,
|
||||
SdkDataPlaneMigrationStore,
|
||||
)
|
||||
from one_memory.sdk_migration_log import log_path
|
||||
from one_memory.sdk_migration_restore import (
|
||||
RESTORE_POINT,
|
||||
MigrationRestoreFailed,
|
||||
failed_state_path,
|
||||
finish_interrupted_restore,
|
||||
restore_pending,
|
||||
)
|
||||
from one_memory.sdk_migration_runner import run_startup_migration
|
||||
from one_memory.sdk_migration_verify import MemoryReadbackFailed
|
||||
from one_memory.target_schema import legacy_tables_present
|
||||
|
||||
NAMESPACE = "governed-v2"
|
||||
CONFIG = {"schema_version": 1, "sources": {"cursor": {"sync_enabled": True}}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def profile(monkeypatch):
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
|
||||
snapshot = builtin_retrieval_snapshot()
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: snapshot)
|
||||
monkeypatch.setattr("one_memory.sdk_migration_verify.read_active_profile", lambda _db: snapshot)
|
||||
return snapshot
|
||||
|
||||
|
||||
def _chroma(home: Path):
|
||||
from one_memory.sdk_vector_reattach import open_offline_chroma
|
||||
|
||||
return open_offline_chroma(home / "hy-memory" / NAMESPACE / "vector")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def home(tmp_path, profile) -> Path:
|
||||
"""A 1.0.4-shaped home: legacy memories, queued legacy work, legacy vectors."""
|
||||
from one_memory.hy_sdk_runtime import vector_collection_bases
|
||||
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
asyncio.run(migrate(SqliteMigrationStore(database), MIGRATIONS))
|
||||
with main_database(database).write() as connection, connection:
|
||||
for memory_id in ("m1", "m2"):
|
||||
connection.execute(
|
||||
"INSERT INTO governed_memories (memory_id,user_id,kind,status,content,"
|
||||
"scopes_json,revision,created_at,updated_at) "
|
||||
"VALUES (?,'local','chat','confirmed','x','[]',1,'now','now')",
|
||||
(memory_id,),
|
||||
)
|
||||
verbatim = "text of e1"
|
||||
connection.execute(
|
||||
"INSERT INTO evidence (evidence_id,idempotency_key,user_id,source_id,"
|
||||
"conversation_id,content_kind,role,written_by,verbatim,content_hash,"
|
||||
"revision,imported_at,updated_at,ingest_seq) "
|
||||
"VALUES ('e1','e1','local','cursor','session-1','raw_conversation','user',"
|
||||
"'person',?,?,1,'2026-10-22T00:00:00+00:00','2026-10-22T00:00:00+00:00',1)",
|
||||
(verbatim, hashlib.sha256(verbatim.encode()).hexdigest()),
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_tasks "
|
||||
"(task_id,request_id,user_id,kind,input_ref,status,profile_id,"
|
||||
"profile_revision,profile_snapshot_json,runner_token,attempts,"
|
||||
"last_error_code,created_at,updated_at,duration_seconds,hy_invocation_count) "
|
||||
"VALUES ('t1','request-t1','local','extract','e1','pending','profile',1,'{}',"
|
||||
"NULL,0,NULL,'now','now',NULL,0)"
|
||||
)
|
||||
close_main_databases()
|
||||
assert EvidenceStore(database).get("e1") is not None
|
||||
close_main_databases()
|
||||
current, legacy = vector_collection_bases(profile)
|
||||
client = _chroma(tmp_path)
|
||||
for name, ids in ((f"{legacy[0]}_4", ["m1", "m2"]), (f"{current}_4", ["after"])):
|
||||
collection = client.get_or_create_collection(name, metadata={"hnsw:space": "cosine"})
|
||||
collection.upsert(
|
||||
ids=ids,
|
||||
embeddings=[[float(index + 1)] * 4 for index, _ in enumerate(ids)],
|
||||
documents=[f"doc {identifier}" for identifier in ids],
|
||||
)
|
||||
client.close()
|
||||
(tmp_path / "config.json").write_text(json.dumps(CONFIG), encoding="utf-8")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _collections(home: Path) -> dict[str, int]:
|
||||
client = _chroma(home)
|
||||
try:
|
||||
return {item.name: item.count() for item in client.list_collections()}
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _state(home: Path) -> dict[str, object]:
|
||||
"""Everything a migration may change, compared byte for byte or row for row."""
|
||||
close_main_databases()
|
||||
with closing(sqlite3.connect(home / "one-memory.sqlite3")) as connection:
|
||||
dump = list(connection.iterdump())
|
||||
sdk = home / "hy-memory" / NAMESPACE
|
||||
files = {
|
||||
path.relative_to(sdk).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in sorted(sdk.rglob("*"))
|
||||
if path.is_file()
|
||||
}
|
||||
from one_memory.sdk_vector_reattach import _stored_names
|
||||
|
||||
return {
|
||||
"database": dump,
|
||||
"sdk_files": files,
|
||||
"collection_names": _stored_names(home),
|
||||
"config": (home / "config.json").read_bytes(),
|
||||
}
|
||||
|
||||
|
||||
def _reading(readable: set[str]):
|
||||
class Client:
|
||||
def list_memories(self, *, user_id, statuses, limit, offset, order):
|
||||
del user_id, statuses, order
|
||||
ids = sorted(readable)
|
||||
rows = [{"memory_id": item} for item in ids[offset : offset + limit]]
|
||||
return {"vdb": {"memories": rows, "total": len(ids)}}
|
||||
|
||||
def get(self, memory_id):
|
||||
return {"memory_id": memory_id} if memory_id in readable else None
|
||||
|
||||
@contextmanager
|
||||
def open_client(_profile):
|
||||
yield Client()
|
||||
|
||||
return open_client
|
||||
|
||||
|
||||
def _run(home: Path, *, readable=frozenset({"m1", "m2"}), report=None):
|
||||
return run_startup_migration(
|
||||
data_dir=home,
|
||||
runtime_build="22",
|
||||
sdk_version="2.0.5",
|
||||
namespace=NAMESPACE,
|
||||
open_client=_reading(set(readable)),
|
||||
restore_report=report,
|
||||
)
|
||||
|
||||
|
||||
def _events(home: Path) -> list[dict[str, object]]:
|
||||
return [json.loads(line) for line in log_path(home).read_text().splitlines()]
|
||||
|
||||
|
||||
def _fail_schema_migrate(monkeypatch, home):
|
||||
import one_memory.memory_server.migration_runner as runner_module
|
||||
|
||||
async def half_migrated(store, migrations):
|
||||
del store, migrations
|
||||
with main_database(home / "one-memory.sqlite3").write() as connection, connection:
|
||||
connection.execute("CREATE TABLE half_applied_migration (x INTEGER)")
|
||||
raise DataPlaneMigrationError("sdk_migration_schema_failed")
|
||||
|
||||
monkeypatch.setattr(runner_module, "migrate", half_migrated)
|
||||
|
||||
|
||||
def _fail_legacy_work(monkeypatch, home):
|
||||
import one_memory.sdk_migration_runner as runner
|
||||
|
||||
def broken(_database_path):
|
||||
raise DataPlaneMigrationError("sdk_data_plane_dispatch_failed")
|
||||
|
||||
monkeypatch.setattr(runner, "move_pending_dispatches", broken)
|
||||
|
||||
|
||||
def _fail_vectors(monkeypatch, home):
|
||||
import one_memory.sdk_migration_runner as runner
|
||||
|
||||
real = runner.reattach_legacy_vectors
|
||||
|
||||
def renamed_then_failed(data_dir, profile):
|
||||
real(data_dir, profile)
|
||||
raise DataPlaneMigrationError("sdk_migration_vector_reattach_incomplete")
|
||||
|
||||
monkeypatch.setattr(runner, "reattach_legacy_vectors", renamed_then_failed)
|
||||
|
||||
|
||||
FAILURES = {
|
||||
"prepare.schema_migrate": (_fail_schema_migrate, "sdk_migration_schema_failed"),
|
||||
"prepare.legacy_work": (_fail_legacy_work, "sdk_data_plane_dispatch_failed"),
|
||||
"vectors.reattach": (_fail_vectors, "sdk_migration_vector_reattach_incomplete"),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("step", sorted(FAILURES))
|
||||
def test_a_failure_before_cutover_restores_the_home_exactly(home, monkeypatch, step):
|
||||
counts = _collections(home)
|
||||
before = _state(home)
|
||||
inject, code = FAILURES[step]
|
||||
inject(monkeypatch, home)
|
||||
report: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError) as caught:
|
||||
_run(home, report=report)
|
||||
|
||||
# The migration's own failure is what the caller sees.
|
||||
assert caught.value.code == code
|
||||
assert report == {"state": "restored"}
|
||||
assert _state(home) == before
|
||||
assert _collections(home) == counts
|
||||
store = SdkDataPlaneMigrationStore(home / "one-memory.sqlite3")
|
||||
assert store.authority() is MemoryAuthority.LEGACY
|
||||
# Nothing left to resume: the next run starts from the original data.
|
||||
assert not (home / RESTORE_POINT).exists()
|
||||
assert not restore_pending(home)
|
||||
# The failed state is kept aside for diagnosis; capture stays frozen.
|
||||
assert (failed_state_path(home) / "management" / "one-memory.sqlite3").is_file()
|
||||
assert sdk_migration_capture_fenced(home)
|
||||
events = _events(home)
|
||||
failed_at = next(
|
||||
index
|
||||
for index, line in enumerate(events)
|
||||
if line["event"] == "step.failed" and line["details"]["step"] == step
|
||||
)
|
||||
finished = next(line for line in events if line["event"] == "restore.finished")
|
||||
assert events.index(finished) > failed_at
|
||||
assert finished["details"]["files"] > 0 and finished["details"]["bytes"] > 0
|
||||
assert finished["details"]["reason"] == "migration_failed"
|
||||
assert "duration_ms" in finished["details"]
|
||||
|
||||
monkeypatch.undo()
|
||||
_retry_succeeds(home, monkeypatch)
|
||||
|
||||
|
||||
def test_a_readback_failure_restores_and_a_fixed_build_succeeds(home, monkeypatch):
|
||||
before = _state(home)
|
||||
report: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(MemoryReadbackFailed):
|
||||
_run(home, readable={"m1"}, report=report)
|
||||
|
||||
assert report == {"state": "restored"}
|
||||
assert _state(home) == before
|
||||
# The ledger row the failure wrote went with the rest.
|
||||
with closing(sqlite3.connect(home / "one-memory.sqlite3")) as connection:
|
||||
assert not connection.execute(
|
||||
"SELECT 1 FROM sdk_data_plane_migrations WHERE state='blocked'"
|
||||
).fetchall()
|
||||
_retry_succeeds(home, monkeypatch)
|
||||
|
||||
|
||||
def _retry_succeeds(home: Path, monkeypatch) -> None:
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot, vector_collection_bases
|
||||
|
||||
snapshot = builtin_retrieval_snapshot()
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: snapshot)
|
||||
restored_bundles = sorted((home / "backups").glob("*.sdk-bundle/restored.json"))
|
||||
assert restored_bundles
|
||||
|
||||
result = _run(home)
|
||||
|
||||
assert result.state == "cutover"
|
||||
store = SdkDataPlaneMigrationStore(home / "one-memory.sqlite3")
|
||||
assert store.authority() is MemoryAuthority.SDK
|
||||
current, _legacy = vector_collection_bases(snapshot)
|
||||
assert _collections(home) == {f"{current}_4": 3}
|
||||
assert not sdk_migration_capture_fenced(home)
|
||||
assert not (home / RESTORE_POINT).exists()
|
||||
assert not failed_state_path(home).exists()
|
||||
# The restored bundle was superseded by this run's verified one.
|
||||
assert not any(path.exists() for path in restored_bundles)
|
||||
assert len(list((home / "backups").glob("*-pre-cutover.sdk-bundle"))) == 1
|
||||
|
||||
|
||||
def test_a_failure_after_the_cutover_commit_is_not_restored(home, monkeypatch):
|
||||
import one_memory.sdk_migration_runner as runner
|
||||
|
||||
def broken_cleanup(**_options):
|
||||
raise DataPlaneMigrationError("sdk_data_plane_cleanup_failed")
|
||||
|
||||
monkeypatch.setattr(runner, "resume_legacy_cleanup", broken_cleanup)
|
||||
report: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError):
|
||||
_run(home, report=report)
|
||||
|
||||
assert report == {}
|
||||
store = SdkDataPlaneMigrationStore(home / "one-memory.sqlite3")
|
||||
assert store.authority() is MemoryAuthority.SDK
|
||||
assert legacy_tables_present(home / "one-memory.sqlite3")
|
||||
assert not (home / RESTORE_POINT).exists()
|
||||
assert not any(line["event"].startswith("restore.") for line in _events(home))
|
||||
|
||||
monkeypatch.undo()
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
|
||||
snapshot = builtin_retrieval_snapshot()
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: snapshot)
|
||||
assert _run(home).state == "complete"
|
||||
|
||||
|
||||
def test_a_commit_that_landed_before_its_step_failed_is_not_restored(home, monkeypatch):
|
||||
import one_memory.sdk_migration_runner as runner
|
||||
|
||||
real = runner.commit_sdk_cutover
|
||||
|
||||
def committed_then_failed(**options):
|
||||
real(**options)
|
||||
raise DataPlaneMigrationError("sdk_data_plane_cutover_missing")
|
||||
|
||||
monkeypatch.setattr(runner, "commit_sdk_cutover", committed_then_failed)
|
||||
report: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError):
|
||||
_run(home, report=report)
|
||||
|
||||
assert report == {"state": "skipped"}
|
||||
assert (
|
||||
SdkDataPlaneMigrationStore(home / "one-memory.sqlite3").authority() is MemoryAuthority.SDK
|
||||
)
|
||||
skipped = [line for line in _events(home) if line["event"] == "restore.skipped"]
|
||||
assert len(skipped) == 1
|
||||
|
||||
|
||||
def test_a_failed_restore_is_logged_and_the_original_code_is_kept(home, monkeypatch):
|
||||
import one_memory.sdk_migration_runner as runner
|
||||
|
||||
def corrupt_bundle_then_fail(_database_path):
|
||||
bundle = next((home / "backups").glob("*-pre-cutover.sdk-bundle"))
|
||||
(bundle / "management" / "one-memory.sqlite3").write_bytes(b"not the backup")
|
||||
raise DataPlaneMigrationError("sdk_data_plane_dispatch_failed")
|
||||
|
||||
monkeypatch.setattr(runner, "move_pending_dispatches", corrupt_bundle_then_fail)
|
||||
monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=home))
|
||||
monkeypatch.setattr(
|
||||
cli,
|
||||
"inspect_runtime",
|
||||
lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
|
||||
)
|
||||
output = io.StringIO()
|
||||
|
||||
code = cli.run(["sdk-migrate", "--runtime-build", "22", "--yes", "--json"], output=output)
|
||||
|
||||
document = json.loads(output.getvalue().splitlines()[-1])
|
||||
assert code == 1
|
||||
assert document["code"] == "sdk_data_plane_dispatch_failed"
|
||||
assert document["data_restored"] is False
|
||||
assert document["restore_code"] == "sdk_migration_restore_failed"
|
||||
events = _events(home)
|
||||
restore_failed = next(line for line in events if line["event"] == "restore.failed")
|
||||
assert restore_failed["code"] == "sdk_migration_restore_failed"
|
||||
assert restore_failed["error"]["chain"]
|
||||
migrate_failed = next(line for line in events if line["event"] == "migrate.failed")
|
||||
assert migrate_failed["code"] == "sdk_data_plane_dispatch_failed"
|
||||
assert migrate_failed["details"]["restore"] == "failed"
|
||||
# Nothing was swapped; the restore point stays and the next run refuses
|
||||
# to migrate over the half-migrated home until a restore succeeds.
|
||||
assert (home / RESTORE_POINT).exists()
|
||||
assert not restore_pending(home)
|
||||
monkeypatch.undo()
|
||||
with pytest.raises(MigrationRestoreFailed):
|
||||
_run(home)
|
||||
|
||||
|
||||
def test_a_restore_killed_between_renames_is_finished_before_anything_reads(home, monkeypatch):
|
||||
import one_memory.sdk_migration_restore as restore_module
|
||||
|
||||
before = _state(home)
|
||||
real_apply = restore_module._apply
|
||||
|
||||
def killed_part_way(target, steps, done):
|
||||
real_apply(target, steps[:5], done)
|
||||
raise KeyboardInterrupt
|
||||
|
||||
_fail_vectors(monkeypatch, home)
|
||||
monkeypatch.setattr(restore_module, "_apply", killed_part_way)
|
||||
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
_run(home)
|
||||
|
||||
assert restore_pending(home)
|
||||
# The database was renamed aside: Core must not serve, and the probe
|
||||
# must not call this a fresh home.
|
||||
assert not (home / "one-memory.sqlite3").exists()
|
||||
monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=home))
|
||||
status = io.StringIO()
|
||||
assert cli.run(["sdk-migration-status", "--json"], output=status) == 0
|
||||
assert json.loads(status.getvalue())["state"] == "required"
|
||||
with pytest.raises(SystemExit) as refused:
|
||||
cli._refuse_unmigrated_home(home)
|
||||
assert refused.value.code == cli.SDK_MIGRATION_REQUIRED_EXIT
|
||||
|
||||
monkeypatch.undo()
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
|
||||
snapshot = builtin_retrieval_snapshot()
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: snapshot)
|
||||
|
||||
assert finish_interrupted_restore(home)
|
||||
|
||||
assert not restore_pending(home)
|
||||
assert _state(home) == before
|
||||
|
||||
|
||||
def test_a_run_killed_after_its_restore_point_is_rolled_back_on_the_next_start(home, monkeypatch):
|
||||
import one_memory.sdk_migration_runner as runner
|
||||
|
||||
before = _state(home)
|
||||
real = runner.reattach_legacy_vectors
|
||||
|
||||
def renamed_then_killed(data_dir, profile):
|
||||
real(data_dir, profile)
|
||||
raise KeyboardInterrupt
|
||||
|
||||
monkeypatch.setattr(runner, "reattach_legacy_vectors", renamed_then_killed)
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
_run(home)
|
||||
assert (home / RESTORE_POINT).exists()
|
||||
monkeypatch.undo()
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
|
||||
snapshot = builtin_retrieval_snapshot()
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: snapshot)
|
||||
|
||||
# The next start first puts the home back, then migrates it from there.
|
||||
def failing_readback(_profile):
|
||||
raise DataPlaneMigrationError("sdk_migration_secure_storage_unavailable")
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError):
|
||||
run_startup_migration(
|
||||
data_dir=home,
|
||||
runtime_build="22",
|
||||
sdk_version="2.0.5",
|
||||
namespace=NAMESPACE,
|
||||
open_client=failing_readback,
|
||||
)
|
||||
reasons = [
|
||||
line["details"]["reason"] for line in _events(home) if line["event"] == "restore.finished"
|
||||
]
|
||||
assert reasons == ["interrupted_attempt", "migration_failed"]
|
||||
assert _state(home) == before
|
||||
|
||||
|
||||
def test_a_preflight_failure_changes_nothing_and_needs_no_restore(home, monkeypatch):
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: None)
|
||||
before = _state(home)
|
||||
report: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError, match="sdk_migration_model_profile_required"):
|
||||
_run(home, report=report)
|
||||
|
||||
assert report == {}
|
||||
assert _state(home) == before
|
||||
assert not (home / "backups").exists() or not list((home / "backups").iterdir())
|
||||
|
||||
|
||||
def test_a_frozen_capture_fence_holds_until_a_migration_lifts_it(tmp_path):
|
||||
marker = tmp_path / ".sdk-migration-capture-fence.json"
|
||||
marker.write_text(json.dumps({"schema_version": 1, "frozen": True}), encoding="utf-8")
|
||||
|
||||
assert sdk_migration_capture_fenced(tmp_path)
|
||||
assert marker.exists()
|
||||
@@ -25,11 +25,14 @@ import {
|
||||
installUserService,
|
||||
RecallBrokerSupervisor,
|
||||
RuntimeSupervisor,
|
||||
readInstalledRuntimeSource,
|
||||
readPublishedCorePort,
|
||||
readRuntimeServiceOwner,
|
||||
readSdkMigrationBlock,
|
||||
removeRuntimeServiceOwner,
|
||||
runtimeInstallLayout,
|
||||
runtimeServiceEnableMarkerPath,
|
||||
type SdkMigrationSource,
|
||||
stopRuntimeProcess,
|
||||
waitForRuntimeProcess,
|
||||
writeRuntimeServiceOwner,
|
||||
@@ -1075,6 +1078,9 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
|
||||
);
|
||||
installPackagedWebProtocol();
|
||||
|
||||
// The Runtime this launch replaced, when it moved `current`: the migration
|
||||
// log (and so the issue a person files) names the version upgraded from.
|
||||
let upgradeSource: SdkMigrationSource | null = null;
|
||||
const migrateAtStartup = async (core: {
|
||||
command: string;
|
||||
args: string[];
|
||||
@@ -1084,6 +1090,7 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
|
||||
dataDir,
|
||||
core,
|
||||
caller: "desktop",
|
||||
sourceRuntime: upgradeSource,
|
||||
runtimeBuild:
|
||||
packagedBuildNumber === undefined
|
||||
? typeof app.getVersion === "function"
|
||||
@@ -1200,6 +1207,12 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
|
||||
}
|
||||
if (installed !== null) {
|
||||
runtimeChanged = !installed.reused_current;
|
||||
if (runtimeChanged) {
|
||||
upgradeSource = await readInstalledRuntimeSource(
|
||||
installRoot,
|
||||
installed.previous_current_target,
|
||||
);
|
||||
}
|
||||
await installBundledCliIfNeeded({
|
||||
home: app.getPath("home"),
|
||||
installRoot,
|
||||
@@ -1217,6 +1230,12 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
|
||||
command: stable?.captureCommand ?? layout.coreCommand,
|
||||
args: [],
|
||||
});
|
||||
} else if (desktopBackgroundLaunch) {
|
||||
// A login launch does not migrate, and the service exits on a home a
|
||||
// failed migration froze: say so now instead of waiting for a
|
||||
// Runtime that will not come. Opening MemHub retries the migration.
|
||||
const frozen = await readSdkMigrationBlock(dataDir);
|
||||
if (frozen?.frozen !== undefined) throw new Error(frozen.code);
|
||||
}
|
||||
const alreadyRunning = runtimeChanged
|
||||
? false
|
||||
@@ -1707,6 +1726,23 @@ if (desktopVersion !== null) {
|
||||
showItemInFolder: (target) => shell.showItemInFolder(target),
|
||||
openExternal: (url) => shell.openExternal(url),
|
||||
},
|
||||
environment: {
|
||||
appVersion:
|
||||
typeof app.getVersion === "function" ? app.getVersion() : null,
|
||||
buildNumber: packagedBuildNumber ?? null,
|
||||
osName:
|
||||
process.platform === "darwin"
|
||||
? "macOS"
|
||||
: process.platform === "win32"
|
||||
? "Windows"
|
||||
: "Linux",
|
||||
osVersion:
|
||||
typeof process.getSystemVersion === "function"
|
||||
? process.getSystemVersion()
|
||||
: null,
|
||||
arch: process.arch,
|
||||
home: app.getPath("home"),
|
||||
},
|
||||
})
|
||||
)
|
||||
return;
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
displayPath,
|
||||
MAX_ISSUE_URL_LENGTH,
|
||||
MIGRATION_ISSUE_URL,
|
||||
type MigrationDialogDependencies,
|
||||
migrationFailureCode,
|
||||
migrationIssueBody,
|
||||
migrationIssueUrl,
|
||||
migrationLogPath,
|
||||
readMigrationLogFacts,
|
||||
showMigrationFailureDialog,
|
||||
} from "./migration-failure-dialog";
|
||||
|
||||
@@ -57,8 +61,15 @@ describe("migration failure dialog", () => {
|
||||
expect(deps.showItemInFolder).toHaveBeenCalledWith(
|
||||
migrationLogPath(dataDir),
|
||||
);
|
||||
expect(deps.openExternal).toHaveBeenCalledWith(
|
||||
migrationIssueUrl("sdk_migration_preflight_failed"),
|
||||
const opened = new URL(
|
||||
String((deps.openExternal.mock.calls[0] as unknown[] | undefined)?.[0]),
|
||||
);
|
||||
expect(`${opened.origin}${opened.pathname}`).toBe(MIGRATION_ISSUE_URL);
|
||||
expect(opened.searchParams.get("title")).toBe(
|
||||
"[升级迁移失败 / migration failed] sdk_migration_preflight_failed",
|
||||
);
|
||||
expect(opened.searchParams.get("body")).toContain(
|
||||
"- 错误码:sdk_migration_preflight_failed",
|
||||
);
|
||||
expect(migrationIssueUrl("sdk_migration_preflight_failed")).toMatch(
|
||||
new RegExp(
|
||||
@@ -115,6 +126,170 @@ describe("migration failure dialog", () => {
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("fills the issue body from the migration log, never naming the user", async () => {
|
||||
const accountHome = await home();
|
||||
const dataDir = path.join(accountHome, ".one-memory");
|
||||
await mkdir(path.join(dataDir, "logs"), { recursive: true });
|
||||
const log = [
|
||||
{
|
||||
event: "migration.invoked",
|
||||
details: { source_version: "unknown", source_build: "unknown" },
|
||||
},
|
||||
{
|
||||
event: "migrate.invoked",
|
||||
details: { source_version: "1.0.4", source_build: "104" },
|
||||
},
|
||||
{
|
||||
event: "step.failed",
|
||||
code: "sdk_data_plane_dispatch_failed",
|
||||
details: {
|
||||
step: "prepare.legacy_work",
|
||||
sub_step: "pending_dispatches",
|
||||
},
|
||||
},
|
||||
{ event: "restore.started", details: {} },
|
||||
{ event: "restore.finished", details: { files: 9 } },
|
||||
{
|
||||
event: "migrate.failed",
|
||||
code: "sdk_data_plane_dispatch_failed",
|
||||
details: {},
|
||||
},
|
||||
]
|
||||
.map((line) => JSON.stringify(line))
|
||||
.join("\n");
|
||||
await writeFile(migrationLogPath(dataDir), `${log}\nnot json\n`);
|
||||
await writeFile(
|
||||
path.join(dataDir, "sdk-migration-blocked.json"),
|
||||
JSON.stringify({
|
||||
schema_version: 1,
|
||||
code: "sdk_data_plane_dispatch_failed",
|
||||
caller: "desktop",
|
||||
at: "2026-10-27T00:00:00.000Z",
|
||||
frozen: { service_was_enabled: true },
|
||||
data_restored: true,
|
||||
}),
|
||||
);
|
||||
const deps = dependencies(0);
|
||||
|
||||
expect(
|
||||
showMigrationFailureDialog({
|
||||
// Desktop's own migration throws the code the note records.
|
||||
error: new Error("sdk_data_plane_dispatch_failed"),
|
||||
dataDir,
|
||||
locale: "zh-CN",
|
||||
dependencies: deps,
|
||||
environment: {
|
||||
appVersion: "2.0.6",
|
||||
buildNumber: 206,
|
||||
osName: "macOS",
|
||||
osVersion: "15.6",
|
||||
arch: "arm64",
|
||||
home: accountHome,
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
expect(deps.showMessageBoxSync.mock.calls[0]?.[0]?.detail).toContain(
|
||||
"记忆数据已恢复到升级前的状态。",
|
||||
);
|
||||
expect(deps.showItemInFolder).toHaveBeenCalledWith(
|
||||
migrationLogPath(dataDir),
|
||||
);
|
||||
const url = String(
|
||||
(deps.openExternal.mock.calls[0] as unknown[] | undefined)?.[0],
|
||||
);
|
||||
expect(url.length).toBeLessThanOrEqual(MAX_ISSUE_URL_LENGTH);
|
||||
const body = new URL(url).searchParams.get("body");
|
||||
expect(body).toBe(
|
||||
[
|
||||
"## 升级迁移失败",
|
||||
"- 从 MemHub 1.0.4(build 104)升级到 2.0.6(build 206),迁移数据失败。",
|
||||
"- 错误码:sdk_data_plane_dispatch_failed",
|
||||
"- 失败步骤:prepare.legacy_work / pending_dispatches",
|
||||
"- 数据已恢复到升级前状态:是",
|
||||
"- 系统:macOS 15.6(arm64)",
|
||||
`- 迁移日志(本地):~${path.sep}${path.join(".one-memory", "logs", "sdk-migration.log")}`,
|
||||
"",
|
||||
"请把上面这个日志文件拖到这里作为附件,再提交。",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(body).not.toContain(accountHome);
|
||||
});
|
||||
|
||||
it("says 未知 for what the log does not know instead of guessing", () => {
|
||||
const body = migrationIssueBody({
|
||||
code: "sdk_migration_preflight_failed",
|
||||
fromVersion: "../../etc",
|
||||
logPath: "~/.one-memory/logs/sdk-migration.log",
|
||||
});
|
||||
expect(body).toContain(
|
||||
"- 从 MemHub 未知(build 未知)升级到 未知(build 未知)",
|
||||
);
|
||||
expect(body).toContain("- 失败步骤:未知");
|
||||
expect(body).toContain("- 数据已恢复到升级前状态:未知");
|
||||
expect(body).toContain("- 系统:未知");
|
||||
});
|
||||
|
||||
it("keeps the issue URL within GitHub's limit and encodes it", () => {
|
||||
const body = `${"迁移".repeat(4000)}\n& = # ?`;
|
||||
const url = migrationIssueUrl("sdk_migration_preflight_failed", body);
|
||||
expect(url.length).toBeLessThanOrEqual(MAX_ISSUE_URL_LENGTH);
|
||||
const parsed = new URL(url);
|
||||
expect(parsed.searchParams.get("title")).toContain(
|
||||
"sdk_migration_preflight_failed",
|
||||
);
|
||||
expect(body.startsWith(parsed.searchParams.get("body") ?? "x")).toBe(true);
|
||||
const small = migrationIssueUrl("sdk_migration_x", "a & b = c #d");
|
||||
expect(new URL(small).searchParams.get("body")).toBe("a & b = c #d");
|
||||
});
|
||||
|
||||
it("shows a custom data dir as it is and the default one under ~", () => {
|
||||
expect(
|
||||
displayPath("/Users/alice/.one-memory/logs/x.log", "/Users/alice"),
|
||||
).toBe(`~${path.sep}${path.join(".one-memory", "logs", "x.log")}`);
|
||||
expect(displayPath("/Volumes/data/memhub/logs/x.log", "/Users/alice")).toBe(
|
||||
"/Volumes/data/memhub/logs/x.log",
|
||||
);
|
||||
expect(displayPath("/Users/alice2/x.log", "/Users/alice")).toBe(
|
||||
"/Users/alice2/x.log",
|
||||
);
|
||||
});
|
||||
|
||||
it("reads only the latest attempt's step and restore from the log", async () => {
|
||||
const dataDir = await home();
|
||||
await mkdir(path.join(dataDir, "logs"), { recursive: true });
|
||||
await writeFile(
|
||||
migrationLogPath(dataDir),
|
||||
[
|
||||
{
|
||||
event: "migrate.invoked",
|
||||
details: { source_version: "1.0.4", source_build: 104 },
|
||||
},
|
||||
{ event: "step.failed", details: { step: "vectors.reattach" } },
|
||||
{ event: "restore.finished", details: {} },
|
||||
{
|
||||
event: "migrate.invoked",
|
||||
details: { source_version: "unknown", source_build: "unknown" },
|
||||
},
|
||||
{
|
||||
event: "step.failed",
|
||||
details: { step: "preflight.model_profile_check" },
|
||||
},
|
||||
]
|
||||
.map((line) => JSON.stringify(line))
|
||||
.join("\n"),
|
||||
);
|
||||
expect(readMigrationLogFacts(dataDir)).toEqual({
|
||||
fromVersion: "1.0.4",
|
||||
fromBuild: 104,
|
||||
step: "preflight.model_profile_check",
|
||||
subStep: null,
|
||||
code: null,
|
||||
dataRestored: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves other startup failures to the generic error box", async () => {
|
||||
const deps = dependencies(0);
|
||||
expect(
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import { appendFileSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import {
|
||||
appendFileSync,
|
||||
closeSync,
|
||||
fstatSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
readSync,
|
||||
} from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
/**
|
||||
@@ -7,30 +15,48 @@ import path from "node:path";
|
||||
* MemHub does not run on a home whose migration did not finish: one data shape
|
||||
* per release, no legacy fallback. The app therefore stops, and this dialog is
|
||||
* the only thing between the failure and a useful report — it names the code
|
||||
* and hands the person the one file support needs (`sdk-migration.log`).
|
||||
* and hands the person the one file support needs (`sdk-migration.log`), with
|
||||
* a GitHub issue already filled in from that log.
|
||||
*/
|
||||
|
||||
export const MIGRATION_ISSUE_URL =
|
||||
"https://github.com/Tencent-Hunyuan/Hy-Memory/issues/new";
|
||||
|
||||
const CODE = /^[a-z][a-z0-9_]{0,95}$/;
|
||||
const STEP = /^[a-z][a-z0-9_.]{0,79}$/;
|
||||
const VERSION = /^[0-9A-Za-z][0-9A-Za-z.+-]{0,63}$/;
|
||||
/** Generic startup codes that hide a migration block recorded by the service. */
|
||||
const MASKING_CODES = new Set([
|
||||
"desktop_start_failed",
|
||||
"runtime_service_not_ready",
|
||||
]);
|
||||
/** Only the tail of the log is read: the last attempt is at its end. */
|
||||
const LOG_TAIL_BYTES = 256 * 1024;
|
||||
/** GitHub rejects very long `issues/new` URLs; stay well below its limit. */
|
||||
export const MAX_ISSUE_URL_LENGTH = 6000;
|
||||
|
||||
export function migrationLogPath(dataDir: string): string {
|
||||
return path.join(dataDir, "logs", "sdk-migration.log");
|
||||
}
|
||||
|
||||
function readBlockCode(dataDir: string): string | null {
|
||||
interface BlockNote {
|
||||
code: string;
|
||||
dataRestored: boolean | null;
|
||||
}
|
||||
|
||||
function readBlock(dataDir: string): BlockNote | null {
|
||||
try {
|
||||
const value = JSON.parse(
|
||||
readFileSync(path.join(dataDir, "sdk-migration-blocked.json"), "utf8"),
|
||||
) as { code?: unknown };
|
||||
) as { code?: unknown; data_restored?: unknown };
|
||||
return typeof value.code === "string" && CODE.test(value.code)
|
||||
? value.code
|
||||
? {
|
||||
code: value.code,
|
||||
dataRestored:
|
||||
typeof value.data_restored === "boolean"
|
||||
? value.data_restored
|
||||
: null,
|
||||
}
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
@@ -53,8 +79,11 @@ export function migrationFailureCode(
|
||||
return message;
|
||||
}
|
||||
if (dataDir === null) return null;
|
||||
const blocked = readBlockCode(dataDir);
|
||||
const blocked = readBlock(dataDir)?.code ?? null;
|
||||
if (blocked === null) return null;
|
||||
// Desktop's own migration threw the code the note records (for example
|
||||
// `sdk_data_plane_write_job_input_missing`).
|
||||
if (message === blocked) return blocked;
|
||||
return MASKING_CODES.has(message) ||
|
||||
!CODE.test(message) ||
|
||||
message.startsWith("core_")
|
||||
@@ -62,9 +91,201 @@ export function migrationFailureCode(
|
||||
: null;
|
||||
}
|
||||
|
||||
export function migrationIssueUrl(code: string): string {
|
||||
/** What the issue says about the failed upgrade; every field is optional. */
|
||||
export interface MigrationIssueFacts {
|
||||
code: string;
|
||||
fromVersion?: string | null;
|
||||
fromBuild?: number | null;
|
||||
toVersion?: string | null;
|
||||
toBuild?: number | null;
|
||||
step?: string | null;
|
||||
subStep?: string | null;
|
||||
dataRestored?: boolean | null;
|
||||
osName?: string | null;
|
||||
osVersion?: string | null;
|
||||
arch?: string | null;
|
||||
/** The local log path, already shown with the home directory as `~`. */
|
||||
logPath: string;
|
||||
}
|
||||
|
||||
const UNKNOWN = "未知";
|
||||
|
||||
function versionText(value: string | null | undefined): string {
|
||||
return typeof value === "string" && VERSION.test(value) ? value : UNKNOWN;
|
||||
}
|
||||
|
||||
function buildText(value: number | null | undefined): string {
|
||||
return typeof value === "number" && Number.isSafeInteger(value) && value > 0
|
||||
? String(value)
|
||||
: UNKNOWN;
|
||||
}
|
||||
|
||||
function stepText(
|
||||
step: string | null | undefined,
|
||||
sub: string | null | undefined,
|
||||
) {
|
||||
if (typeof step !== "string" || !STEP.test(step)) return UNKNOWN;
|
||||
return typeof sub === "string" && STEP.test(sub) ? `${step} / ${sub}` : step;
|
||||
}
|
||||
|
||||
/** The issue body: short, content-free, and honest about what is unknown. */
|
||||
export function migrationIssueBody(facts: MigrationIssueFacts): string {
|
||||
const restored =
|
||||
facts.dataRestored === true
|
||||
? "是"
|
||||
: facts.dataRestored === false
|
||||
? "否"
|
||||
: UNKNOWN;
|
||||
const system = [facts.osName, facts.osVersion]
|
||||
.map((part) => versionText(part))
|
||||
.filter((part) => part !== UNKNOWN)
|
||||
.join(" ");
|
||||
const arch = versionText(facts.arch);
|
||||
return [
|
||||
"## 升级迁移失败",
|
||||
`- 从 MemHub ${versionText(facts.fromVersion)}(build ${buildText(facts.fromBuild)})升级到 ${versionText(facts.toVersion)}(build ${buildText(facts.toBuild)}),迁移数据失败。`,
|
||||
`- 错误码:${CODE.test(facts.code) ? facts.code : UNKNOWN}`,
|
||||
`- 失败步骤:${stepText(facts.step, facts.subStep)}`,
|
||||
`- 数据已恢复到升级前状态:${restored}`,
|
||||
`- 系统:${system || UNKNOWN}${arch === UNKNOWN ? "" : `(${arch})`}`,
|
||||
`- 迁移日志(本地):${facts.logPath}`,
|
||||
"",
|
||||
"请把上面这个日志文件拖到这里作为附件,再提交。",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function migrationIssueUrl(code: string, body?: string): string {
|
||||
const title = `[升级迁移失败 / migration failed] ${code}`;
|
||||
return `${MIGRATION_ISSUE_URL}?${new URLSearchParams({ title }).toString()}`;
|
||||
const plain = `${MIGRATION_ISSUE_URL}?${new URLSearchParams({ title }).toString()}`;
|
||||
if (body === undefined) return plain;
|
||||
let text = body;
|
||||
for (;;) {
|
||||
const url = `${MIGRATION_ISSUE_URL}?${new URLSearchParams({ title, body: text }).toString()}`;
|
||||
if (url.length <= MAX_ISSUE_URL_LENGTH) return url;
|
||||
if (text.length === 0) return plain;
|
||||
// Cut whole characters until it fits; the log attachment carries the rest.
|
||||
text = Array.from(text)
|
||||
.slice(0, Math.floor(Array.from(text).length * 0.8))
|
||||
.join("");
|
||||
}
|
||||
}
|
||||
|
||||
/** `/Users/name/.one-memory/...` as `~/.one-memory/...`: never the user name. */
|
||||
export function displayPath(target: string, home: string | null): string {
|
||||
if (home) {
|
||||
const normalized = path.resolve(home);
|
||||
const resolved = path.resolve(target);
|
||||
if (resolved === normalized) return "~";
|
||||
if (resolved.startsWith(`${normalized}${path.sep}`)) {
|
||||
return `~${path.sep}${path.relative(normalized, resolved)}`;
|
||||
}
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
export interface MigrationLogFacts {
|
||||
fromVersion: string | null;
|
||||
fromBuild: number | null;
|
||||
step: string | null;
|
||||
subStep: string | null;
|
||||
code: string | null;
|
||||
dataRestored: boolean | null;
|
||||
}
|
||||
|
||||
function readTail(file: string): string {
|
||||
const descriptor = openSync(file, "r");
|
||||
try {
|
||||
const size = fstatSync(descriptor).size;
|
||||
const length = Math.min(size, LOG_TAIL_BYTES);
|
||||
const buffer = Buffer.alloc(length);
|
||||
readSync(descriptor, buffer, 0, length, size - length);
|
||||
return buffer.toString("utf8");
|
||||
} finally {
|
||||
closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function sourceBuild(value: unknown): number | null {
|
||||
const number = typeof value === "string" ? Number(value) : value;
|
||||
return typeof number === "number" &&
|
||||
Number.isSafeInteger(number) &&
|
||||
number > 0
|
||||
? number
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the migration log says about the last failed attempt.
|
||||
*
|
||||
* The version upgraded from is the latest one an invocation recorded; the
|
||||
* failing step is the last `step.failed` Core wrote; the restore outcome is
|
||||
* the last `restore.finished`/`restore.failed`. Anything absent stays null.
|
||||
*/
|
||||
export function readMigrationLogFacts(dataDir: string): MigrationLogFacts {
|
||||
const facts: MigrationLogFacts = {
|
||||
fromVersion: null,
|
||||
fromBuild: null,
|
||||
step: null,
|
||||
subStep: null,
|
||||
code: null,
|
||||
dataRestored: null,
|
||||
};
|
||||
let text: string;
|
||||
try {
|
||||
text = readTail(migrationLogPath(dataDir));
|
||||
} catch {
|
||||
return facts;
|
||||
}
|
||||
for (const line of text.split("\n")) {
|
||||
let entry: {
|
||||
event?: unknown;
|
||||
code?: unknown;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
try {
|
||||
entry = JSON.parse(line) as typeof entry;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const details =
|
||||
entry.details && typeof entry.details === "object" ? entry.details : {};
|
||||
if (
|
||||
entry.event === "migrate.invoked" ||
|
||||
entry.event === "migration.invoked"
|
||||
) {
|
||||
// Each invocation is a new attempt; the version upgraded from stays.
|
||||
facts.step = null;
|
||||
facts.subStep = null;
|
||||
facts.dataRestored = null;
|
||||
const version = details.source_version;
|
||||
const build = sourceBuild(details.source_build);
|
||||
if (
|
||||
typeof version === "string" &&
|
||||
VERSION.test(version) &&
|
||||
version !== "unknown"
|
||||
) {
|
||||
facts.fromVersion = version;
|
||||
}
|
||||
if (build !== null) facts.fromBuild = build;
|
||||
} else if (entry.event === "step.failed") {
|
||||
facts.step = typeof details.step === "string" ? details.step : null;
|
||||
facts.subStep =
|
||||
typeof details.sub_step === "string" ? details.sub_step : null;
|
||||
if (typeof entry.code === "string" && CODE.test(entry.code)) {
|
||||
facts.code = entry.code;
|
||||
}
|
||||
} else if (entry.event === "migrate.failed") {
|
||||
if (typeof entry.code === "string" && CODE.test(entry.code)) {
|
||||
facts.code = entry.code;
|
||||
}
|
||||
} else if (entry.event === "restore.finished") {
|
||||
facts.dataRestored = true;
|
||||
} else if (entry.event === "restore.failed") {
|
||||
facts.dataRestored = false;
|
||||
}
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
export interface MigrationDialogText {
|
||||
@@ -79,6 +300,7 @@ export function migrationDialogText(
|
||||
code: string,
|
||||
logPath: string,
|
||||
locale: string,
|
||||
dataRestored = false,
|
||||
): MigrationDialogText {
|
||||
if (locale.toLowerCase().startsWith("en")) {
|
||||
return {
|
||||
@@ -86,7 +308,11 @@ export function migrationDialogText(
|
||||
message: "The data migration for this MemHub upgrade did not finish.",
|
||||
detail:
|
||||
"To protect your data, MemHub will not start. Please attach the migration log to a GitHub issue and send it to us. " +
|
||||
`(Error code: ${code})\n\nLog: ${logPath}`,
|
||||
`(Error code: ${code})` +
|
||||
(dataRestored
|
||||
? "\n\nYour data was restored to its state before the upgrade."
|
||||
: "") +
|
||||
`\n\nLog: ${logPath}`,
|
||||
openLog: "Open log",
|
||||
quit: "Quit",
|
||||
};
|
||||
@@ -96,7 +322,9 @@ export function migrationDialogText(
|
||||
message: "MemHub 升级时数据迁移没有完成。",
|
||||
detail:
|
||||
"为保护你的数据,MemHub 不会启动。请把迁移日志附在 GitHub 反馈里发给我们。" +
|
||||
`(错误码:${code})\n\n日志:${logPath}`,
|
||||
`(错误码:${code})` +
|
||||
(dataRestored ? "\n\n记忆数据已恢复到升级前的状态。" : "") +
|
||||
`\n\n日志:${logPath}`,
|
||||
openLog: "打开日志",
|
||||
quit: "退出",
|
||||
};
|
||||
@@ -117,6 +345,17 @@ export interface MigrationDialogDependencies {
|
||||
openExternal(url: string): Promise<void> | void;
|
||||
}
|
||||
|
||||
/** The running app and system, for the issue body. */
|
||||
export interface MigrationDialogEnvironment {
|
||||
appVersion?: string | null;
|
||||
buildNumber?: number | null;
|
||||
osName?: string | null;
|
||||
osVersion?: string | null;
|
||||
arch?: string | null;
|
||||
/** The account home, shown as `~` in the issue body. */
|
||||
home?: string | null;
|
||||
}
|
||||
|
||||
/** Make sure the file the dialog points at exists and ends with this failure. */
|
||||
function appendFinalLine(dataDir: string, code: string): void {
|
||||
try {
|
||||
@@ -150,12 +389,25 @@ export function showMigrationFailureDialog(options: {
|
||||
dataDir: string | null;
|
||||
locale: string;
|
||||
dependencies: MigrationDialogDependencies;
|
||||
environment?: MigrationDialogEnvironment;
|
||||
}): boolean {
|
||||
const code = migrationFailureCode(options.error, options.dataDir);
|
||||
if (code === null || options.dataDir === null) return false;
|
||||
const block = readBlock(options.dataDir);
|
||||
const logFacts = readMigrationLogFacts(options.dataDir);
|
||||
// The log says whether this attempt restored (是) or failed to (否); a
|
||||
// failure before anything changed has no restore line and stays 未知
|
||||
// rather than a misleading 否.
|
||||
const dataRestored =
|
||||
logFacts.dataRestored ?? (block?.dataRestored === true ? true : null);
|
||||
appendFinalLine(options.dataDir, code);
|
||||
const logPath = migrationLogPath(options.dataDir);
|
||||
const text = migrationDialogText(code, logPath, options.locale);
|
||||
const text = migrationDialogText(
|
||||
code,
|
||||
logPath,
|
||||
options.locale,
|
||||
dataRestored === true,
|
||||
);
|
||||
const choice = options.dependencies.showMessageBoxSync({
|
||||
type: "error",
|
||||
title: text.title,
|
||||
@@ -168,8 +420,23 @@ export function showMigrationFailureDialog(options: {
|
||||
});
|
||||
if (choice === 0) {
|
||||
options.dependencies.showItemInFolder(logPath);
|
||||
const environment = options.environment ?? {};
|
||||
const body = migrationIssueBody({
|
||||
code,
|
||||
fromVersion: logFacts.fromVersion,
|
||||
fromBuild: logFacts.fromBuild,
|
||||
toVersion: environment.appVersion ?? null,
|
||||
toBuild: environment.buildNumber ?? null,
|
||||
step: logFacts.step,
|
||||
subStep: logFacts.subStep,
|
||||
dataRestored,
|
||||
osName: environment.osName ?? null,
|
||||
osVersion: environment.osVersion ?? null,
|
||||
arch: environment.arch ?? null,
|
||||
logPath: displayPath(logPath, environment.home ?? null),
|
||||
});
|
||||
void Promise.resolve(
|
||||
options.dependencies.openExternal(migrationIssueUrl(code)),
|
||||
options.dependencies.openExternal(migrationIssueUrl(code, body)),
|
||||
).catch(() => undefined);
|
||||
}
|
||||
return true;
|
||||
|
||||
@@ -176,6 +176,7 @@ export {
|
||||
} from "./runtime-supervisor.js";
|
||||
export {
|
||||
type CoreCommand,
|
||||
readInstalledRuntimeSource,
|
||||
runSdkMigration,
|
||||
type SdkMigrationCommandResult,
|
||||
type SdkMigrationOptions,
|
||||
@@ -183,13 +184,18 @@ export {
|
||||
} from "./sdk-migration.js";
|
||||
export {
|
||||
clearSdkMigrationBlock,
|
||||
freezeCapture,
|
||||
readSdkMigrationBlock,
|
||||
recordSdkMigrationBlock,
|
||||
SDK_MIGRATION_CAPTURE_FENCE,
|
||||
SDK_MIGRATION_LOG_NAME,
|
||||
type SdkMigrationBlock,
|
||||
type SdkMigrationBlockDetails,
|
||||
type SdkMigrationCaller,
|
||||
type SdkMigrationLogEntry,
|
||||
type SdkMigrationSource,
|
||||
sdkMigrationLogPath,
|
||||
thawCapture,
|
||||
writeSdkMigrationLog,
|
||||
} from "./sdk-migration-log.js";
|
||||
export {
|
||||
|
||||
@@ -841,12 +841,17 @@ export async function gateServeOnSdkMigration(
|
||||
* Run the migration gate until the home is on SDK authority, or the service is
|
||||
* asked to stop. Core is never spawned while this has not returned "serve".
|
||||
*
|
||||
* A deferred migration leaves the service alive but idle (no Core), instead of
|
||||
* exiting: an exit would have launchd/systemd re-run the failed migration
|
||||
* every few seconds. The refusal goes to the migration log and the Runtime
|
||||
* diagnostics, the block note tells `status`/`doctor` and Desktop why. When a
|
||||
* migration elsewhere (Desktop, `memhub upgrade`) succeeds it clears the
|
||||
* note, and the gate runs again here.
|
||||
* A failed migration freezes the home (see `runSdkMigration`): the service
|
||||
* then exits instead of idling, and its enable marker is off so launchd and
|
||||
* systemd do not start it again to retry on their own. A service started on a
|
||||
* frozen home (a Desktop login launch, `memhub install` re-registering it)
|
||||
* exits the same way without migrating; the next foreground attempt (Desktop
|
||||
* launch, `memhub install`/`upgrade`/`start`/`restart`) retries and lifts the
|
||||
* freeze. The refusal goes to the migration log and the Runtime diagnostics,
|
||||
* the block note tells `status`/`doctor` and Desktop why.
|
||||
*
|
||||
* When the freeze could not be recorded, the service falls back to staying
|
||||
* alive but idle (no Core) until another migrator clears the block note.
|
||||
*/
|
||||
export async function holdUntilSdkMigrated(
|
||||
config: Pick<RuntimeCliConfig, "dataDir" | "coreCommand" | "buildIdentity">,
|
||||
@@ -860,18 +865,15 @@ export async function holdUntilSdkMigrated(
|
||||
void options.stopped.then(() => {
|
||||
stop = true;
|
||||
});
|
||||
for (;;) {
|
||||
if (stop) return "stop";
|
||||
const migration = await gateServeOnSdkMigration(config, options.migrate);
|
||||
if (migration.state !== "deferred") return "serve";
|
||||
const code = migration.code ?? "sdk_migration_incomplete";
|
||||
const runtimeBuild =
|
||||
config.buildIdentity?.buildNumber ?? config.buildIdentity?.version;
|
||||
const refuse = async (code: string, frozen: boolean) => {
|
||||
await writeSdkMigrationLog(config.dataDir, {
|
||||
writer: "runtime-service",
|
||||
event: "service.core_refused",
|
||||
event: frozen ? "service.frozen_exit" : "service.core_refused",
|
||||
outcome: "refused",
|
||||
code,
|
||||
runtimeBuild:
|
||||
config.buildIdentity?.buildNumber ?? config.buildIdentity?.version,
|
||||
runtimeBuild,
|
||||
});
|
||||
await new DiagnosticLogger({
|
||||
dataDir: config.dataDir,
|
||||
@@ -887,6 +889,28 @@ export async function holdUntilSdkMigrated(
|
||||
})
|
||||
.catch(() => undefined);
|
||||
process.stderr.write(`sdk_migration_required: ${code}\n`);
|
||||
};
|
||||
const frozenHome = async (): Promise<string | null> => {
|
||||
const block = await readSdkMigrationBlock(config.dataDir);
|
||||
if (block?.frozen === undefined) return null;
|
||||
// Whoever started this service turned the marker on; a frozen home keeps
|
||||
// it off so the exit below is not followed by a relaunch.
|
||||
await disableRuntimeService(config.dataDir).catch(() => undefined);
|
||||
return block.code;
|
||||
};
|
||||
const frozenAtStart = await frozenHome();
|
||||
if (frozenAtStart !== null) {
|
||||
await refuse(frozenAtStart, true);
|
||||
return "stop";
|
||||
}
|
||||
for (;;) {
|
||||
if (stop) return "stop";
|
||||
const migration = await gateServeOnSdkMigration(config, options.migrate);
|
||||
if (migration.state !== "deferred") return "serve";
|
||||
const code = migration.code ?? "sdk_migration_incomplete";
|
||||
const frozen = (await frozenHome()) !== null;
|
||||
await refuse(code, frozen);
|
||||
if (frozen) return "stop";
|
||||
// Wait for a stop, or for another migrator to clear the block note.
|
||||
for (;;) {
|
||||
const outcome = await Promise.race([
|
||||
|
||||
@@ -51,6 +51,20 @@ it("restores the prior enable marker when offline maintenance fails", async () =
|
||||
expect(events).toEqual(["disable", "stop", "migrate", "enable"]);
|
||||
});
|
||||
|
||||
it("leaves the service off when the maintenance froze the home", async () => {
|
||||
const { events, value } = dependencies(true);
|
||||
|
||||
await withRuntimeMaintenanceFence(
|
||||
"/data",
|
||||
async () => void events.push("migrate"),
|
||||
{ ...value, homeFrozen: async () => true },
|
||||
);
|
||||
|
||||
// A failed migration froze the home: the service stays off until the next
|
||||
// migration succeeds and turns it back on itself.
|
||||
expect(events).toEqual(["disable", "stop", "migrate"]);
|
||||
});
|
||||
|
||||
it("does not invent an enable marker for a manually stopped Runtime", async () => {
|
||||
const { events, value } = dependencies(false);
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { stopRuntimeProcess } from "./runtime-launcher.js";
|
||||
import { readSdkMigrationBlock } from "./sdk-migration-log.js";
|
||||
import {
|
||||
disableRuntimeService,
|
||||
enableRuntimeService,
|
||||
@@ -10,6 +11,12 @@ export interface RuntimeMaintenanceDependencies {
|
||||
disableService(dataDir: string): Promise<void>;
|
||||
enableService(dataDir: string): Promise<void>;
|
||||
stopRuntime(options: { dataDir: string }): Promise<"stopped" | "not-running">;
|
||||
/** True when the maintenance left the home frozen after a failed migration. */
|
||||
homeFrozen?(dataDir: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
async function homeFrozen(dataDir: string): Promise<boolean> {
|
||||
return (await readSdkMigrationBlock(dataDir))?.frozen !== undefined;
|
||||
}
|
||||
|
||||
const DEFAULT_DEPENDENCIES: RuntimeMaintenanceDependencies = {
|
||||
@@ -17,12 +24,15 @@ const DEFAULT_DEPENDENCIES: RuntimeMaintenanceDependencies = {
|
||||
disableService: disableRuntimeService,
|
||||
enableService: enableRuntimeService,
|
||||
stopRuntime: stopRuntimeProcess,
|
||||
homeFrozen,
|
||||
};
|
||||
|
||||
/**
|
||||
* Run offline maintenance while preventing a managed service from restarting
|
||||
* Runtime between stop and completion. The prior enable-marker state is
|
||||
* restored even when maintenance fails.
|
||||
* restored even when maintenance fails, unless the maintenance froze the home
|
||||
* (a failed SDK migration): then the service stays off until the next
|
||||
* migration succeeds, which turns it back on itself.
|
||||
*/
|
||||
export async function withRuntimeMaintenanceFence<T>(
|
||||
dataDir: string,
|
||||
@@ -35,6 +45,12 @@ export async function withRuntimeMaintenanceFence<T>(
|
||||
await dependencies.stopRuntime({ dataDir });
|
||||
return await maintain();
|
||||
} finally {
|
||||
if (wasEnabled) await dependencies.enableService(dataDir);
|
||||
if (
|
||||
wasEnabled &&
|
||||
!(await (dependencies.homeFrozen ?? homeFrozen)(dataDir).catch(
|
||||
() => false,
|
||||
))
|
||||
)
|
||||
await dependencies.enableService(dataDir);
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,10 @@ import {
|
||||
sdkMigrationLogPath,
|
||||
writeSdkMigrationLog,
|
||||
} from "./sdk-migration-log.js";
|
||||
import {
|
||||
enableRuntimeService,
|
||||
runtimeServiceEnabled,
|
||||
} from "./service-ownership.js";
|
||||
|
||||
const core = { command: "/core/one-memory", args: [] };
|
||||
const noFence = async <T>(
|
||||
@@ -45,6 +49,87 @@ async function lines(dataDir: string): Promise<Record<string, unknown>[]> {
|
||||
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
describe("a frozen home's Runtime service", () => {
|
||||
const config = (dataDir: string) => ({
|
||||
dataDir,
|
||||
coreCommand: "/runtime/core/one-memory",
|
||||
buildIdentity: { buildNumber: "22" },
|
||||
});
|
||||
|
||||
it("exits without migrating and keeps its marker off when started on a frozen home", async () => {
|
||||
const dataDir = await home();
|
||||
await enableRuntimeService(dataDir);
|
||||
await recordSdkMigrationBlock(
|
||||
dataDir,
|
||||
"sdk_migration_memory_unreadable",
|
||||
"desktop",
|
||||
{ frozen: { service_was_enabled: true } },
|
||||
);
|
||||
const migrate = vi.fn();
|
||||
|
||||
await expect(
|
||||
holdUntilSdkMigrated(config(dataDir), {
|
||||
stopped: new Promise(() => undefined),
|
||||
migrate,
|
||||
}),
|
||||
).resolves.toBe("stop");
|
||||
|
||||
expect(migrate).not.toHaveBeenCalled();
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(false);
|
||||
expect(
|
||||
(await lines(dataDir)).find(
|
||||
(line) => line.event === "service.frozen_exit",
|
||||
),
|
||||
).toMatchObject({
|
||||
writer: "runtime-service",
|
||||
outcome: "refused",
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
});
|
||||
});
|
||||
|
||||
it("exits instead of idling when its own migration fails and freezes the home", async () => {
|
||||
const dataDir = await home();
|
||||
await enableRuntimeService(dataDir);
|
||||
const migrate = vi.fn(async (options: SdkMigrationOptions) =>
|
||||
runSdkMigration({
|
||||
...options,
|
||||
execute: vi.fn(async (_core, args: string[]) =>
|
||||
args[0] === "sdk-migration-status"
|
||||
? {
|
||||
exitCode: 0,
|
||||
document: { authority: "legacy", state: "required" },
|
||||
}
|
||||
: {
|
||||
exitCode: 1,
|
||||
document: {
|
||||
state: "failed",
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
data_restored: true,
|
||||
},
|
||||
},
|
||||
),
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
holdUntilSdkMigrated(config(dataDir), {
|
||||
stopped: new Promise(() => undefined),
|
||||
migrate,
|
||||
pollIntervalMs: 5,
|
||||
}),
|
||||
).resolves.toBe("stop");
|
||||
|
||||
expect(migrate).toHaveBeenCalledTimes(1);
|
||||
// launchd/systemd will not start it again to retry on its own.
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(false);
|
||||
expect(await readSdkMigrationBlock(dataDir)).toMatchObject({
|
||||
caller: "runtime-service",
|
||||
frozen: { service_was_enabled: true },
|
||||
data_restored: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("sdk-migration.log", () => {
|
||||
it("records every orchestrator decision of a deferred migration, content-free", async () => {
|
||||
const dataDir = await home();
|
||||
|
||||
@@ -25,6 +25,8 @@ export type SdkMigrationCaller =
|
||||
| "desktop-background"
|
||||
| "cli-install"
|
||||
| "cli-upgrade"
|
||||
| "cli-start"
|
||||
| "cli-restart"
|
||||
| "runtime-service";
|
||||
|
||||
export const SDK_MIGRATION_LOG_NAME = "sdk-migration.log";
|
||||
@@ -128,6 +130,33 @@ export interface SdkMigrationBlock {
|
||||
code: string;
|
||||
caller: SdkMigrationCaller;
|
||||
at: string;
|
||||
/**
|
||||
* The home is frozen after a failed migration: the Runtime service marker
|
||||
* is off, the service exits instead of idling and host capture is fenced.
|
||||
* The next foreground migration (Desktop launch, `memhub install`,
|
||||
* `upgrade`, `start`/`restart`) lifts it, and re-enables the service when it
|
||||
* was enabled before the freeze.
|
||||
*/
|
||||
frozen?: { service_was_enabled: boolean };
|
||||
/** Core put the data back exactly as it was before the upgrade. */
|
||||
data_restored?: boolean;
|
||||
/** Why that restore did not happen, when Core tried and failed. */
|
||||
restore_code?: string;
|
||||
/** The Runtime the home was upgraded from, when it is known. */
|
||||
source?: SdkMigrationSource;
|
||||
}
|
||||
|
||||
/** Build and version of the Runtime an upgrade replaced, from its manifest. */
|
||||
export interface SdkMigrationSource {
|
||||
version: string | null;
|
||||
build_number: number | null;
|
||||
}
|
||||
|
||||
export interface SdkMigrationBlockDetails {
|
||||
frozen?: { service_was_enabled: boolean };
|
||||
data_restored?: boolean;
|
||||
restore_code?: string;
|
||||
source?: SdkMigrationSource | null;
|
||||
}
|
||||
|
||||
const BLOCK_NAME = "sdk-migration-blocked.json";
|
||||
@@ -140,6 +169,7 @@ export async function recordSdkMigrationBlock(
|
||||
dataDir: string,
|
||||
code: string,
|
||||
caller: SdkMigrationCaller,
|
||||
details: SdkMigrationBlockDetails = {},
|
||||
): Promise<void> {
|
||||
try {
|
||||
await mkdir(dataDir, { recursive: true, mode: 0o700 });
|
||||
@@ -147,7 +177,20 @@ export async function recordSdkMigrationBlock(
|
||||
const temporary = `${target}.${process.pid}.tmp`;
|
||||
await writeFile(
|
||||
temporary,
|
||||
`${JSON.stringify({ schema_version: 1, code, caller, at: new Date().toISOString() })}\n`,
|
||||
`${JSON.stringify({
|
||||
schema_version: 1,
|
||||
code,
|
||||
caller,
|
||||
at: new Date().toISOString(),
|
||||
...(details.frozen === undefined ? {} : { frozen: details.frozen }),
|
||||
...(details.data_restored === undefined
|
||||
? {}
|
||||
: { data_restored: details.data_restored }),
|
||||
...(details.restore_code === undefined
|
||||
? {}
|
||||
: { restore_code: details.restore_code }),
|
||||
...(details.source ? { source: details.source } : {}),
|
||||
})}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
await chmod(temporary, 0o600);
|
||||
@@ -161,6 +204,25 @@ export async function clearSdkMigrationBlock(dataDir: string): Promise<void> {
|
||||
await rm(blockPath(dataDir), { force: true }).catch(() => undefined);
|
||||
}
|
||||
|
||||
const SAFE_CODE = /^[a-z][a-z0-9_]{0,95}$/;
|
||||
const SAFE_VERSION = /^[0-9A-Za-z][0-9A-Za-z.+-]{0,63}$/;
|
||||
|
||||
function sourceFrom(value: unknown): SdkMigrationSource | undefined {
|
||||
if (!value || typeof value !== "object") return undefined;
|
||||
const { version, build_number } = value as Record<string, unknown>;
|
||||
const safeVersion =
|
||||
typeof version === "string" && SAFE_VERSION.test(version) ? version : null;
|
||||
const safeBuild =
|
||||
typeof build_number === "number" &&
|
||||
Number.isSafeInteger(build_number) &&
|
||||
build_number > 0
|
||||
? build_number
|
||||
: null;
|
||||
return safeVersion === null && safeBuild === null
|
||||
? undefined
|
||||
: { version: safeVersion, build_number: safeBuild };
|
||||
}
|
||||
|
||||
export async function readSdkMigrationBlock(
|
||||
dataDir: string,
|
||||
): Promise<SdkMigrationBlock | null> {
|
||||
@@ -170,14 +232,74 @@ export async function readSdkMigrationBlock(
|
||||
) as Partial<SdkMigrationBlock>;
|
||||
if (
|
||||
typeof value.code !== "string" ||
|
||||
!/^[a-z][a-z0-9_]{0,95}$/.test(value.code) ||
|
||||
!SAFE_CODE.test(value.code) ||
|
||||
typeof value.caller !== "string" ||
|
||||
typeof value.at !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return value as SdkMigrationBlock;
|
||||
const block: SdkMigrationBlock = {
|
||||
code: value.code,
|
||||
caller: value.caller,
|
||||
at: value.at,
|
||||
};
|
||||
if (
|
||||
value.frozen &&
|
||||
typeof value.frozen === "object" &&
|
||||
typeof value.frozen.service_was_enabled === "boolean"
|
||||
) {
|
||||
block.frozen = { service_was_enabled: value.frozen.service_was_enabled };
|
||||
}
|
||||
if (typeof value.data_restored === "boolean") {
|
||||
block.data_restored = value.data_restored;
|
||||
}
|
||||
if (
|
||||
typeof value.restore_code === "string" &&
|
||||
SAFE_CODE.test(value.restore_code)
|
||||
) {
|
||||
block.restore_code = value.restore_code;
|
||||
}
|
||||
const source = sourceFrom(value.source);
|
||||
if (source !== undefined) block.source = source;
|
||||
return block;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `<dataDir>/.sdk-migration-capture-fence.json`, shared with Core: while it is
|
||||
* live, host capture hooks (`one-memory capture <host>`) return without
|
||||
* writing. Core holds it with its pid during `sdk-migrate`; a frozen home
|
||||
* holds it without one until the next migration lifts it.
|
||||
*/
|
||||
export const SDK_MIGRATION_CAPTURE_FENCE = ".sdk-migration-capture-fence.json";
|
||||
|
||||
export async function freezeCapture(dataDir: string): Promise<void> {
|
||||
try {
|
||||
const target = path.join(dataDir, SDK_MIGRATION_CAPTURE_FENCE);
|
||||
const temporary = `${target}.${process.pid}.tmp`;
|
||||
await writeFile(
|
||||
temporary,
|
||||
`${JSON.stringify({ schema_version: 1, frozen: true, reason: "sdk_migration_failed" })}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
await chmod(temporary, 0o600);
|
||||
await rename(temporary, target);
|
||||
} catch {
|
||||
// Core leaves the same fence on its own failures.
|
||||
}
|
||||
}
|
||||
|
||||
/** Lift a frozen capture fence; a live one (a running migration) is kept. */
|
||||
export async function thawCapture(dataDir: string): Promise<void> {
|
||||
const target = path.join(dataDir, SDK_MIGRATION_CAPTURE_FENCE);
|
||||
try {
|
||||
const value = JSON.parse(await readFile(target, "utf8")) as {
|
||||
frozen?: unknown;
|
||||
};
|
||||
if (value.frozen === true) await rm(target, { force: true });
|
||||
} catch {
|
||||
// Absent or unreadable: Core treats an unreadable fence as stale.
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,23 @@
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { runSdkMigration } from "./sdk-migration.js";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { withRuntimeMaintenanceFence } from "./runtime-maintenance.js";
|
||||
import {
|
||||
readInstalledRuntimeSource,
|
||||
runSdkMigration,
|
||||
} from "./sdk-migration.js";
|
||||
import {
|
||||
freezeCapture,
|
||||
readSdkMigrationBlock,
|
||||
SDK_MIGRATION_CAPTURE_FENCE,
|
||||
thawCapture,
|
||||
} from "./sdk-migration-log.js";
|
||||
import {
|
||||
disableRuntimeService,
|
||||
enableRuntimeService,
|
||||
runtimeServiceEnabled,
|
||||
} from "./service-ownership.js";
|
||||
|
||||
const core = { command: "/core/one-memory", args: [] };
|
||||
const noFence = async <T>(
|
||||
@@ -218,3 +233,212 @@ it("streams Core's progress lines to the caller", async () => {
|
||||
expect(result).toEqual({ state: "cutover" });
|
||||
expect(steps).toEqual(["bundle", "cutover"]);
|
||||
});
|
||||
|
||||
describe("a failed migration freezes the home until the next attempt", () => {
|
||||
async function homeWithService(enabled: boolean): Promise<string> {
|
||||
const dataDir = await mkdtemp(path.join(os.tmpdir(), "sdk-freeze-"));
|
||||
if (enabled) await enableRuntimeService(dataDir);
|
||||
return dataDir;
|
||||
}
|
||||
// The real fence, with only the Runtime stop faked.
|
||||
const fence = <T>(dataDir: string, action: () => Promise<T>) =>
|
||||
withRuntimeMaintenanceFence(dataDir, action, {
|
||||
serviceEnabled: runtimeServiceEnabled,
|
||||
disableService: disableRuntimeService,
|
||||
enableService: enableRuntimeService,
|
||||
stopRuntime: async () => "stopped" as const,
|
||||
});
|
||||
const failed = (document: Record<string, unknown> = {}) =>
|
||||
statusThen({
|
||||
exitCode: 1,
|
||||
document: {
|
||||
state: "failed",
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
...document,
|
||||
},
|
||||
});
|
||||
const fenceFile = (dataDir: string) =>
|
||||
path.join(dataDir, SDK_MIGRATION_CAPTURE_FENCE);
|
||||
|
||||
it("keeps the service off, fences capture and records the freeze", async () => {
|
||||
const dataDir = await homeWithService(true);
|
||||
|
||||
const result = await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "22",
|
||||
execute: failed({ data_restored: true }),
|
||||
fence,
|
||||
sourceRuntime: { version: "1.0.4", build_number: 104 },
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
state: "deferred",
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
dataRestored: true,
|
||||
});
|
||||
// The fence did not turn the service back on.
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(false);
|
||||
expect(
|
||||
JSON.parse(await readFile(fenceFile(dataDir), "utf8")),
|
||||
).toMatchObject({ frozen: true });
|
||||
expect(await readSdkMigrationBlock(dataDir)).toMatchObject({
|
||||
code: "sdk_migration_memory_unreadable",
|
||||
frozen: { service_was_enabled: true },
|
||||
data_restored: true,
|
||||
source: { version: "1.0.4", build_number: 104 },
|
||||
});
|
||||
});
|
||||
|
||||
it("lifts the freeze and turns the service back on after a successful retry", async () => {
|
||||
const dataDir = await homeWithService(true);
|
||||
await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "22",
|
||||
execute: failed(),
|
||||
fence,
|
||||
sourceRuntime: { version: "1.0.4", build_number: 104 },
|
||||
});
|
||||
const retry = statusThen({ exitCode: 0, document: { state: "cutover" } });
|
||||
|
||||
const result = await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "23",
|
||||
execute: retry,
|
||||
fence,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ state: "cutover" });
|
||||
expect(await readSdkMigrationBlock(dataDir)).toBeNull();
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(true);
|
||||
await expect(readFile(fenceFile(dataDir), "utf8")).rejects.toThrow();
|
||||
// The retry still names the version the home was upgraded from.
|
||||
const call = retry.mock.calls[1]?.[0] as
|
||||
| { env?: NodeJS.ProcessEnv }
|
||||
| undefined;
|
||||
const env = call?.env;
|
||||
expect(env?.ONE_MEMORY_MIGRATION_SOURCE_VERSION).toBe("1.0.4");
|
||||
expect(env?.ONE_MEMORY_MIGRATION_SOURCE_BUILD).toBe("104");
|
||||
const log = await readFile(
|
||||
path.join(dataDir, "logs", "sdk-migration.log"),
|
||||
"utf8",
|
||||
);
|
||||
expect(log).toContain('"event":"migration.unfrozen"');
|
||||
expect(log).toContain('"source_version":"1.0.4"');
|
||||
});
|
||||
|
||||
it("does not turn on a service that was off before the freeze", async () => {
|
||||
const dataDir = await homeWithService(false);
|
||||
await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "22",
|
||||
execute: failed(),
|
||||
fence,
|
||||
});
|
||||
|
||||
await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "23",
|
||||
execute: statusThen({ exitCode: 0, document: { state: "cutover" } }),
|
||||
fence,
|
||||
});
|
||||
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(false);
|
||||
expect(await readSdkMigrationBlock(dataDir)).toBeNull();
|
||||
});
|
||||
|
||||
it("a second failure keeps what the first freeze recorded", async () => {
|
||||
const dataDir = await homeWithService(true);
|
||||
await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "22",
|
||||
execute: failed(),
|
||||
fence,
|
||||
sourceRuntime: { version: "1.0.4", build_number: 104 },
|
||||
});
|
||||
|
||||
await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "23",
|
||||
execute: failed(),
|
||||
fence,
|
||||
});
|
||||
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(false);
|
||||
expect(await readSdkMigrationBlock(dataDir)).toMatchObject({
|
||||
frozen: { service_was_enabled: true },
|
||||
source: { version: "1.0.4", build_number: 104 },
|
||||
});
|
||||
});
|
||||
|
||||
it("an unreadable status freezes inside the fence", async () => {
|
||||
const dataDir = await homeWithService(true);
|
||||
|
||||
const result = await runSdkMigration({
|
||||
dataDir,
|
||||
core,
|
||||
runtimeBuild: "22",
|
||||
execute: vi.fn(async () => ({ exitCode: 1, document: {} })),
|
||||
fence,
|
||||
});
|
||||
|
||||
expect(result.state).toBe("deferred");
|
||||
expect(await runtimeServiceEnabled(dataDir)).toBe(false);
|
||||
expect((await readSdkMigrationBlock(dataDir))?.frozen).toEqual({
|
||||
service_was_enabled: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("thawing never lifts the live fence of a running migration", async () => {
|
||||
const dataDir = await homeWithService(false);
|
||||
await writeFile(
|
||||
fenceFile(dataDir),
|
||||
JSON.stringify({ schema_version: 1, pid: process.pid }),
|
||||
);
|
||||
await thawCapture(dataDir);
|
||||
expect(JSON.parse(await readFile(fenceFile(dataDir), "utf8"))).toEqual({
|
||||
schema_version: 1,
|
||||
pid: process.pid,
|
||||
});
|
||||
await freezeCapture(dataDir);
|
||||
await thawCapture(dataDir);
|
||||
await expect(readFile(fenceFile(dataDir), "utf8")).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("the version an upgrade replaced", () => {
|
||||
it("reads a release version and build from the previous install manifest", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "sdk-source-"));
|
||||
await mkdir(path.join(root, "versions", "old"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(root, "versions", "old", "manifest.json"),
|
||||
JSON.stringify({ version: "1.0.4", build_number: 104 }),
|
||||
);
|
||||
await mkdir(path.join(root, "versions", "app"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(root, "versions", "app", "manifest.json"),
|
||||
JSON.stringify({ version: "ab".repeat(32), build_number: 99 }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
readInstalledRuntimeSource(root, "versions/old"),
|
||||
).resolves.toEqual({ version: "1.0.4", build_number: 104 });
|
||||
// A packaged App runtime is versioned by a digest: only its build is told.
|
||||
await expect(
|
||||
readInstalledRuntimeSource(root, "versions/app"),
|
||||
).resolves.toEqual({ version: null, build_number: 99 });
|
||||
await expect(readInstalledRuntimeSource(root, null)).resolves.toBeNull();
|
||||
await expect(
|
||||
readInstalledRuntimeSource(root, "../outside"),
|
||||
).resolves.toBeNull();
|
||||
await expect(
|
||||
readInstalledRuntimeSource(root, "versions/missing"),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,24 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { withRuntimeMaintenanceFence } from "./runtime-maintenance.js";
|
||||
import {
|
||||
clearSdkMigrationBlock,
|
||||
freezeCapture,
|
||||
migrationErrorCode,
|
||||
readSdkMigrationBlock,
|
||||
recordSdkMigrationBlock,
|
||||
type SdkMigrationCaller,
|
||||
type SdkMigrationLogEntry,
|
||||
type SdkMigrationSource,
|
||||
thawCapture,
|
||||
writeSdkMigrationLog,
|
||||
} from "./sdk-migration-log.js";
|
||||
import {
|
||||
disableRuntimeService,
|
||||
enableRuntimeService,
|
||||
runtimeServiceEnabled,
|
||||
} from "./service-ownership.js";
|
||||
|
||||
const MAX_OUTPUT_BYTES = 1024 * 1024;
|
||||
|
||||
@@ -15,6 +26,8 @@ export interface SdkMigrationResult {
|
||||
/** `complete`: already on SDK authority; leftovers were repaired. */
|
||||
state: "not_required" | "cutover" | "complete" | "deferred";
|
||||
code?: string;
|
||||
/** Deferred only: Core put the data back as it was before the upgrade. */
|
||||
dataRestored?: boolean;
|
||||
}
|
||||
|
||||
export interface SdkMigrationCommandResult {
|
||||
@@ -46,19 +59,68 @@ export interface SdkMigrationOptions {
|
||||
onProgress?: (step: string) => void;
|
||||
/** Who is migrating; stamped on every line of the migration log. */
|
||||
caller?: SdkMigrationCaller;
|
||||
/**
|
||||
* The Runtime this upgrade replaced, read from its install manifest before
|
||||
* `current` moved. Unknown (`null`) falls back to what an earlier failed
|
||||
* attempt recorded in the block note; it is never guessed.
|
||||
*/
|
||||
sourceRuntime?: SdkMigrationSource | null;
|
||||
}
|
||||
|
||||
/** Build and version of an installed Runtime, from its `manifest.json`. */
|
||||
export async function readInstalledRuntimeSource(
|
||||
installRoot: string,
|
||||
target: string | null | undefined,
|
||||
): Promise<SdkMigrationSource | null> {
|
||||
if (!target) return null;
|
||||
const directory = path.resolve(installRoot, target);
|
||||
if (
|
||||
path.dirname(directory) !== path.join(path.resolve(installRoot), "versions")
|
||||
)
|
||||
return null;
|
||||
try {
|
||||
const manifest = JSON.parse(
|
||||
await readFile(path.join(directory, "manifest.json"), "utf8"),
|
||||
) as { version?: unknown; build_number?: unknown };
|
||||
const version =
|
||||
typeof manifest.version === "string" &&
|
||||
// Packaged App runtimes are versioned by a tree digest, which says
|
||||
// nothing to a person; only a release version is reported.
|
||||
/^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]{1,40})?$/.test(manifest.version)
|
||||
? manifest.version
|
||||
: null;
|
||||
const build =
|
||||
typeof manifest.build_number === "number" &&
|
||||
Number.isSafeInteger(manifest.build_number) &&
|
||||
manifest.build_number > 0
|
||||
? manifest.build_number
|
||||
: null;
|
||||
return version === null && build === null
|
||||
? null
|
||||
: { version, build_number: build };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring a home to SDK memory authority before a Runtime serves it.
|
||||
*
|
||||
* Core's `sdk-migrate` does the whole migration and resumes after any failure;
|
||||
* the caller (Desktop startup, CLI install/upgrade, the Runtime service) only
|
||||
* Core's `sdk-migrate` does the whole migration. A failure before SDK
|
||||
* authority is committed puts the data back as it was before the upgrade; the
|
||||
* caller (Desktop startup, CLI install/upgrade, the Runtime service) only
|
||||
* keeps the Runtime stopped, and its service from restarting, while it runs.
|
||||
*
|
||||
* Every decision is appended to `<dataDir>/logs/sdk-migration.log`, and Core
|
||||
* writes its own phase and failure lines to the same file, so one attachment
|
||||
* explains an upgrade that did not finish. A deferred result leaves a durable
|
||||
* block note that `memhub status` and Desktop read; a successful one clears it.
|
||||
* explains an upgrade that did not finish.
|
||||
*
|
||||
* A deferred result freezes the home until the next attempt: the block note
|
||||
* (read by `memhub status` and Desktop) records the freeze, the Runtime
|
||||
* service marker stays off so launchd/systemd do not start a Runtime that
|
||||
* would retry on its own, and host capture stays fenced so nothing writes
|
||||
* into the restored data. A successful attempt lifts all of it and turns the
|
||||
* service back on when it was on before the freeze.
|
||||
*/
|
||||
export async function runSdkMigration(
|
||||
options: SdkMigrationOptions,
|
||||
@@ -71,6 +133,13 @@ export async function runSdkMigration(
|
||||
...entry,
|
||||
});
|
||||
const started = Date.now();
|
||||
const prior = await readSdkMigrationBlock(options.dataDir);
|
||||
// Whether the service ran before this home was (first) frozen: a freeze
|
||||
// turned the marker off, so the marker alone no longer says.
|
||||
const serviceWasEnabled =
|
||||
prior?.frozen?.service_was_enabled === true ||
|
||||
(await runtimeServiceEnabled(options.dataDir).catch(() => false));
|
||||
const source = options.sourceRuntime ?? prior?.source ?? null;
|
||||
const core: CoreCommand = {
|
||||
...options.core,
|
||||
env: {
|
||||
@@ -78,11 +147,18 @@ export async function runSdkMigration(
|
||||
// Core stamps these on its own lines in the shared log.
|
||||
ONE_MEMORY_MIGRATION_CALLER: caller,
|
||||
ONE_MEMORY_MIGRATION_RUNTIME_BUILD: options.runtimeBuild,
|
||||
...(source?.version
|
||||
? { ONE_MEMORY_MIGRATION_SOURCE_VERSION: source.version }
|
||||
: {}),
|
||||
...(source?.build_number
|
||||
? { ONE_MEMORY_MIGRATION_SOURCE_BUILD: String(source.build_number) }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
const finish = async (
|
||||
result: SdkMigrationResult,
|
||||
details: Record<string, unknown> = {},
|
||||
restoreCode?: string,
|
||||
): Promise<SdkMigrationResult> => {
|
||||
await log({
|
||||
event: "migration.finished",
|
||||
@@ -91,23 +167,58 @@ export async function runSdkMigration(
|
||||
details: {
|
||||
state: result.state,
|
||||
duration_ms: Date.now() - started,
|
||||
...(result.state === "deferred"
|
||||
? {
|
||||
data_restored: result.dataRestored === true,
|
||||
frozen: true,
|
||||
service_was_enabled: serviceWasEnabled,
|
||||
}
|
||||
: {}),
|
||||
...details,
|
||||
},
|
||||
});
|
||||
if (result.state === "deferred") {
|
||||
// Marker first: from here launchd/systemd do not start a Runtime.
|
||||
await disableRuntimeService(options.dataDir).catch(() => undefined);
|
||||
await freezeCapture(options.dataDir);
|
||||
await recordSdkMigrationBlock(
|
||||
options.dataDir,
|
||||
result.code ?? "sdk_migration_incomplete",
|
||||
caller,
|
||||
{
|
||||
frozen: { service_was_enabled: serviceWasEnabled },
|
||||
data_restored: result.dataRestored === true,
|
||||
...(restoreCode === undefined ? {} : { restore_code: restoreCode }),
|
||||
source,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
await clearSdkMigrationBlock(options.dataDir);
|
||||
await thawCapture(options.dataDir);
|
||||
if (prior?.frozen !== undefined) {
|
||||
if (serviceWasEnabled) {
|
||||
await enableRuntimeService(options.dataDir).catch(() => undefined);
|
||||
}
|
||||
await log({
|
||||
event: "migration.unfrozen",
|
||||
outcome: "succeeded",
|
||||
details: { service_enabled: serviceWasEnabled },
|
||||
});
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
const execute = options.execute ?? executeCoreCommand;
|
||||
const fence = options.fence ?? withRuntimeMaintenanceFence;
|
||||
await log({ event: "migration.invoked", outcome: "started" });
|
||||
await log({
|
||||
event: "migration.invoked",
|
||||
outcome: "started",
|
||||
details: {
|
||||
source_version: source?.version ?? "unknown",
|
||||
source_build: source?.build_number ?? "unknown",
|
||||
frozen: prior?.frozen !== undefined,
|
||||
},
|
||||
});
|
||||
let statusFailure = "";
|
||||
const status = await execute(
|
||||
core,
|
||||
@@ -130,12 +241,15 @@ export async function runSdkMigration(
|
||||
});
|
||||
if (status === null || status.exitCode !== 0) {
|
||||
// An unreadable status still must not leave a pre-upgrade Runtime serving
|
||||
// the home: cycle the service so launchd brings back the gated `current`.
|
||||
await fence(options.dataDir, async () => undefined).catch(() => undefined);
|
||||
return finish({
|
||||
// the home: stop it, and freeze inside the fence so the fence does not
|
||||
// turn the service back on.
|
||||
const unavailable: SdkMigrationResult = {
|
||||
state: "deferred",
|
||||
code: "sdk_migration_status_unavailable",
|
||||
});
|
||||
};
|
||||
return fence(options.dataDir, () => finish(unavailable)).catch(() =>
|
||||
finish(unavailable),
|
||||
);
|
||||
}
|
||||
if (status.document.state !== "required") {
|
||||
return finish({ state: "not_required" });
|
||||
@@ -172,15 +286,21 @@ export async function runSdkMigration(
|
||||
{ stage: "sdk_migrate_spawn" },
|
||||
);
|
||||
}
|
||||
const restoreCode =
|
||||
typeof migrated.document.restore_code === "string" &&
|
||||
/^[a-z][a-z0-9_]{0,95}$/.test(migrated.document.restore_code)
|
||||
? migrated.document.restore_code
|
||||
: undefined;
|
||||
const childDetails = {
|
||||
exit_code: migrated.exitCode,
|
||||
stderr_bytes: migrated.stderrBytes ?? null,
|
||||
document_state: migrated.document.state ?? null,
|
||||
document_code: migrated.document.code ?? null,
|
||||
migration_id: migrated.document.migration_id ?? null,
|
||||
...(restoreCode === undefined ? {} : { restore_code: restoreCode }),
|
||||
};
|
||||
if (migrated.exitCode !== 0)
|
||||
return finish(deferred(migrated), childDetails);
|
||||
return finish(deferred(migrated), childDetails, restoreCode);
|
||||
const state = migrated.document.state;
|
||||
if (state === "cutover" || state === "complete") {
|
||||
return finish({ state }, childDetails);
|
||||
@@ -188,7 +308,7 @@ export async function runSdkMigration(
|
||||
if (state === "not_required") {
|
||||
return finish({ state: "not_required" }, childDetails);
|
||||
}
|
||||
return finish(deferred(migrated), childDetails);
|
||||
return finish(deferred(migrated), childDetails, restoreCode);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -199,6 +319,7 @@ function deferred(result: SdkMigrationCommandResult): SdkMigrationResult {
|
||||
typeof result.document.code === "string"
|
||||
? result.document.code
|
||||
: "sdk_migration_incomplete",
|
||||
...(result.document.data_restored === true ? { dataRestored: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user