From 4e1748f65a926f4ffa7e2a42fe4f5e557b28fb05 Mon Sep 17 00:00:00 2001 From: alvinfei Date: Sun, 1 Nov 2026 17:48:24 +0800 Subject: [PATCH] feat(memhub): memory mode (lite/pro/ultra), evolve and its nightly schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mode: a saved setting (config.json memory_mode, default pro); ONE_MEMORY_GRAPH_STORE still forces ultra for development. - API: GET/PUT /api/v1/settings/memory-mode, which reports the switch's effects and the graph backfill. - CLI: memhub settings mode [lite|pro|ultra] [--yes], which prints the plan first. - The mode drives every SDK client. - acquire_client passes it to HyMemoryClient. The read handle rebuilds on a change. - Write handles are keyed by (profile, revision, mode) and retired with the #218 use count, so a batch in flight finishes in the old mode. - Jobs read the mode when they run: a job completed in lite is never reprocessed; pending jobs take the new mode. - Mode never touches the vector collection identity. - lite needs no generation model: the built-in embedder writes raw, and the claim gate admits lite without an active profile. lite never calls add_extracted (a note is one raw user turn through add). Recall serves the raw layer. - → ultra: build_memory_graph once per known user, in the background, with a status (graph_build). - Raw retention removed in every mode: MemHub deletes no raw (owner decision); the retention loop, its startup step and its test are gone. - P1-3 / R7: POST /api/v2/evolve answers 202 with a run receipt; GET /api/v2/evolve[/{run_id}]. - Ultra only: 409 memory_mode_not_ultra otherwise. - One run per user. Never beside that user's write: it waits for the write in flight, and holds that user's jobs (released pre-SDK) until it ends. - Loops while should_continue, up to 20 rounds. - Per-user last_run_at, last_result and last_error in evolve-state.json (counts and codes only). - CLI: memhub evolve [--user-id] [--agent-id] [--max-batches] [--wait]. - Nightly scheduler: 03:00 local by default, ultra only, with catch-up when the last run is older than the slot or 24h. - Users come from Core's records (local, grant holders, write-queue users) that the SDK holds memories for. - GET/PUT /api/v1/settings/evolve-schedule. - R10: pipeline {mode, configured_mode, forced_by_env, evolve, last_evolve_at, graph_build} in /api/v1/memory-processing, memhub status --json and memhub progress. - Contract for the App: docs/contracts/memory-mode-evolve-v1.md, including what SDK ultra additionally enables and zh/en/es copy. Co-Authored-By: Claude Opus 5.5 (1M context) --- memhub/MEMORY.md | 4 +- .../apps/cli/src/cli-contract-fixes.test.ts | 2 + memhub/apps/cli/src/cli.ts | 71 +- memhub/apps/cli/src/command-catalog.ts | 37 + memhub/apps/cli/src/command-contract.test.ts | 2 + memhub/apps/cli/src/core-client.ts | 6 + memhub/apps/cli/src/core-operations.ts | 9 +- memhub/apps/cli/src/memory-mode-cli.test.ts | 309 +++++++ memhub/apps/cli/src/memory-mode-flow.ts | 372 ++++++++ memhub/apps/cli/src/source-progress-report.ts | 23 + memhub/apps/cli/src/status-report.ts | 112 +++ memhub/apps/core/src/one_memory/api.py | 425 ++++++++- .../core/src/one_memory/context_routes.py | 6 +- .../core/src/one_memory/hy_graph_admin.py | 21 +- .../core/src/one_memory/hy_sdk_runtime.py | 13 +- .../core/src/one_memory/memory_evolution.py | 618 +++++++++++++ .../one_memory/memory_server/config_state.py | 64 +- .../one_memory/memory_server/durable_queue.py | 2 + .../core/src/one_memory/sdk_memory_runtime.py | 100 +- .../core/src/one_memory/sdk_read_handle.py | 21 +- .../src/one_memory/sdk_write_processor.py | 119 ++- memhub/apps/core/tests/TEST_INDEX.md | 3 +- .../core/tests/core_test_classification.py | 4 +- .../tests/test_core_ready_before_recovery.py | 2 +- .../core/tests/test_memory_mode_and_evolve.py | 861 ++++++++++++++++++ .../tests/test_sdk_data_plane_migration.py | 54 +- .../core/tests/test_search_without_profile.py | 2 +- memhub/docs/ARCHITECTURE.md | 2 +- memhub/docs/cli/EXPERIENCE.md | 14 + .../docs/contracts/memory-mode-evolve-v1.md | 346 +++++++ .../contracts/sdk-data-plane-migration-v1.md | 3 +- 31 files changed, 3430 insertions(+), 197 deletions(-) create mode 100644 memhub/apps/cli/src/memory-mode-cli.test.ts create mode 100644 memhub/apps/cli/src/memory-mode-flow.ts create mode 100644 memhub/apps/core/src/one_memory/memory_evolution.py create mode 100644 memhub/apps/core/tests/test_memory_mode_and_evolve.py create mode 100644 memhub/docs/contracts/memory-mode-evolve-v1.md diff --git a/memhub/MEMORY.md b/memhub/MEMORY.md index 4ce93c5..f5ed0cb 100644 --- a/memhub/MEMORY.md +++ b/memhub/MEMORY.md @@ -13,8 +13,8 @@ 接口先补 SDK。Runtime writers 停止后的 App 启动迁移可直接操作底层存储， 但不得把迁移专用方法加入公共 Client。 3. MemHub **不存 memory 实体**；**可以存 `memory_id`** 做组织/回执。原文进入 **SDK raw**。 -4. SDK raw 只保留滚动三天；一条 raw 对应一个完整 turn，只含真人 user 与 - assistant，tool 不持久化。提取后的高层 memory 不受该窗口影响。 +4. SDK raw 全部保留，任何模式（lite / pro / ultra）都不删除；一条 raw 对应一个 + 完整 turn，只含真人 user 与 assistant，tool 不持久化。 5. `source_raw_ids` 是 SDK 内部溯源关系，不是产品 Evidence 合同；记忆详情不提供 “原始依据”或按 raw ID 展开原文的入口。 diff --git a/memhub/apps/cli/src/cli-contract-fixes.test.ts b/memhub/apps/cli/src/cli-contract-fixes.test.ts index 53163c2..bd692e4 100644 --- a/memhub/apps/cli/src/cli-contract-fixes.test.ts +++ b/memhub/apps/cli/src/cli-contract-fixes.test.ts @@ -315,6 +315,8 @@ describe("catalog exit codes", () => { "models", "memories", "profile", + "settings", + "evolve", "graph-width", "host", "recall", diff --git a/memhub/apps/cli/src/cli.ts b/memhub/apps/cli/src/cli.ts index 7c7ca90..2c07f9d 100644 --- a/memhub/apps/cli/src/cli.ts +++ b/memhub/apps/cli/src/cli.ts @@ -188,6 +188,18 @@ import { removeManagedCliLink, rollbackManagedCliLink, } from "./managed-cli-link.js"; +import { + type EvolveDocument, + evolveExitCode, + type ModeDocument, + modeExitCode, + parseEvolveCommand, + parseSettingsCommand, + renderEvolveText, + renderModeText, + runEvolveCommand, + runModeCommand, +} from "./memory-mode-flow.js"; import { addMemory, DEFAULT_USER_ID, @@ -325,6 +337,8 @@ export interface RunControls { watchSignal?: AbortSignal; watchWait?: (milliseconds: number, signal: AbortSignal) => Promise; watchNow?: () => string; + /** `memhub evolve --wait` polls on this; tests inject an instant one. */ + sleep?: (milliseconds: number) => Promise; readStdin?: () => Promise; stdinIsTTY?: boolean; /** Injectable onboarding prompter so the wizard can be driven in tests @@ -2228,6 +2242,55 @@ async function runCommand( ); return outcome.state === "committed" ? EXIT_OK : EXIT_FAILED; } + if (named === "settings") { + const parsed = parseSettingsCommand(arguments_.slice(1)); + if (!parsed) return usage(); + const connected = await connectRuntime(environment); + if (!connected.ok) { + const document: ModeDocument = { + schema_version: 1, + product: PRODUCT, + command: "settings mode", + state: "unavailable", + code: connected.code, + }; + emit(asJson ? `${JSON.stringify(document)}\n` : renderModeText(document)); + return connected.exitCode; + } + const document = await runModeCommand( + parsed, + connected.reader, + connected.operations, + ); + emit(asJson ? `${JSON.stringify(document)}\n` : renderModeText(document)); + return modeExitCode(document); + } + if (named === "evolve") { + const parsed = parseEvolveCommand(arguments_.slice(1)); + if (!parsed) return usage(); + const connected = await connectRuntime(environment); + if (!connected.ok) { + const document: EvolveDocument = { + schema_version: 1, + product: PRODUCT, + command: "evolve", + state: "unavailable", + code: connected.code, + }; + emit( + asJson ? `${JSON.stringify(document)}\n` : renderEvolveText(document), + ); + return connected.exitCode; + } + const document = await runEvolveCommand(parsed, { + userId: userIdParameters(userId).user_id, + reader: connected.reader, + operations: connected.operations, + sleep: controls.sleep, + }); + emit(asJson ? `${JSON.stringify(document)}\n` : renderEvolveText(document)); + return evolveExitCode(document); + } if (named === "graph-width") { const parsed = parseGraphWidthCommand(arguments_.slice(1)); if (!parsed) return usage(); @@ -6814,7 +6877,13 @@ const OWN_DATA_DIR_COMMANDS = new Set([ /** Remove one `--data-dir ` from argv; null when it is * repeated, missing its value, or not an absolute path. */ /** Memory commands that honour the global `--user-id`. */ -const USER_ID_COMMANDS = new Set(["memories", "recall", "profile", "write"]); +const USER_ID_COMMANDS = new Set([ + "memories", + "recall", + "profile", + "write", + "evolve", +]); /** * Take `--user-id ` out of the arguments. The default user (`local`) is diff --git a/memhub/apps/cli/src/command-catalog.ts b/memhub/apps/cli/src/command-catalog.ts index 9bbb0aa..77258db 100644 --- a/memhub/apps/cli/src/command-catalog.ts +++ b/memhub/apps/cli/src/command-catalog.ts @@ -448,6 +448,43 @@ export const COMMANDS: readonly CommandDescriptor[] = [ ], exit_codes: RUNTIME_EXITS, }, + { + name: "settings", + summary: + "查看或切换记忆模式（lite / pro / ultra）。lite 不调用模型，只在原始对话里检索；pro 由模型提取记忆（System 1）；ultra 在 pro 之上每晚自动进化（System 2，使用知识图谱）。任何模式都保留全部原始对话。切换不迁移数据：已处理的不会重做，排队中的和新的按新模式处理；需 --yes，不加时只显示计划。环境变量 ONE_MEMORY_GRAPH_STORE 会强制 ultra。", + usage: "memhub settings mode [lite|pro|ultra] [--yes] [--json]", + options: [ + { + flag: "mode [lite|pro|ultra]", + summary: "不带模式只显示当前模式；带模式时先显示切换计划。", + }, + CONFIRM_OPTION, + JSON_OPTION, + ], + exit_codes: RUNTIME_EXITS, + }, + { + name: "evolve", + summary: + "立即为一个用户运行记忆进化（System 2，仅 ultra 模式）。在后台运行并返回运行 ID；--wait 等待结果。同一用户同时只运行一次；每晚的自动进化由 Runtime 按设置安排。", + usage: + "memhub evolve [--user-id ] [--agent-id ] [--max-batches ] [--wait] [--json]", + options: [ + USER_ID_OPTION, + { + flag: "--agent-id ", + summary: + "进化哪个 agent 的记忆，默认 all-apps（MemHub 写入记忆用的 agent）。", + }, + { + flag: "--max-batches ", + summary: "每轮最多处理的批次数（1–1000），默认由 SDK 决定。", + }, + { flag: "--wait", summary: "等待这次进化结束再返回。" }, + JSON_OPTION, + ], + exit_codes: RUNTIME_EXITS, + }, { name: "graph-width", summary: diff --git a/memhub/apps/cli/src/command-contract.test.ts b/memhub/apps/cli/src/command-contract.test.ts index 7c2508c..f3f7263 100644 --- a/memhub/apps/cli/src/command-contract.test.ts +++ b/memhub/apps/cli/src/command-contract.test.ts @@ -76,6 +76,8 @@ const COMMANDS = [ "models", "memories", "profile", + "settings", + "evolve", "graph-width", "host", "recall", diff --git a/memhub/apps/cli/src/core-client.ts b/memhub/apps/cli/src/core-client.ts index 52d64f6..81aa522 100644 --- a/memhub/apps/cli/src/core-client.ts +++ b/memhub/apps/cli/src/core-client.ts @@ -18,7 +18,12 @@ const READ_ONLY_PATHS = [ "/api/v2/memory-library", "/api/v2/user-profile", "/api/v2/user-profile/versions", + "/api/v1/settings/memory-mode", + "/api/v2/evolve", ] as const; +/** One evolve run's receipt (`memhub evolve --wait`). */ +const EVOLVE_RUN = + /^\/api\/v2\/evolve\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; const SDK_MEMORY_DETAIL = /^\/api\/v2\/memories\/[A-Za-z0-9][\w.-]{0,127}$/; /** Status of one `POST /api/v2/memories` addition, by its request id. */ const MEMORY_ADDITION = @@ -221,6 +226,7 @@ export function isReadOnlyPath(path: string): boolean { if (IMPORT_JOB.test(path)) return true; if (OPERATION_RECEIPT.test(path)) return true; if (MEMORY_ADDITION.test(path)) return true; + if (EVOLVE_RUN.test(path)) return true; if (!SDK_MEMORY_DETAIL.test(path)) return false; const segment = path.slice("/api/v2/memories/".length); return !(RESERVED_MEMORY_SEGMENTS as readonly string[]).includes(segment); diff --git a/memhub/apps/cli/src/core-operations.ts b/memhub/apps/cli/src/core-operations.ts index 867db38..035df49 100644 --- a/memhub/apps/cli/src/core-operations.ts +++ b/memhub/apps/cli/src/core-operations.ts @@ -26,6 +26,9 @@ const SOURCE_IMPORT_JOB_PATH = "/api/v1/import-jobs"; const IMPORT_JOB_ACTION_PATH = /^\/api\/v1\/import-jobs\/[A-Za-z0-9][\w.-]{0,127}\/(?:pause|resume|cancel)$/; const GRAPH_WIDTH_PATH = "/api/v1/graph-width"; +/** `memhub evolve` starts a run; `memhub settings mode --yes` sets the mode. */ +const EVOLVE_PATH = "/api/v2/evolve"; +const MEMORY_MODE_PATH = "/api/v1/settings/memory-mode"; const ONBOARDING_COMPLETE_PATH = "/api/v1/onboarding/complete"; const MAX_BYTES = 4 * 1024 * 1024; @@ -97,6 +100,7 @@ export function createCoreOperations(attachment: { !SOURCE_AUTHORIZE_PATH.test(path) && !SOURCE_REVOKE_PATH.test(path) && path !== GRAPH_WIDTH_PATH && + path !== EVOLVE_PATH && path !== ONBOARDING_COMPLETE_PATH && !MODEL_PROFILE_ACTIVATE.test(path) ) { @@ -132,7 +136,10 @@ export function createCoreOperations(attachment: { body: Record, signal?: AbortSignal, ) { - if (!SDK_MEMORY_PATH.test(path) || path.includes("?")) { + if ( + path !== MEMORY_MODE_PATH && + (!SDK_MEMORY_PATH.test(path) || path.includes("?")) + ) { throw new Error("core_operation_path_not_allowed"); } let response: { status: number; data: unknown }; diff --git a/memhub/apps/cli/src/memory-mode-cli.test.ts b/memhub/apps/cli/src/memory-mode-cli.test.ts new file mode 100644 index 0000000..1ef746b --- /dev/null +++ b/memhub/apps/cli/src/memory-mode-cli.test.ts @@ -0,0 +1,309 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { run } from "./cli.js"; +import { modeEffects, parseSettingsCommand } from "./memory-mode-flow.js"; +import { composeStatusReport, renderStatusText } from "./status-report.js"; + +const transport = vi.hoisted(() => ({ + get: vi.fn(), + post: vi.fn(), + put: vi.fn(), + patch: vi.fn(), + delete: vi.fn(), +})); +vi.mock("axios", () => ({ + default: { + create: vi.fn(() => transport), + get: vi.fn(), + post: vi.fn(), + isAxiosError: () => false, + }, +})); + +const runtime = vi.hoisted(() => ({ + inspectRuntimeStatus: vi.fn(), + discoverRuntimeAttachment: vi.fn(), +})); +vi.mock("@one-memory/runtime-control", async (importOriginal) => ({ + ...(await importOriginal()), + ...runtime, +})); + +const RUN_ID = "11111111-2222-4333-8444-555555555555"; + +function modeState(mode: string, configured = mode) { + return { + mode, + configured_mode: configured, + forced_by_env: false, + modes: ["lite", "pro", "ultra"], + graph_build: { state: "idle", users: {} }, + }; +} + +function receipt(state: string, extra: Record = {}) { + return { + run_id: RUN_ID, + user_id: "wife", + agent_id: "all-apps", + state, + trigger: "manual", + started_at: "2026-11-01T03:00:00Z", + rounds: state === "running" ? 0 : 2, + tasks_processed: state === "running" ? 0 : 5, + batch_failures: 0, + capped: false, + elapsed_ms: 10, + results: { schemas: ["never printed"] }, + ...extra, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + for (const method of Object.values(transport)) method.mockReset(); + runtime.inspectRuntimeStatus.mockResolvedValue({ + state: "running", + code: "runtime_verified", + instanceId: "instance-1", + host: "127.0.0.1", + port: 8765, + version: "1.2.3", + }); + runtime.discoverRuntimeAttachment.mockResolvedValue({ + instanceId: "instance-1", + apiToken: "synthetic-runtime-token", + host: "127.0.0.1", + port: 8765, + servicePid: 4242, + health: { + status: "ok", + service: "one-memory-core", + version: "1.2.3", + instance_id: "instance-1", + }, + }); + transport.get.mockImplementation(async (url: string) => + url === "/api/v1/settings/memory-mode" + ? { status: 200, data: modeState("pro") } + : { status: 404, data: {} }, + ); +}); + +describe("memhub settings mode", () => { + it("shows the current mode without changing it", async () => { + const result = await capturedRun(["settings", "mode", "--json"]); + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout).mode.mode).toBe("pro"); + expect(transport.put).not.toHaveBeenCalled(); + }); + + it("prints the plan and changes nothing without --yes", async () => { + const result = await capturedRun(["settings", "mode", "lite"]); + expect(result.code).toBe(0); + expect(transport.put).not.toHaveBeenCalled(); + expect(result.stdout).toContain("更高模式生成的记忆仍保存着"); + expect(result.stdout).toContain("memhub settings mode lite --yes"); + const json = JSON.parse( + (await capturedRun(["settings", "mode", "ultra", "--json"])).stdout, + ); + expect(json.state).toBe("preview"); + expect(json.applied).toBe(false); + expect(json.effects).toEqual(["existing_memories_added_to_graph"]); + }); + + it("changes the mode with --yes and reports what changed", async () => { + transport.put.mockResolvedValue({ + status: 200, + data: { + ...modeState("lite"), + previous_mode: "pro", + effects: ["higher_mode_memories_hidden"], + }, + }); + const result = await capturedRun(["settings", "mode", "lite", "--yes"]); + expect(result.code).toBe(0); + expect(transport.put).toHaveBeenCalledWith( + "/api/v1/settings/memory-mode", + { mode: "lite" }, + undefined, + ); + expect(result.stdout).toContain("已切换"); + expect(result.stdout).toContain("原始对话：全部保留"); + }); + + it("does nothing for the mode already set, and refuses bad input", async () => { + const same = await capturedRun(["settings", "mode", "pro", "--yes"]); + expect(same.code).toBe(0); + expect(same.stdout).toContain("已经是这个模式"); + expect(transport.put).not.toHaveBeenCalled(); + for (const bad of [ + ["settings"], + ["settings", "mode", "max"], + ["settings", "mode", "--yes"], + ["settings", "mode", "lite", "pro"], + ["settings", "language", "en"], + ]) { + expect((await capturedRun(bad)).code, bad.join(" ")).toBe(2); + } + }); + + it("plans the same effects Core reports", () => { + expect(modeEffects("lite", "pro")).toEqual([ + "earlier_raw_not_extracted", + "lite_search_stops", + ]); + expect(modeEffects("ultra", "lite")).toEqual([ + "higher_mode_memories_hidden", + "evolve_stopped", + ]); + expect(modeEffects("pro", "pro")).toEqual([]); + expect(parseSettingsCommand(["mode", "ultra", "--yes"])).toEqual({ + kind: "set", + mode: "ultra", + confirmed: true, + }); + }); +}); + +describe("memhub evolve", () => { + it("starts a run for the named user and returns at once", async () => { + transport.post.mockResolvedValue({ status: 202, data: receipt("running") }); + const result = await capturedRun([ + "evolve", + "--user-id", + "wife", + "--agent-id", + "all-apps", + "--max-batches", + "3", + "--json", + ]); + expect(result.code).toBe(0); + expect(transport.post).toHaveBeenCalledWith( + "/api/v2/evolve", + { user_id: "wife", agent_id: "all-apps", max_batches: 3 }, + undefined, + ); + const document = JSON.parse(result.stdout); + expect(document.state).toBe("running"); + expect(document.run.results).toBeUndefined(); + }); + + it("follows the run with --wait until it ends", async () => { + transport.post.mockResolvedValue({ status: 202, data: receipt("running") }); + const reads = [receipt("running"), receipt("succeeded")]; + transport.get.mockImplementation(async (url: string) => + url === `/api/v2/evolve/${RUN_ID}` + ? { status: 200, data: reads.shift() } + : { status: 404, data: {} }, + ); + const result = await capturedRun(["evolve", "--wait"], { + sleep: async () => undefined, + }); + expect(result.code).toBe(0); + expect(transport.post).toHaveBeenCalledWith( + "/api/v2/evolve", + {}, + undefined, + ); + expect(result.stdout).toContain("进化完成"); + expect(result.stdout).toContain("2 轮"); + }); + + it("names the mode when Core refuses outside ultra", async () => { + transport.post.mockResolvedValue({ + status: 409, + data: { detail: "memory_mode_not_ultra" }, + }); + const result = await capturedRun(["evolve"]); + expect(result.code).toBe(1); + expect(result.stdout).toContain("memhub settings mode ultra --yes"); + expect((await capturedRun(["evolve", "--max-batches", "0"])).code).toBe(2); + expect((await capturedRun(["evolve", "--now"])).code).toBe(2); + }); +}); + +describe("status pipeline", () => { + it("reports pipeline.mode and the last evolve from memory-processing", async () => { + const processing = { + pending: 0, + fresh_pending: 0, + backlog_pending: 0, + outside_window_pending: 0, + running: 0, + waiting_for_model: 0, + completed: 3, + failed: 0, + blocked_stages: [], + pipeline: { + mode: "ultra", + configured_mode: "ultra", + forced_by_env: false, + last_evolve_at: "2026-11-01T03:00:05+08:00", + graph_build: { state: "succeeded", users: {} }, + evolve: { + mode: "ultra", + schedule: { enabled: true, time: "03:00" }, + running: [], + users: { + local: { + last_run_at: "2026-11-01T03:00:05+08:00", + last_result: { state: "succeeded", tasks_processed: 4 }, + last_error: null, + }, + }, + }, + }, + }; + const report = await composeStatusReport({ + dataDir: "/tmp/isolated", + runtime: { state: "running", code: "ok", host: "127.0.0.1", port: 1 }, + background: null, + reader: (async (path: string) => + path === "/api/v1/memory-processing" + ? { state: "ok", data: processing } + : { state: "unavailable", code: "x" }) as never, + }); + expect(report.pipeline?.mode).toBe("ultra"); + expect(report.pipeline?.last_evolve_at).toBe("2026-11-01T03:00:05+08:00"); + expect(report.pipeline?.evolve_users.local?.state).toBe("succeeded"); + const text = renderStatusText(report, "MemHub"); + expect(text).toContain("记忆模式：ultra"); + expect(text).toContain("每日 03:00 自动进化"); + }); +}); + +async function capturedRun( + arguments_: string[], + controls: Parameters[2] = {}, +) { + let stdout = ""; + let stderr = ""; + const stdoutSpy = vi + .spyOn(process.stdout, "write") + .mockImplementation((chunk) => { + stdout += String(chunk); + return true; + }); + const stderrSpy = vi + .spyOn(process.stderr, "write") + .mockImplementation((chunk) => { + stderr += String(chunk); + return true; + }); + try { + return { + code: await run( + arguments_, + { ONE_MEMORY_DATA_DIR: "/synthetic/data" }, + controls, + ), + stdout, + stderr, + }; + } finally { + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + } +} diff --git a/memhub/apps/cli/src/memory-mode-flow.ts b/memhub/apps/cli/src/memory-mode-flow.ts new file mode 100644 index 0000000..47b9557 --- /dev/null +++ b/memhub/apps/cli/src/memory-mode-flow.ts @@ -0,0 +1,372 @@ +/** + * `memhub settings mode` and `memhub evolve`. + * + * The mode (lite / pro / ultra) decides what the SDK does with every + * conversation, and switching it migrates nothing: what one mode made is not + * shown by a lower one, and raw kept by lite is not extracted until it is + * imported again. So a change prints its plan first and needs `--yes`, like + * every other command whose effect a person cannot simply take back. + * + * `memhub evolve` starts System 2 (ultra only) and, with `--wait`, follows + * the run until it ends. + */ + +import type { CoreReader } from "./core-client.js"; +import type { CoreOperations, CorePutOperations } from "./core-operations.js"; +import { PRODUCT } from "./version.js"; + +export const MEMORY_MODES = ["lite", "pro", "ultra"] as const; +export type MemoryMode = (typeof MEMORY_MODES)[number]; + +export type ModeEffect = + | "earlier_raw_not_extracted" + | "higher_mode_memories_hidden" + | "existing_memories_added_to_graph" + | "evolve_stopped" + | "lite_search_stops" + | "graph_backfill_already_running"; + +const MODE_SUMMARY: Record = { + lite: "lite：不调用模型，保留全部原始对话，只在原始对话里检索", + pro: "pro：模型从对话中提取记忆（System 1）", + ultra: "ultra：pro + 每晚自动进化（System 2，使用知识图谱）", +}; + +const EFFECT_TEXT: Record = { + earlier_raw_not_extracted: + "lite 下已处理完的对话不会再被提取；只有尚未处理的和之后的新对话会由模型整理成记忆，检索立即改为记忆检索", + higher_mode_memories_hidden: + "更高模式生成的记忆仍保存着，但在这个模式下不会显示", + existing_memories_added_to_graph: + "已有记忆会在后台加入知识图谱（每个用户一次）", + evolve_stopped: "每晚的自动进化（System 2）停止", + lite_search_stops: "lite 只在原始对话里的检索立即停止，改为检索记忆", + graph_backfill_already_running: "知识图谱的补建已在进行中", +}; + +/** The same rule Core applies (`memory_mode_effects`), for the plan. */ +export function modeEffects( + previous: MemoryMode, + next: MemoryMode, +): ModeEffect[] { + if (previous === next) return []; + const rank: Record = { lite: 0, pro: 1, ultra: 2 }; + const effects: ModeEffect[] = []; + if (previous === "lite") { + effects.push("earlier_raw_not_extracted", "lite_search_stops"); + } + if (rank[next] < rank[previous]) effects.push("higher_mode_memories_hidden"); + if (previous === "ultra") effects.push("evolve_stopped"); + if (next === "ultra") effects.push("existing_memories_added_to_graph"); + return effects; +} + +export type ModeCommand = + | { kind: "show" } + | { kind: "set"; mode: MemoryMode; confirmed: boolean }; + +/** `settings mode [lite|pro|ultra] [--yes]` after `settings`; null = usage. */ +export function parseSettingsCommand( + values: readonly string[], +): ModeCommand | null { + const rest = values.filter((value) => value !== "--json"); + if (rest[0] !== "mode") return null; + const tail = rest.slice(1); + const confirmed = tail.includes("--yes"); + const positional = tail.filter((value) => value !== "--yes"); + if (tail.filter((value) => value === "--yes").length > 1) return null; + if (positional.length === 0) return confirmed ? null : { kind: "show" }; + if (positional.length !== 1) return null; + const mode = positional[0] as MemoryMode; + if (!(MEMORY_MODES as readonly string[]).includes(mode)) return null; + return { kind: "set", mode, confirmed }; +} + +export interface ModeState { + mode: MemoryMode; + configured_mode: MemoryMode; + forced_by_env: boolean; + graph_build?: { state?: string } | null; + previous_mode?: MemoryMode; + effects?: ModeEffect[]; +} + +function modeState(value: unknown): ModeState | null { + if (!value || typeof value !== "object") return null; + const item = value as Record; + if ( + !(MEMORY_MODES as readonly unknown[]).includes(item.mode) || + !(MEMORY_MODES as readonly unknown[]).includes(item.configured_mode) || + typeof item.forced_by_env !== "boolean" + ) { + return null; + } + return item as unknown as ModeState; +} + +export type ModeDocument = { + schema_version: 1; + product: string; + command: "settings mode"; +} & ( + | { state: "ok"; mode: ModeState } + | { + state: "preview"; + mode: ModeState; + target: MemoryMode; + effects: ModeEffect[]; + applied: false; + next_step: string; + } + | { state: "unchanged"; mode: ModeState } + | { state: "committed"; mode: ModeState; effects: ModeEffect[] } + | { state: "rejected" | "unknown" | "unavailable"; code: string } +); + +const BASE = { + schema_version: 1 as const, + product: PRODUCT, + command: "settings mode" as const, +}; + +export async function runModeCommand( + command: ModeCommand, + reader: CoreReader, + operations: CorePutOperations, +): Promise { + const read = await reader("/api/v1/settings/memory-mode"); + if (read.state !== "ok") { + return { ...BASE, state: "unavailable", code: read.code }; + } + const current = modeState(read.data); + if (current === null) { + return { ...BASE, state: "unavailable", code: "response_invalid" }; + } + if (command.kind === "show") return { ...BASE, state: "ok", mode: current }; + if (current.configured_mode === command.mode) { + return { ...BASE, state: "unchanged", mode: current }; + } + if (!command.confirmed) { + return { + ...BASE, + state: "preview", + mode: current, + target: command.mode, + effects: modeEffects(current.mode, command.mode), + applied: false, + next_step: `${PRODUCT} settings mode ${command.mode} --yes`, + }; + } + const result = await operations.put("/api/v1/settings/memory-mode", { + mode: command.mode, + }); + if (result.state !== "committed") { + return { ...BASE, state: result.state, code: result.code }; + } + const changed = modeState(result.data); + if (changed === null) { + return { ...BASE, state: "unknown", code: "response_invalid" }; + } + return { + ...BASE, + state: "committed", + mode: changed, + effects: changed.effects ?? [], + }; +} + +export function modeExitCode(document: ModeDocument): number { + return ["ok", "preview", "unchanged", "committed"].includes(document.state) + ? 0 + : 1; +} + +export function renderModeText(document: ModeDocument): string { + if ("code" in document) { + return `记忆模式未能${document.state === "unavailable" ? "读取" : "修改"}：${document.code}\n`; + } + const mode = document.mode; + const lines = [`当前模式：${MODE_SUMMARY[mode.mode]}`]; + if (mode.forced_by_env) { + lines.push( + `设置里是 ${mode.configured_mode}；环境变量 ONE_MEMORY_GRAPH_STORE 强制使用 ultra。`, + ); + } + lines.push("原始对话：全部保留（任何模式都不删除）"); + if (document.state === "unchanged") { + lines.push("已经是这个模式，未做改动。"); + } + if (document.state === "preview") { + lines.push(`将切换到：${MODE_SUMMARY[document.target]}`); + for (const effect of document.effects) + lines.push(`- ${EFFECT_TEXT[effect]}`); + lines.push(`未执行任何改动。确认后运行：${document.next_step}`); + } + if (document.state === "committed") { + lines.push("已切换。"); + for (const effect of document.effects) + lines.push(`- ${EFFECT_TEXT[effect]}`); + } + return `${lines.join("\n")}\n`; +} + +export type EvolveCommand = { + agentId?: string; + maxBatches?: number; + wait: boolean; +}; + +/** `evolve [--agent-id ] [--max-batches ] [--wait]`; null = usage. */ +export function parseEvolveCommand( + values: readonly string[], +): EvolveCommand | null { + const parsed: EvolveCommand = { wait: false }; + const rest = values.filter((value) => value !== "--json"); + for (let index = 0; index < rest.length; index += 1) { + const flag = rest[index]; + if (flag === "--wait" && !parsed.wait) { + parsed.wait = true; + } else if (flag === "--agent-id" && parsed.agentId === undefined) { + const value = rest[index + 1]; + if (!value || !/^[A-Za-z0-9._:-]{1,128}$/.test(value)) return null; + parsed.agentId = value; + index += 1; + } else if (flag === "--max-batches" && parsed.maxBatches === undefined) { + const value = Number(rest[index + 1]); + if (!Number.isInteger(value) || value < 1 || value > 1000) return null; + parsed.maxBatches = value; + index += 1; + } else { + return null; + } + } + return parsed; +} + +export type EvolveDocument = { + schema_version: 1; + product: string; + command: "evolve"; +} & ( + | { state: "running" | "succeeded" | "failed"; run: Record } + | { state: "rejected" | "unknown" | "unavailable"; code: string } +); + +const RUN_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +function evolveRun(value: unknown): Record | null { + if (!value || typeof value !== "object") return null; + const run = value as Record; + if ( + typeof run.run_id !== "string" || + !RUN_ID.test(run.run_id) || + !["running", "succeeded", "failed"].includes(String(run.state)) + ) { + return null; + } + return run; +} + +export async function runEvolveCommand( + command: EvolveCommand, + options: { + userId?: string; + reader: CoreReader; + operations: CoreOperations; + sleep?: (ms: number) => Promise; + pollMs?: number; + }, +): Promise { + const base = { + schema_version: 1 as const, + product: PRODUCT, + command: "evolve" as const, + }; + const body: Record = {}; + if (options.userId !== undefined) body.user_id = options.userId; + if (command.agentId !== undefined) body.agent_id = command.agentId; + if (command.maxBatches !== undefined) body.max_batches = command.maxBatches; + const started = await options.operations.post( + "/api/v2/evolve", + body, + ); + if (started.state !== "committed") { + return { ...base, state: started.state, code: started.code }; + } + let run = evolveRun(started.data); + if (run === null) + return { ...base, state: "unknown", code: "response_invalid" }; + const sleep = + options.sleep ?? + ((ms: number) => new Promise((done) => setTimeout(done, ms))); + while (command.wait && run.state === "running") { + await sleep(options.pollMs ?? 2_000); + const read = await options.reader( + `/api/v2/evolve/${run.run_id}`, + options.userId === undefined ? {} : { user_id: options.userId }, + ); + if (read.state !== "ok") { + return { ...base, state: "unavailable", code: read.code }; + } + const next = evolveRun(read.data); + if (next === null) + return { ...base, state: "unknown", code: "response_invalid" }; + run = next; + } + return { + ...base, + state: run.state as "running" | "succeeded" | "failed", + run: projectRun(run), + }; +} + +/** Counts and codes; SDK result bodies stay in the Core receipt. */ +function projectRun(run: Record): Record { + const keep = [ + "run_id", + "user_id", + "agent_id", + "state", + "trigger", + "started_at", + "finished_at", + "rounds", + "tasks_processed", + "batch_failures", + "cycle_status", + "should_continue", + "continuation_reason", + "remaining_clusters", + "remaining_actionable_clusters", + "remaining_fresh_facts", + "capped", + "elapsed_ms", + "error_code", + "failure_kind", + ]; + return Object.fromEntries( + keep.filter((key) => key in run).map((key) => [key, run[key]]), + ); +} + +export function evolveExitCode(document: EvolveDocument): number { + return document.state === "running" || document.state === "succeeded" ? 0 : 1; +} + +export function renderEvolveText(document: EvolveDocument): string { + if ("code" in document) { + return document.code === "memory_mode_not_ultra" + ? "进化只在 ultra 模式下可用。切换：memhub settings mode ultra --yes\n" + : `进化未能开始：${document.code}\n`; + } + const run = document.run; + if (document.state === "running") { + return `进化已开始（${run.run_id}），在后台运行；加 --wait 等待结果。\n`; + } + if (document.state === "failed") { + return `进化失败（${run.run_id}）：${String(run.error_code ?? "unknown")}\n`; + } + return `进化完成（${run.run_id}）：${run.rounds} 轮，处理 ${run.tasks_processed} 项${ + run.cycle_status ? `，周期 ${run.cycle_status}` : "" + }${run.capped ? "，已到轮数上限，可再次运行继续" : ""}\n`; +} diff --git a/memhub/apps/cli/src/source-progress-report.ts b/memhub/apps/cli/src/source-progress-report.ts index 5fe95d0..1c87555 100644 --- a/memhub/apps/cli/src/source-progress-report.ts +++ b/memhub/apps/cli/src/source-progress-report.ts @@ -14,6 +14,11 @@ import { validateProcessing, validateSources, } from "./core-responses.js"; +import { + describePipeline, + type PipelineSummary, + pipelineLine, +} from "./status-report.js"; import { PRODUCT } from "./version.js"; type ReadState = @@ -84,6 +89,8 @@ export interface ProgressDocument { summary_state: ReadState; summary: Record | null; }; + /** The SDK mode and last evolve per user; absent on an older Core. */ + pipeline?: PipelineSummary | null; } export async function readSources( @@ -283,6 +290,10 @@ export async function readProgress( summary: processing.state === "ok" ? projectProcessing(processing.data) : null, }, + pipeline: + processingRead.state === "ok" + ? describePipeline(processingRead.data) + : null, }; } @@ -353,6 +364,18 @@ export function renderProgressText(document: ProgressDocument): string { ? `记忆整理：待处理 ${document.generation.summary.pending} · 进行中 ${document.generation.summary.running} · 等待模型 ${document.generation.summary.waiting_for_model} · 失败 ${document.generation.summary.failed}` : "记忆整理：暂时无法读取", ); + if (document.pipeline) { + lines.push(`记忆模式：${pipelineLine(document.pipeline)}`); + for (const [userId, user] of Object.entries( + document.pipeline.evolve_users, + )) { + lines.push( + ` 进化 ${userId}：${user.last_run_at ?? "尚未进化"}${ + user.last_error ? `（失败：${user.last_error}）` : "" + }`, + ); + } + } if (document.imports.output_truncated) { lines.push("这里只显示汇总；完整记录请使用 --json。"); } diff --git a/memhub/apps/cli/src/status-report.ts b/memhub/apps/cli/src/status-report.ts index ca2fdc3..4833f33 100644 --- a/memhub/apps/cli/src/status-report.ts +++ b/memhub/apps/cli/src/status-report.ts @@ -57,6 +57,9 @@ export interface StatusDocument { broker?: BackgroundHealth["broker"]; }; sections: Record; + /** R10: the SDK mode and System 2 state; null when the Runtime is not read + * or is older than the setting. */ + pipeline?: PipelineSummary | null; summary: { state: "ok" | "partial"; degraded: string[]; @@ -64,6 +67,88 @@ export interface StatusDocument { }; } +export interface PipelineSummary { + mode: "lite" | "pro" | "ultra"; + configured_mode: "lite" | "pro" | "ultra"; + forced_by_env: boolean; + last_evolve_at: string | null; + evolve_schedule: { enabled: boolean; time: string } | null; + evolve_running: number; + evolve_users: Record< + string, + { + last_run_at: string | null; + last_error: string | null; + state: string | null; + } + >; + graph_build: string | null; +} + +const MODES = new Set(["lite", "pro", "ultra"]); + +/** Projection of `pipeline` in `/api/v1/memory-processing`; null if absent or malformed. */ +export function describePipeline(value: unknown): PipelineSummary | null { + if (!value || typeof value !== "object") return null; + const pipeline = (value as { pipeline?: unknown }).pipeline; + if (!pipeline || typeof pipeline !== "object") return null; + const item = pipeline as Record; + if ( + !MODES.has(String(item.mode)) || + !MODES.has(String(item.configured_mode)) + ) { + return null; + } + const evolve = + item.evolve && typeof item.evolve === "object" + ? (item.evolve as Record) + : {}; + const schedule = + evolve.schedule && typeof evolve.schedule === "object" + ? (evolve.schedule as Record) + : null; + const users: PipelineSummary["evolve_users"] = {}; + if (evolve.users && typeof evolve.users === "object") { + for (const [userId, raw] of Object.entries( + evolve.users as Record, + )) { + const user = + raw && typeof raw === "object" ? (raw as Record) : {}; + const result = + user.last_result && typeof user.last_result === "object" + ? (user.last_result as Record) + : {}; + users[userId] = { + last_run_at: + typeof user.last_run_at === "string" ? user.last_run_at : null, + last_error: + typeof user.last_error === "string" ? user.last_error : null, + state: typeof result.state === "string" ? result.state : null, + }; + } + } + const graph = + item.graph_build && typeof item.graph_build === "object" + ? (item.graph_build as Record) + : {}; + return { + mode: item.mode as PipelineSummary["mode"], + configured_mode: item.configured_mode as PipelineSummary["mode"], + forced_by_env: item.forced_by_env === true, + last_evolve_at: + typeof item.last_evolve_at === "string" ? item.last_evolve_at : null, + evolve_schedule: + schedule && + typeof schedule.enabled === "boolean" && + typeof schedule.time === "string" + ? { enabled: schedule.enabled, time: schedule.time } + : null, + evolve_running: Array.isArray(evolve.running) ? evolve.running.length : 0, + evolve_users: users, + graph_build: typeof graph.state === "string" ? graph.state : null, + }; +} + const SECTION_NAMES = [ "sources", "sync", @@ -97,6 +182,7 @@ export async function composeStatusReport(options: { attachFailure?: string; }): Promise { const sections: Record = {}; + let pipeline: PipelineSummary | null = null; if (options.reader === null) { const state: SectionState = options.attachFailure === undefined ? "not_checked" : "unavailable"; @@ -118,6 +204,8 @@ export async function composeStatusReport(options: { validateProcessing, describeProcessing, ); + pipeline = + processing.state === "ok" ? describePipeline(processing.data) : null; sections.model = section(model, validateModelProfiles, describeModel); sections.memories = section( memories, @@ -169,6 +257,7 @@ export async function composeStatusReport(options: { }, bridge, sections, + pipeline, summary: { state: degraded.length === 0 && notChecked.length === 0 ? "ok" : "partial", @@ -205,6 +294,7 @@ export function renderStatusText( `${SECTION_LABELS[name]}：${sectionLine(report.sections[name])}`, ); } + if (report.pipeline) lines.push(`记忆模式：${pipelineLine(report.pipeline)}`); if (report.summary.state === "partial") { const notes = [ report.summary.degraded.length > 0 @@ -220,6 +310,28 @@ export function renderStatusText( return lines.join("\n"); } +export function pipelineLine(pipeline: PipelineSummary): string { + const parts: string[] = [pipeline.mode]; + if (pipeline.forced_by_env) + parts.push("（ONE_MEMORY_GRAPH_STORE 强制 ultra）"); + if (pipeline.mode === "ultra") { + const schedule = pipeline.evolve_schedule; + parts.push( + schedule?.enabled + ? `，每日 ${schedule.time} 自动进化` + : "，自动进化已关闭", + ); + parts.push( + pipeline.last_evolve_at + ? `，上次进化 ${pipeline.last_evolve_at}` + : "，尚未进化", + ); + if (pipeline.evolve_running > 0) + parts.push(`，进行中 ${pipeline.evolve_running}`); + } + return parts.join(""); +} + function runtimeLine(report: StatusDocument): string { if (report.runtime.state !== "running") { return report.runtime.state === "stopped" ? "未运行" : "暂时无法确认"; diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py index 8275507..ed959fa 100644 --- a/memhub/apps/core/src/one_memory/api.py +++ b/memhub/apps/core/src/one_memory/api.py @@ -1,5 +1,6 @@ import asyncio import hashlib +import importlib.util import json import logging import os @@ -83,7 +84,12 @@ from one_memory.external_memory_access import ( ) from one_memory.extraction_settings import ExtractionSettings, ExtractionSettingsStore from one_memory.file_imports import FileImportStore, file_import_router -from one_memory.hy_graph_admin import GraphWidthDecision, GraphWidthMismatch +from one_memory.hy_graph_admin import ( + GraphWidthDecision, + GraphWidthMismatch, + effective_memory_mode, + graph_store_enabled, +) from one_memory.hy_runtime_config import ( HY_MEMORY_READER, DiagnosticLogLevel, @@ -176,6 +182,16 @@ from one_memory.memory_additions import ( MemoryAdditionStore, memory_add_router, ) +from one_memory.memory_evolution import ( + EvolveModeError, + EvolveRunReceipt, + EvolveScheduler, + EvolveService, + EvolveStateStore, + GraphBuildState, + MemoryGraphBackfill, + UserEvolveState, +) from one_memory.memory_governance import ( MemoryGovernanceStore, ) @@ -183,7 +199,12 @@ from one_memory.memory_qa import MemoryAnswerError from one_memory.memory_queue import ( ExtractionWindowPreset, ) -from one_memory.memory_server.config_state import ConfigStore +from one_memory.memory_server.config_state import ( + MEMORY_MODES, + ConfigStore, + EvolveSchedule, + MemoryMode, +) from one_memory.memory_tasks import ( NON_RETRYABLE_POST_WRITE_CODES, MemoryTaskStatus, @@ -629,6 +650,119 @@ class MemoryProcessingSummary(BaseModel): active_queue: int = 0 in_flight_count: int = 0 metrics: ProcessingMetrics = Field(default_factory=ProcessingMetrics) + #: R10: the SDK mode and System 2 state. Optional so older readers and + #: test doubles stay valid; Core always sends it. + pipeline: "PipelineStatus | None" = None + + +class MemoryModeState(BaseModel): + """`GET /api/v1/settings/memory-mode`.""" + + model_config = ConfigDict(extra="forbid") + + #: What SDK clients are built with now. + mode: MemoryMode + #: The saved setting (default pro). + configured_mode: MemoryMode + #: `ONE_MEMORY_GRAPH_STORE` (dev) forces ultra over the setting. + forced_by_env: bool + modes: tuple[MemoryMode, ...] = MEMORY_MODES + graph_build: GraphBuildState + + +class MemoryModeUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: MemoryMode + + +MemoryModeEffect = Literal[ + "earlier_raw_not_extracted", + "higher_mode_memories_hidden", + "existing_memories_added_to_graph", + "evolve_stopped", + "lite_search_stops", + "graph_backfill_already_running", +] + + +class MemoryModeChange(MemoryModeState): + """`PUT /api/v1/settings/memory-mode`: the new state and what it changes.""" + + previous_mode: MemoryMode + effects: tuple[MemoryModeEffect, ...] = () + + +class EvolveScheduleUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + enabled: StrictBool | None = None + time: str | None = Field(default=None, pattern=r"^(?:[01][0-9]|2[0-3]):[0-5][0-9]$") + + +class EvolveRequest(BaseModel): + """`POST /api/v2/evolve`.""" + + model_config = ConfigDict(extra="forbid") + + user_id: UserId = DEFAULT_USER_ID + #: Default `all-apps`, the agent MemHub writes every memory under. + agent_id: str | None = Field(default=None, min_length=1, max_length=128) + max_batches: int | None = Field(default=None, ge=1, le=1_000) + + +class EvolveOverview(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: MemoryMode + schedule: EvolveSchedule + users: dict[str, UserEvolveState] + running: tuple[EvolveRunReceipt, ...] = () + + +class PipelineStatus(BaseModel): + """`pipeline` in `/api/v1/memory-processing` and `memhub status --json`.""" + + model_config = ConfigDict(extra="forbid") + + mode: MemoryMode + configured_mode: MemoryMode + forced_by_env: bool + evolve: EvolveOverview + #: Newest finished run across users, for a one-line status. + last_evolve_at: datetime | None = None + graph_build: GraphBuildState + + +def memory_mode_effects( + previous: str, + new: str, + *, + graph_backfill: bool | None = None, +) -> tuple[MemoryModeEffect, ...]: + """What a mode switch changes; the SDK migrates nothing on its own. + + `graph_backfill`: True when this switch started the one-time backfill, + False when one was already running, None when not asked (no switch). + """ + if previous == new: + return () + rank = {"lite": 0, "pro": 1, "ultra": 2} + effects: list[MemoryModeEffect] = [] + if previous == "lite": + effects.extend(("earlier_raw_not_extracted", "lite_search_stops")) + if rank[new] < rank[previous]: + effects.append("higher_mode_memories_hidden") + if previous == "ultra": + effects.append("evolve_stopped") + if new == "ultra" and graph_backfill is True: + effects.append("existing_memories_added_to_graph") + if new == "ultra" and graph_backfill is False: + effects.append("graph_backfill_already_running") + return tuple(effects) + + +MemoryProcessingSummary.model_rebuild() class RetryUnknownMemoryTasksRequest(BaseModel): @@ -983,6 +1117,7 @@ def create_app( source_id: asyncio.Lock() for source_id in app.state.source_catalog } app.state.sedimentation_lock = asyncio.Lock() + app.state.memory_mode_lock = asyncio.Lock() _follow_workbuddy_home(app, runtime_settings.source_home) app.state.source_detection = {} @@ -1084,6 +1219,7 @@ def create_app( ) ), sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(), + memory_mode=lambda: _memory_mode(app), ) app.state.sdk_memory_runtime = SdkMemoryRuntime( coordinator=app.state.sdk_write_coordinator, @@ -1091,7 +1227,22 @@ def create_app( write_queue=memory_write_queue, extraction_settings=app.state.extraction_settings_store, data_dir=runtime_settings.data_dir, - user_ids=lambda: _known_user_ids(app), + memory_mode=lambda: _memory_mode(app), + ) + app.state.evolve_state_store = EvolveStateStore(runtime_settings.data_dir) + app.state.evolve_service = EvolveService( + coordinator=app.state.sdk_write_coordinator, + state=app.state.evolve_state_store, + memory_mode=lambda: _memory_mode(app), + ) + app.state.evolve_scheduler = EvolveScheduler( + service=app.state.evolve_service, + schedule=lambda: _config_store(app).evolve_schedule(), + users_with_memories=lambda: _users_with_memories(app), + ) + app.state.memory_graph_backfill = MemoryGraphBackfill( + coordinator=app.state.sdk_write_coordinator, + state=app.state.evolve_state_store, ) # C2 owns the single production budget (handle 5s + reserve 250ms, # total 5.25s, lease 10s, renew 1s, claim 250ms). The composition @@ -1427,8 +1578,9 @@ def create_app( name="one-memory-durable-write-drain", ) - async def start_raw_retention() -> None: - app.state.sdk_memory_runtime.start_raw_retention() + async def start_evolve_schedule() -> None: + # Idle outside ultra: each tick asks the mode first. + app.state.evolve_scheduler.start() async def replay_pending_dispatch() -> None: # A file's Evidence and its pending-dispatch row commit atomically, @@ -1509,7 +1661,7 @@ def create_app( await step("sdk_metadata_repair", repair_sdk_metadata) await step("write_drain", start_write_drain) if sdk: - await step("raw_retention", start_raw_retention) + await step("evolve_schedule", start_evolve_schedule) else: await step("pending_dispatch_replay", replay_pending_dispatch) await step("import_resume", resume_imports) @@ -1606,6 +1758,9 @@ def create_app( if app.state.memory_write_drain_task is not None: with suppress(asyncio.CancelledError): await app.state.memory_write_drain_task + await app.state.evolve_scheduler.shutdown() + await app.state.evolve_service.shutdown() + await app.state.memory_graph_backfill.shutdown() await app.state.sdk_memory_runtime.shutdown() await app.state.sdk_write_coordinator.close() if app.state.sdk_client_handle is not None: @@ -1635,6 +1790,7 @@ def create_app( version=__version__, lifespan=lifespan, ) + app.state.memory_mode = lambda: _memory_mode(app) app.state.service_levels = ServiceLevelScheduler( { ServiceResource.IMPORT: ResourceLimit( @@ -4079,8 +4235,148 @@ def create_app( active_queue=active_queue, in_flight_count=in_flight_count, metrics=ProcessingMetrics.model_validate(store.processing_metrics()), + pipeline=_pipeline_status(app), ) + @app.get( + "/api/v1/settings/memory-mode", + response_model=MemoryModeState, + dependencies=strict, + ) + def get_memory_mode() -> MemoryModeState: + """The SDK mode (lite / pro / ultra): configured, effective, and why.""" + return _memory_mode_state(app) + + @app.put( + "/api/v1/settings/memory-mode", + response_model=MemoryModeChange, + dependencies=strict, + ) + async def put_memory_mode(request: MemoryModeUpdate) -> MemoryModeChange: + """Choose the SDK mode. Nothing is migrated; `effects` says what changes. + + The mode is read when a queued job runs, not when it was queued: a + job already done (in any mode) stays done and is never reprocessed; + jobs still waiting, and new input, run in the new mode. A batch in + flight finishes in the old mode. Search follows the new mode at once. + lite → pro/ultra: what lite stored stays raw, and lite's raw-only + search stops. Down to a lower mode: memories the higher mode made stay + stored but are not shown. → ultra: every user's existing facts are + added to the graph once, in the background (`graph_build`). No raw is + ever deleted, in any mode. See docs/contracts/memory-mode-evolve-v1.md. + """ + if request.mode == "ultra" and importlib.util.find_spec("kuzu") is None: + raise HTTPException(status_code=409, detail="graph_store_unavailable") + store = _config_store(app) + async with app.state.memory_mode_lock: + previous_effective = _memory_mode(app) + previous = store.memory_mode() + if previous != request.mode or "memory_mode" not in store.document().model_fields_set: + await asyncio.to_thread(store.set_memory_mode, request.mode) + effective = _memory_mode(app) + backfill: bool | None = None + if effective != previous_effective: + # Reads rebuild on their next request (`ActiveProfileReadHandle`). + await app.state.sdk_write_coordinator.retire_stale_modes() + if effective == "ultra": + backfill = not app.state.memory_graph_backfill.running + if backfill: + users = await asyncio.to_thread(_known_user_ids, app) + app.state.memory_graph_backfill.start(users) + diagnostic_event( + level="info", + component="memory.mode", + event="memory_mode.changed", + outcome="succeeded", + details={"mode": effective, "previous_mode": previous_effective}, + ) + return MemoryModeChange( + **_memory_mode_state(app).model_dump(), + previous_mode=previous_effective, + effects=memory_mode_effects(previous_effective, effective, graph_backfill=backfill), + ) + + @app.get( + "/api/v1/settings/evolve-schedule", + response_model=EvolveSchedule, + dependencies=strict, + ) + def get_evolve_schedule() -> EvolveSchedule: + return _config_store(app).evolve_schedule() + + @app.put( + "/api/v1/settings/evolve-schedule", + response_model=EvolveSchedule, + dependencies=strict, + ) + async def put_evolve_schedule(request: EvolveScheduleUpdate) -> EvolveSchedule: + """Turn the nightly evolve on or off, or move it (local `HH:MM`).""" + store = _config_store(app) + current = store.evolve_schedule() + updated = EvolveSchedule( + enabled=current.enabled if request.enabled is None else request.enabled, + time=current.time if request.time is None else request.time, + ) + await asyncio.to_thread(store.set_evolve_schedule, updated) + return updated + + @app.post( + "/api/v2/evolve", + response_model=EvolveRunReceipt, + status_code=202, + dependencies=strict, + ) + async def start_evolve(request: EvolveRequest) -> EvolveRunReceipt: + """Run System 2 for one user, in the background; read the run by its id. + + Ultra only (409 `memory_mode_not_ultra`). One run per user: asking + while one runs answers that run. The run waits for the user's write + in flight and holds their next writes until it ends, and repeats + while the SDK says there is more to do (up to 20 rounds). + """ + try: + return _evolve_service(app).start( + request.user_id, + agent_id=request.agent_id, + max_batches=request.max_batches, + ) + except EvolveModeError: + raise HTTPException(status_code=409, detail="memory_mode_not_ultra") from None + + @app.get( + "/api/v2/evolve", + response_model=EvolveOverview, + dependencies=strict, + ) + def evolve_overview( + user_id: str | None = Query(default=None, pattern=USER_ID_PATTERN), + ) -> EvolveOverview: + """Last run per user and the runs in progress; one user's with `user_id`.""" + overview = _evolve_overview(app) + if user_id is None: + return overview + return overview.model_copy( + update={ + "users": {key: value for key, value in overview.users.items() if key == user_id}, + "running": tuple(run for run in overview.running if run.user_id == user_id), + } + ) + + @app.get( + "/api/v2/evolve/{run_id}", + response_model=EvolveRunReceipt, + dependencies=strict, + ) + def get_evolve_run( + run_id: str, + user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN), + ) -> EvolveRunReceipt: + """One run; another user's run answers exactly like a missing one.""" + receipt = _evolve_service(app).get(run_id) + if receipt is None or receipt.user_id != user_id: + raise HTTPException(status_code=404, detail="evolve_run_not_found") + return receipt + @app.post( "/api/v1/memory-processing/retry-unknown", response_model=RetryUnknownMemoryTasksReceipt, @@ -4413,14 +4709,50 @@ def _other_user_lanes( def _known_user_ids(app: FastAPI) -> tuple[str, ...]: - """`local` and every user holding a grant: the users whose data Core keeps.""" + """Every user whose data Core keeps, from Core's own records. + + The SDK has no "list users": `local`, every user holding a grant (a + Hermes profile), and every user whose turns or notes passed the write + queue (`POST /api/v2/memories` or `/api/v2/memory-writes` with a user). + """ users = [DEFAULT_USER_ID] for receipt, _root in _grant_store(app).grants(): if receipt.user_id not in users: users.append(receipt.user_id) + queue = getattr(app.state, "memory_write_queue", None) + if queue is not None: + with main_database(queue.database_path).read() as database: + for (source_key,) in database.execute( + "SELECT DISTINCT source_key FROM memory_write_jobs" + ): + user_id = split_lane(str(source_key))[0] + if user_id not in users: + users.append(user_id) return tuple(users) +async def _users_with_memories(app: FastAPI) -> tuple[str, ...]: + """Known users the SDK holds any memory for: the nightly evolve's users.""" + users = await asyncio.to_thread(_known_user_ids, app) + client = await _sdk_client_handle(app).client() + found: list[str] = [] + for user_id in users: + try: + stats = await asyncio.to_thread(client.memory_stats, user_id=user_id) + except Exception: + continue + scenes = stats.get("scenes") if isinstance(stats, dict) else None + total = stats.get("total") if isinstance(stats, dict) else None + facts = sum( + int(scene.get("facts") or 0) + for scene in (scenes or {}).values() + if isinstance(scene, dict) + ) + if (isinstance(total, int) and total > 0) or facts > 0: + found.append(user_id) + return tuple(found) + + def _granted_roots( source_ids: Iterable[str], grant_store: SourceGrantStore, @@ -5161,7 +5493,13 @@ def _sdk_memory_runtime(app: FastAPI) -> SdkMemoryRuntime: async def _recall_model_ready(app: FastAPI) -> bool: - """A generation model is configured and usable, so memories can exist.""" + """A generation model is configured and usable, so memories can exist. + + In lite nothing is extracted whatever model is configured: recall reads + the raw conversations. + """ + if _memory_mode(app) == "lite": + return False try: profile = await asyncio.to_thread(_model_profile_store(app).active, _secret_store(app)) except Exception: @@ -5310,16 +5648,68 @@ def _sdk_client_handle(app: FastAPI) -> SdkClientHandle: handle = cast(SdkClientHandle | None, app.state.sdk_client_handle) if handle is None: # Reads open the embedding space writes fill: the active profile's, - # the built-in one only when none is active (#242). + # the built-in one only when none is active (#242). A profile or a + # memory-mode change rebuilds the client. handle = ActiveProfileReadHandle( profiles=_model_profile_store(app), secrets=_secret_store(app), data_dir=cast(Settings, app.state.settings).data_dir, + memory_mode=lambda: _memory_mode(app), ) app.state.sdk_client_handle = handle return handle +def _memory_mode(app: FastAPI) -> MemoryMode: + """The mode SDK clients are built with: the setting, or ultra forced by env.""" + store = getattr(app.state, "config_store", None) + configured = store.memory_mode() if store is not None else "pro" + return cast(MemoryMode, effective_memory_mode(configured)) + + +def _memory_mode_state(app: FastAPI) -> MemoryModeState: + mode = _memory_mode(app) + return MemoryModeState( + mode=mode, + configured_mode=_config_store(app).memory_mode(), + forced_by_env=graph_store_enabled(), + graph_build=app.state.evolve_state_store.graph_build(), + ) + + +def _evolve_service(app: FastAPI) -> EvolveService: + return cast(EvolveService, app.state.evolve_service) + + +def _evolve_overview(app: FastAPI) -> EvolveOverview: + document = app.state.evolve_state_store.document() + return EvolveOverview( + mode=_memory_mode(app), + schedule=_config_store(app).evolve_schedule(), + users=document.users, + running=app.state.evolve_service.running(), + ) + + +def _pipeline_status(app: FastAPI) -> PipelineStatus | None: + """R10's `pipeline`, or None before the evolve state exists (legacy tests).""" + if getattr(app.state, "evolve_state_store", None) is None: + return None + state = _memory_mode_state(app) + overview = _evolve_overview(app) + finished = [ + user.last_run_at for user in overview.users.values() if user.last_run_at is not None + ] + return PipelineStatus( + mode=state.mode, + configured_mode=state.configured_mode, + forced_by_env=state.forced_by_env, + evolve=overview, + last_evolve_at=max(finished, default=None), + graph_build=state.graph_build, + ) + + def _log_skipped_sdk_memories(count: int) -> None: if count: logger.warning("sdk_memory_invalid_rows_skipped count=%d", count) @@ -5566,16 +5956,29 @@ def _source_writes_on(app: FastAPI, source_id: str) -> bool: def _sedimentation_claim_enabled(app: FastAPI, source_id: str) -> bool: - """Claim queued model work only when policy, source and model all admit it.""" + """Claim queued model work only when policy, source and model all admit it. + + lite calls no model, so it needs no active generation profile: the + built-in local embedder stores the raw turns. A user evolve is running + for waits until it is done. + """ return ( _config_store(app).sedimentation_enabled() - and _model_profile_store(app).active_identity() is not None + and (_memory_mode(app) == "lite" or _model_profile_store(app).active_identity() is not None) and _history_import_gate_released(app) and _source_writes_on(app, source_id) + and not _user_evolving(app, split_lane(source_id)[0]) ) +def _user_evolving(app: FastAPI, user_id: str) -> bool: + """Whether evolve holds this user's writes right now.""" + coordinator = getattr(app.state, "sdk_write_coordinator", None) + evolving = getattr(coordinator, "evolving", None) + return bool(callable(evolving) and evolving(user_id)) + + def _history_import_gate_released(app: FastAPI) -> bool: """Hold first sedimentation until every accepted history import is terminal.""" diff --git a/memhub/apps/core/src/one_memory/context_routes.py b/memhub/apps/core/src/one_memory/context_routes.py index 3084d7a..e6b9ab0 100644 --- a/memhub/apps/core/src/one_memory/context_routes.py +++ b/memhub/apps/core/src/one_memory/context_routes.py @@ -110,6 +110,8 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter: client = await client_handle.client() except Exception: raise HTTPException(503, "sdk_context_unavailable") from None + memory_mode = getattr(request.app.state, "memory_mode", None) + lite = callable(memory_mode) and memory_mode() == "lite" async def raw_lane() -> tuple[SdkRawContextItem, ...]: if "raw" not in value.kinds or not authority.raw_allowed: @@ -155,7 +157,9 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter: ) async def memory_lane() -> tuple[SdkMemoryContextItem, ...]: - if "memory" not in value.kinds or not authority.memory_allowed: + # lite extracts nothing: its search answers raw turns, which the + # raw lane already reads under the raw authority. + if "memory" not in value.kinds or not authority.memory_allowed or lite: return () overrides = value.search_overrides() raw = await asyncio.to_thread( diff --git a/memhub/apps/core/src/one_memory/hy_graph_admin.py b/memhub/apps/core/src/one_memory/hy_graph_admin.py index 0af5f21..7e1c169 100644 --- a/memhub/apps/core/src/one_memory/hy_graph_admin.py +++ b/memhub/apps/core/src/one_memory/hy_graph_admin.py @@ -71,6 +71,16 @@ def graph_store_enabled() -> bool: return raw != "none" +def effective_memory_mode(configured: str = "pro") -> str: + """The SDK mode clients are built with: the setting, unless dev forces ultra. + + `ONE_MEMORY_GRAPH_STORE` (any value but empty or `none`) predates the + setting and keeps working for development: it forces ultra whatever the + configured mode. Without it the configured mode (default pro) applies. + """ + return "ultra" if graph_store_enabled() else configured + + def inspect_graph_width_mismatch(config: Any) -> GraphWidthMismatch | None: """Return a mismatch without renaming or deleting anything.""" db_path = Path(config.graph_store.db_path) @@ -83,11 +93,12 @@ def inspect_profile_graph_width( *, api_key: str | None, data_dir: Path, + mode: str | None = None, ) -> GraphWidthMismatch | None: """Inspect the graph path and width selected by the active profile.""" del api_key return _inspect_graph_width( - _profile_graph_path(data_dir), + _profile_graph_path(data_dir, mode), _profile_embedding_width(profile), ) @@ -98,11 +109,12 @@ def decide_profile_graph_width( api_key: str | None, data_dir: Path, decision: GraphWidthDecision, + mode: str | None = None, ) -> GraphWidthOutcome: """Apply the user's decision to the active profile's graph.""" del api_key config = _GraphAdminConfig( - graph_store=_GraphStoreConfig(str(_profile_graph_path(data_dir))), + graph_store=_GraphStoreConfig(str(_profile_graph_path(data_dir, mode))), embedder=_EmbeddingWidthConfig(_profile_embedding_width(profile)), ) return apply_graph_width_decision( @@ -240,8 +252,9 @@ def _inspect_graph_width( ) -def _profile_graph_path(data_dir: Path) -> Path: - name = "graph" if graph_store_enabled() else "graph-disabled" +def _profile_graph_path(data_dir: Path, mode: str | None = None) -> Path: + ultra = mode == "ultra" if mode is not None else graph_store_enabled() + name = "graph" if ultra else "graph-disabled" return data_dir / "hy-memory" / HY_MEMORY_NAMESPACE / name diff --git a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py index c52a740..48f7490 100644 --- a/memhub/apps/core/src/one_memory/hy_sdk_runtime.py +++ b/memhub/apps/core/src/one_memory/hy_sdk_runtime.py @@ -31,7 +31,7 @@ from one_memory.embedding_assets import ( ) from one_memory.hy_graph_admin import ( GraphWidthMismatchError, - graph_store_enabled, + effective_memory_mode, inspect_graph_width_mismatch, ) from one_memory.hy_sdk_types import HySdkRuntimeError @@ -170,11 +170,14 @@ class SdkClientHandle: api_key: str | None, data_dir: Path, http_transport_factory: Callable[[], Any] | None = None, + mode: str | None = None, ) -> None: self._profile = profile self._api_key = api_key self._data_dir = data_dir self._http_transport_factory = http_transport_factory + #: The SDK mode this handle's client is built with, fixed for its life. + self.mode = mode or effective_memory_mode() self._lease: ClientLease | None = None self._lock = asyncio.Lock() @@ -189,6 +192,7 @@ class SdkClientHandle: api_key=self._api_key, data_dir=self._data_dir, http_transport_factory=self._http_transport_factory, + mode=self.mode, ) return self._lease.client @@ -352,9 +356,12 @@ def acquire_client( api_key: str | None, data_dir: Path, http_transport_factory: Callable[[], Any] | None = None, + mode: str | None = None, ) -> ClientLease: """Construct a public SDK client and return its explicit runtime lease. + `mode` is the SDK pipeline mode (lite / pro / ultra); None keeps the + pre-setting default (ultra when `ONE_MEMORY_GRAPH_STORE` is set, else pro). Waits while `hold_stores_offline` holds this home's stores. """ key = data_dir.resolve() @@ -368,6 +375,7 @@ def acquire_client( api_key=api_key, data_dir=data_dir, http_transport_factory=http_transport_factory, + mode=mode or effective_memory_mode(), ) finally: with _STORE_GATE: @@ -406,6 +414,7 @@ def _acquire_client( api_key: str | None, data_dir: Path, http_transport_factory: Callable[[], Any] | None, + mode: str = "pro", ) -> ClientLease: config = build_sdk_config(profile, api_key=api_key, data_dir=data_dir) config.http_transport_factory = http_transport_factory @@ -423,7 +432,7 @@ def _acquire_client( with constructing: client = HyMemoryClient( config=config, - mode="ultra" if graph_store_enabled() else "pro", + mode=mode, runtime=runtime, embedding_service=embedding_service, ) diff --git a/memhub/apps/core/src/one_memory/memory_evolution.py b/memhub/apps/core/src/one_memory/memory_evolution.py new file mode 100644 index 0000000..7edebc5 --- /dev/null +++ b/memhub/apps/core/src/one_memory/memory_evolution.py @@ -0,0 +1,618 @@ +"""System 2 for ultra: evolve runs, their receipts, the nightly schedule, graph backfill. + +hy-memory 2.0.9's `client.evolve(user_id, agent_id, max_batches)` is ultra +only, holds no lock, and has no scheduler: MemHub owns all three. + +- A run is asynchronous: `start` answers a receipt at once and the run goes + on in the background (evolve makes model calls across the user's whole + library and can take minutes, longer than any HTTP timeout). One run per + user at a time: asking again while one runs returns that run. A run holds + the write coordinator's per-user exclusivity, so it never overlaps that + user's write batch, and loops while the SDK answers `should_continue`, up + to `EVOLVE_MAX_ROUNDS`. +- The schedule runs every user that has memories once a day at the + configured local time, and catches up after the machine slept through it. +- Switching to ultra backfills each user's existing facts into the graph + once (`build_memory_graph`). + +What is kept on disk is counts and stable codes only, never memory text: +`/evolve-state.json`. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +import threading +from collections.abc import Awaitable, Callable, Iterable, Mapping +from datetime import UTC, datetime, timedelta +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any, Literal, Protocol +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field + +from one_memory.diagnostic_log import diagnostic_event +from one_memory.memory_server.config_state import EvolveSchedule +from one_memory.memory_server.durable_queue import safe_error_code +from one_memory.sdk_write_processor import PRODUCT_AGENT_ID + +logger = logging.getLogger(__name__) + +#: Rounds one run makes while the SDK says there is more to do. +EVOLVE_MAX_ROUNDS = 20 +#: How often the schedule looks at the clock; also bounds catch-up after wake. +SCHEDULE_CHECK_SECONDS = 300.0 +#: Receipts kept in memory for `GET /api/v2/evolve/{run_id}`. +_KEPT_RUNS = 64 +_STABLE_CODE = re.compile(r"[a-z][a-z0-9_]{0,95}") +_COUNTS = ( + "remaining_clusters", + "remaining_actionable_clusters", + "remaining_fresh_facts", +) + +RunState = Literal["running", "succeeded", "failed"] +Trigger = Literal["manual", "scheduled"] + + +class EvolveModeError(RuntimeError): + """Evolve asked for outside ultra.""" + + def __init__(self) -> None: + super().__init__("memory_mode_not_ultra") + + +class EvolveRunReceipt(BaseModel): + """One evolve run. Counts and codes; `results` only while in memory.""" + + model_config = ConfigDict(extra="forbid") + + run_id: str + user_id: str + agent_id: str + max_batches: int | None = None + trigger: Trigger = "manual" + state: RunState = "running" + started_at: datetime + finished_at: datetime | None = None + rounds: int = 0 + tasks_processed: int = 0 + batch_failures: int = 0 + cycle_status: str | None = None + should_continue: bool | None = None + continuation_reason: str | None = None + remaining_clusters: int | None = None + remaining_actionable_clusters: int | None = None + remaining_fresh_facts: int | None = None + #: True when the round cap stopped a run the SDK would have continued. + capped: bool = False + elapsed_ms: float = 0.0 + error_code: str | None = None + failure_kind: str | None = None + #: The last round's SDK `request_id`, for matching SDK logs. + request_id: str | None = None + #: The last round's SDK `results` and `cross_domain_sweeper`. Kept only + #: in this receipt, never written to disk or diagnostics. + results: Any = None + cross_domain_sweeper: Any = None + + def summary(self) -> dict[str, object]: + """What is persisted and shown in status: no SDK result bodies.""" + return self.model_dump(mode="json", exclude={"results", "cross_domain_sweeper"}) + + +class UserEvolveState(BaseModel): + model_config = ConfigDict(extra="forbid") + + last_run_at: datetime | None = None + last_result: dict[str, object] | None = None + last_error: str | None = None + + +class GraphBuildUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + state: Literal["pending", "running", "succeeded", "failed"] = "pending" + memories_scanned: int | None = None + nodes: int | None = None + error_code: str | None = None + + +class GraphBuildState(BaseModel): + """The one-time backfill of existing facts into the graph after → ultra.""" + + model_config = ConfigDict(extra="forbid") + + state: Literal["idle", "running", "succeeded", "failed"] = "idle" + started_at: datetime | None = None + finished_at: datetime | None = None + users: dict[str, GraphBuildUser] = Field(default_factory=dict) + + +class EvolveStateDocument(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: int = Field(default=1, ge=1, le=1) + users: dict[str, UserEvolveState] = Field(default_factory=dict) + #: When the schedule first ran in ultra: the "last run" of a user that + #: never ran, so enabling ultra at noon does not evolve everyone at once. + schedule_anchor: datetime | None = None + graph_build: GraphBuildState = Field(default_factory=GraphBuildState) + + +class EvolveStateStore: + """The only writer of `/evolve-state.json`.""" + + def __init__(self, data_dir: Path) -> None: + self.path = data_dir / "evolve-state.json" + self._lock = threading.RLock() + self._document = self._read() + + def document(self) -> EvolveStateDocument: + with self._lock: + return self._document.model_copy(deep=True) + + def user(self, user_id: str) -> UserEvolveState: + with self._lock: + return self._document.users.get(user_id, UserEvolveState()).model_copy() + + def record_run(self, receipt: EvolveRunReceipt) -> None: + with self._lock: + users = dict(self._document.users) + users[receipt.user_id] = UserEvolveState( + last_run_at=receipt.started_at, + last_result=receipt.summary(), + last_error=receipt.error_code, + ) + self._save(self._document.model_copy(update={"users": users})) + + def anchor(self, now: datetime) -> datetime: + """The schedule anchor, set to `now` the first time it is asked.""" + with self._lock: + if self._document.schedule_anchor is None: + self._save(self._document.model_copy(update={"schedule_anchor": now})) + assert self._document.schedule_anchor is not None + return self._document.schedule_anchor + + def graph_build(self) -> GraphBuildState: + with self._lock: + return self._document.graph_build.model_copy(deep=True) + + def set_graph_build(self, state: GraphBuildState) -> None: + with self._lock: + self._save(self._document.model_copy(update={"graph_build": state})) + + def _save(self, document: EvolveStateDocument) -> None: + payload = json.dumps(document.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path | None = None + try: + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=self.path.parent, + prefix=".evolve-state.", + suffix=".tmp", + delete=False, + ) as stream: + temporary = Path(stream.name) + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, 0o600) + os.replace(temporary, self.path) + except OSError: + if temporary is not None: + temporary.unlink(missing_ok=True) + # Status is best-effort: a run must not fail because its record did. + logger.warning("evolve_state_write_failed") + self._document = document + + def _read(self) -> EvolveStateDocument: + try: + return EvolveStateDocument.model_validate_json(self.path.read_text(encoding="utf-8")) + except FileNotFoundError: + return EvolveStateDocument() + except (OSError, ValueError): + # A damaged status file only loses history; start a fresh one. + logger.warning("evolve_state_unreadable") + return EvolveStateDocument() + + +class EvolveCoordinator(Protocol): + """The write coordinator surface evolve and the graph backfill use.""" + + def exclusive_for_evolve(self, user_id: str) -> Any: ... + + def use_client(self) -> Any: ... + + +class EvolveService: + """Start, serialize and record evolve runs.""" + + def __init__( + self, + *, + coordinator: EvolveCoordinator, + state: EvolveStateStore, + memory_mode: Callable[[], str], + max_rounds: int = EVOLVE_MAX_ROUNDS, + now: Callable[[], datetime] = lambda: datetime.now(UTC), + ) -> None: + self.coordinator = coordinator + self.state = state + self.memory_mode = memory_mode + self.max_rounds = max_rounds + self._now = now + self._runs: dict[str, EvolveRunReceipt] = {} + self._tasks: dict[str, asyncio.Task[None]] = {} + self._active: dict[str, str] = {} + + def start( + self, + user_id: str, + *, + agent_id: str | None = None, + max_batches: int | None = None, + trigger: Trigger = "manual", + ) -> EvolveRunReceipt: + """Start a run, or return the one already running for this user.""" + if self.memory_mode() != "ultra": + raise EvolveModeError() + running = self._active.get(user_id) + if running is not None: + return self._runs[running] + receipt = EvolveRunReceipt( + run_id=str(uuid4()), + user_id=user_id, + agent_id=agent_id or PRODUCT_AGENT_ID, + max_batches=max_batches, + trigger=trigger, + started_at=self._now(), + ) + self._remember(receipt) + self._active[user_id] = receipt.run_id + self._tasks[receipt.run_id] = asyncio.create_task( + self._run(receipt.run_id), name="one-memory-evolve" + ) + return receipt + + def get(self, run_id: str) -> EvolveRunReceipt | None: + return self._runs.get(run_id) + + def running(self) -> tuple[EvolveRunReceipt, ...]: + return tuple(self._runs[run_id] for run_id in self._active.values()) + + async def wait(self, run_id: str) -> EvolveRunReceipt | None: + task = self._tasks.get(run_id) + if task is not None: + await asyncio.shield(task) + return self._runs.get(run_id) + + async def shutdown(self) -> None: + tasks = tuple(self._tasks.values()) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + self._tasks.clear() + + def _remember(self, receipt: EvolveRunReceipt) -> None: + self._runs[receipt.run_id] = receipt + while len(self._runs) > _KEPT_RUNS: + oldest = next(iter(self._runs)) + if oldest in self._active.values(): + break + del self._runs[oldest] + + async def _run(self, run_id: str) -> None: + receipt = self._runs[run_id] + diagnostic_event( + level="info", + component="memory.evolve", + event="memory_evolve.started", + outcome="started", + details={"trigger": receipt.trigger, "max_batches": receipt.max_batches}, + ) + try: + async with ( + self.coordinator.exclusive_for_evolve(receipt.user_id), + self.coordinator.use_client() as client, + ): + while receipt.rounds < self.max_rounds: + result = await asyncio.to_thread( + client.evolve, + receipt.user_id, + agent_id=receipt.agent_id, + max_batches=receipt.max_batches, + ) + receipt = _folded(receipt, result) + self._runs[run_id] = receipt + if receipt.state == "failed" or not receipt.should_continue: + break + else: + receipt = receipt.model_copy(update={"capped": True}) + if receipt.state == "running": + receipt = receipt.model_copy(update={"state": "succeeded"}) + except asyncio.CancelledError: + receipt = receipt.model_copy( + update={"state": "failed", "error_code": "memory_evolve_cancelled"} + ) + raise + except Exception as error: # noqa: BLE001 - recorded as a stable code + code = safe_error_code(error) + receipt = receipt.model_copy( + update={ + "state": "failed", + "error_code": _stable(code, "hy_memory_evolve_failed"), + "failure_kind": type(error).__name__, + } + ) + finally: + receipt = receipt.model_copy(update={"finished_at": self._now()}) + self._runs[run_id] = receipt + self._active.pop(receipt.user_id, None) + self._tasks.pop(run_id, None) + await asyncio.to_thread(self.state.record_run, receipt) + diagnostic_event( + level="info" if receipt.state == "succeeded" else "warning", + component="memory.evolve", + event="memory_evolve.finished", + outcome="succeeded" if receipt.state == "succeeded" else "failed", + details={ + "trigger": receipt.trigger, + "rounds": receipt.rounds, + "tasks_processed": receipt.tasks_processed, + "batch_failures": receipt.batch_failures, + "capped": receipt.capped, + "elapsed_ms": round(receipt.elapsed_ms, 1), + "error_code": receipt.error_code, + }, + ) + + +def _folded(receipt: EvolveRunReceipt, result: object) -> EvolveRunReceipt: + """One more evolve round's result added to the run.""" + if not isinstance(result, Mapping): + return receipt.model_copy( + update={"state": "failed", "error_code": "hy_memory_evolve_shape_unknown"} + ) + failures = result.get("batch_failures") + update: dict[str, object] = { + "rounds": receipt.rounds + 1, + "tasks_processed": receipt.tasks_processed + _int(result.get("tasks_processed")), + "batch_failures": receipt.batch_failures + + (len(failures) if isinstance(failures, list | tuple) else _int(failures)), + "elapsed_ms": receipt.elapsed_ms + _float(result.get("elapsed_ms")), + "should_continue": result.get("should_continue") is True, + "cycle_status": _text(result.get("cycle_status")), + "continuation_reason": _text(result.get("continuation_reason")), + "request_id": _text(result.get("request_id")), + "results": result.get("results"), + "cross_domain_sweeper": result.get("cross_domain_sweeper"), + } + for name in _COUNTS: + value = result.get(name) + update[name] = value if isinstance(value, int) and not isinstance(value, bool) else None + if result.get("success") is not True: + code = result.get("error_code") + kind = result.get("failure_kind") + update.update( + state="failed", + should_continue=False, + error_code=( + code + if isinstance(code, str) and _STABLE_CODE.fullmatch(code) + else "hy_memory_evolve_rejected" + ), + failure_kind=kind if isinstance(kind, str) and _STABLE_CODE.fullmatch(kind) else None, + ) + return receipt.model_copy(update=update) + + +def _stable(code: str, fallback: str) -> str: + return code if _STABLE_CODE.fullmatch(code) else fallback + + +def _int(value: object) -> int: + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def _float(value: object) -> float: + if isinstance(value, int | float) and not isinstance(value, bool): + return float(value) + return 0.0 + + +def _text(value: object) -> str | None: + return value[:128] if isinstance(value, str) else None + + +def evolve_due( + now: datetime, + schedule: EvolveSchedule, + last_run_at: datetime | None, + anchor: datetime, +) -> bool: + """Whether a user's nightly evolve is due at `now` (aware, local time). + + Due once the day's slot (`schedule.time`, local) has passed and the last + run (or, for a user who never ran, the anchor) is older than that slot. + A last run more than 24 hours old is due at once as well: a machine asleep + at the slot catches up when it wakes, even before the next day's slot. + """ + if not schedule.enabled: + return False + local = now.astimezone() + hour, minute = (int(part) for part in schedule.time.split(":")) + slot = local.replace(hour=hour, minute=minute, second=0, microsecond=0) + reference = last_run_at or anchor + if local >= slot and reference < slot: + return True + return last_run_at is not None and local - last_run_at > timedelta(hours=24) + + +class EvolveScheduler: + """Run evolve nightly for every user that has memories (ultra only).""" + + def __init__( + self, + *, + service: EvolveService, + schedule: Callable[[], EvolveSchedule], + users_with_memories: Callable[[], Awaitable[Iterable[str]]], + now: Callable[[], datetime] = lambda: datetime.now().astimezone(), + check_seconds: float = SCHEDULE_CHECK_SECONDS, + ) -> None: + self.service = service + self.schedule = schedule + self.users_with_memories = users_with_memories + self._now = now + self.check_seconds = check_seconds + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + if self._task is None or self._task.done(): + self._task = asyncio.create_task(self._loop(), name="one-memory-evolve-schedule") + + async def tick(self) -> tuple[str, ...]: + """Run every due user once, one after another; the users run.""" + schedule = self.schedule() + if not schedule.enabled or self.service.memory_mode() != "ultra": + return () + now = self._now() + anchor = await asyncio.to_thread(self.service.state.anchor, now) + ran: list[str] = [] + for user_id in await self.users_with_memories(): + last = self.service.state.user(user_id).last_run_at + if not evolve_due(now, schedule, last, anchor): + continue + try: + receipt = self.service.start(user_id, trigger="scheduled") + except EvolveModeError: + break + await self.service.wait(receipt.run_id) + ran.append(user_id) + if ran: + diagnostic_event( + level="info", + component="memory.evolve", + event="memory_evolve.schedule_ran", + outcome="succeeded", + details={"users": len(ran)}, + ) + return tuple(ran) + + async def _loop(self) -> None: + while True: + try: + await self.tick() + except asyncio.CancelledError: + raise + except Exception as error: # noqa: BLE001 - the loop must survive + logger.warning("evolve_schedule_tick_failed error_type=%s", type(error).__name__) + await asyncio.sleep(self.check_seconds) + + async def shutdown(self) -> None: + if self._task is None: + return + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + self._task = None + + +class MemoryGraphBackfill: + """After a switch to ultra, project each user's existing facts into the graph once.""" + + def __init__( + self, + *, + coordinator: EvolveCoordinator, + state: EvolveStateStore, + now: Callable[[], datetime] = lambda: datetime.now(UTC), + ) -> None: + self.coordinator = coordinator + self.state = state + self._now = now + self._task: asyncio.Task[None] | None = None + + @property + def running(self) -> bool: + return self._task is not None and not self._task.done() + + def start(self, user_ids: Iterable[str]) -> GraphBuildState: + if self.running: + return self.state.graph_build() + users = tuple(dict.fromkeys(user_ids)) + state = GraphBuildState( + state="running", + started_at=self._now(), + users={user_id: GraphBuildUser() for user_id in users}, + ) + self.state.set_graph_build(state) + self._task = asyncio.create_task(self._run(users), name="one-memory-graph-backfill") + return state + + async def wait(self) -> None: + if self._task is not None: + await asyncio.shield(self._task) + + async def _run(self, users: tuple[str, ...]) -> None: + state = self.state.graph_build() + for user_id in users: + state.users[user_id] = GraphBuildUser(state="running") + await asyncio.to_thread(self.state.set_graph_build, state.model_copy(deep=True)) + try: + async with ( + self.coordinator.exclusive_for_evolve(user_id), + self.coordinator.use_client() as client, + ): + result = await asyncio.to_thread( + client.build_memory_graph, + user_id, + agent_id=None, + limit=10000, + clear_first=False, + ) + ok = isinstance(result, Mapping) and result.get("success") is True + state.users[user_id] = GraphBuildUser( + state="succeeded" if ok else "failed", + memories_scanned=_int(result.get("memories_scanned")) + if isinstance(result, Mapping) + else None, + nodes=_int(result.get("nodes")) if isinstance(result, Mapping) else None, + error_code=None if ok else "hy_memory_graph_build_rejected", + ) + except asyncio.CancelledError: + raise + except Exception as error: # noqa: BLE001 - recorded as a stable code + code = safe_error_code(error) + state.users[user_id] = GraphBuildUser( + state="failed", + error_code=_stable(code, "hy_memory_graph_build_failed"), + ) + failed = any(user.state == "failed" for user in state.users.values()) + state.state = "failed" if failed else "succeeded" + state.finished_at = self._now() + await asyncio.to_thread(self.state.set_graph_build, state) + diagnostic_event( + level="warning" if failed else "info", + component="memory.evolve", + event="memory_graph_backfill.finished", + outcome="failed" if failed else "succeeded", + details={ + "users": len(state.users), + "failed_users": sum(user.state == "failed" for user in state.users.values()), + }, + ) + + async def shutdown(self) -> None: + if self._task is None: + return + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + self._task = None diff --git a/memhub/apps/core/src/one_memory/memory_server/config_state.py b/memhub/apps/core/src/one_memory/memory_server/config_state.py index c6a8247..3f7419e 100644 --- a/memhub/apps/core/src/one_memory/memory_server/config_state.py +++ b/memhub/apps/core/src/one_memory/memory_server/config_state.py @@ -13,6 +13,7 @@ import threading from collections.abc import Mapping from pathlib import Path from tempfile import NamedTemporaryFile +from typing import Literal from pydantic import BaseModel, ConfigDict, Field, StrictBool, field_validator @@ -27,6 +28,31 @@ class SourceConfig(BaseModel): sync_enabled: StrictBool = False +#: The SDK pipeline mode (hy-memory `HyMemoryClient(mode=...)`): +#: lite keeps raw conversations only, with no model call; pro extracts +#: memories (System 1); ultra adds the knowledge graph and evolve (System 2). +MemoryMode = Literal["lite", "pro", "ultra"] +MEMORY_MODES: tuple[MemoryMode, ...] = ("lite", "pro", "ultra") +_SCHEDULE_TIME = re.compile(r"^(?:[01][0-9]|2[0-3]):[0-5][0-9]$") + + +class EvolveSchedule(BaseModel): + """When ultra runs evolve for every user: once a day, local time.""" + + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + enabled: StrictBool = True + #: `HH:MM`, 24-hour, the machine's local time. + time: str = "03:00" + + @field_validator("time") + @classmethod + def valid_time(cls, value: str) -> str: + if _SCHEDULE_TIME.fullmatch(value) is None: + raise ValueError("invalid_evolve_schedule_time") + return value + + class ConfigDocument(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", strict=True) @@ -39,6 +65,10 @@ class ConfigDocument(BaseModel): #: needs no re-entry of a key. sedimentation_enabled: StrictBool = True extraction_window_preset: ExtractionWindowPreset = ExtractionWindowPreset.THREE_MONTHS + #: The SDK mode every client is built with. `ONE_MEMORY_GRAPH_STORE` (dev) + #: still forces ultra over it; see `hy_graph_admin.effective_memory_mode`. + memory_mode: MemoryMode = "pro" + evolve_schedule: EvolveSchedule = Field(default_factory=EvolveSchedule) @field_validator("sources") @classmethod @@ -61,6 +91,9 @@ class ConfigFileError(RuntimeError): pass +_WRITTEN_WHEN_SET = frozenset({"memory_mode", "evolve_schedule"}) + + class ConfigStore: """The only writer of ``/config.json`` inside Core.""" @@ -107,6 +140,32 @@ class ConfigStore: self._document = updated return updated + def memory_mode(self) -> MemoryMode: + with self._lock: + return self._document.memory_mode + + def set_memory_mode(self, mode: MemoryMode) -> ConfigDocument: + if mode not in MEMORY_MODES: + raise ValueError("invalid_memory_mode") + with self._lock: + updated = self._document.model_copy(update={"memory_mode": mode}) + self._write(updated) + self._document = updated + return updated + + def evolve_schedule(self) -> EvolveSchedule: + with self._lock: + return self._document.evolve_schedule + + def set_evolve_schedule(self, schedule: EvolveSchedule) -> ConfigDocument: + if not isinstance(schedule, EvolveSchedule): + raise ValueError("invalid_evolve_schedule") + with self._lock: + updated = self._document.model_copy(update={"evolve_schedule": schedule}) + self._write(updated) + self._document = updated + return updated + def migrate_extraction_window_preset(self, preset: ExtractionWindowPreset) -> ConfigDocument: """Adopt the legacy SQL value only when config.json has no value.""" if not isinstance(preset, ExtractionWindowPreset): @@ -152,9 +211,12 @@ class ConfigStore: raise ConfigFileError("config_invalid") from error def _write(self, document: ConfigDocument) -> None: + # Settings added after 1.0.5 are written only once a person chose + # them, so an untouched install's file stays what older builds read. + unset = {name for name in _WRITTEN_WHEN_SET if name not in document.model_fields_set} payload = ( json.dumps( - document.model_dump(mode="json"), + document.model_dump(mode="json", exclude=unset), ensure_ascii=False, indent=2, sort_keys=True, diff --git a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py b/memhub/apps/core/src/one_memory/memory_server/durable_queue.py index a708060..878650a 100644 --- a/memhub/apps/core/src/one_memory/memory_server/durable_queue.py +++ b/memhub/apps/core/src/one_memory/memory_server/durable_queue.py @@ -2563,6 +2563,8 @@ async def execute_job( in { "model_content_source_disabled", "model_sedimentation_disabled", + # Evolve holds this user; the job runs once it is done. + "memory_evolve_in_progress", } and not hy_invoked ): diff --git a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py index 8b5ee32..5c010c2 100644 --- a/memhub/apps/core/src/one_memory/sdk_memory_runtime.py +++ b/memhub/apps/core/src/one_memory/sdk_memory_runtime.py @@ -5,8 +5,8 @@ from __future__ import annotations import asyncio import logging import re -from collections.abc import AsyncIterator, Callable, Iterable -from contextlib import AsyncExitStack, asynccontextmanager, suppress +from collections.abc import AsyncIterator, Callable +from contextlib import AsyncExitStack, asynccontextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -33,16 +33,6 @@ from one_memory.semantic_deadline import SemanticMemoryHit _STABLE_CODE = re.compile(r"[a-z][a-z0-9_]{0,95}") logger = logging.getLogger(__name__) -RAW_RETENTION_DAYS = 3 -RAW_RETENTION_CHECK_SECONDS = 6 * 60 * 60 -_RAW_STATUSES = ( - "active", - "superseded", - "shadow", - "negated", - "conflicted", - "archived", -) class SdkMemoryRuntime: @@ -56,84 +46,17 @@ class SdkMemoryRuntime: write_queue: DurableWriteQueue, extraction_settings: ExtractionSettingsStore, data_dir: Path, - user_ids: Callable[[], Iterable[str]] = lambda: ("local",), + memory_mode: Callable[[], str] = lambda: "pro", ) -> None: self.coordinator = coordinator - #: Every user whose raw layer is kept to the retention window. - self.user_ids = user_ids + #: The SDK mode, for the graph store's path (`hy_graph_admin`). + self.memory_mode = memory_mode self.read_handle = read_handle self.write_queue = write_queue self.extraction_settings = extraction_settings self.data_dir = data_dir - self._raw_retention_task: asyncio.Task[None] | None = None - - def start_raw_retention(self) -> None: - if self._raw_retention_task is None or self._raw_retention_task.done(): - self._raw_retention_task = asyncio.create_task( - self._raw_retention_loop(), - name="one-memory-sdk-raw-retention", - ) - - async def prune_expired_raw(self, *, now: datetime | None = None) -> int: - deleted = 0 - for user_id in dict.fromkeys(await asyncio.to_thread(lambda: tuple(self.user_ids()))): - deleted += await self._prune_expired_raw_for(user_id, now=now) - return deleted - - async def _prune_expired_raw_for(self, user_id: str, *, now: datetime | None) -> int: - cutoff = (now or datetime.now(UTC)) - timedelta(days=RAW_RETENTION_DAYS) - cutoff_timestamp = int(cutoff.timestamp()) - client = await self.read_handle.client() - deleted = 0 - while True: - page = await asyncio.to_thread( - client.list_memories, - user_id=user_id, - layers=["l1_raw"], - statuses=list(_RAW_STATUSES), - order="asc", - limit=500, - offset=0, - ) - bucket = page.get("vdb") if isinstance(page, dict) else None - rows = bucket.get("memories") if isinstance(bucket, dict) else None - if not isinstance(rows, list) or not rows: - return deleted - expired = [ - row - for row in rows - if isinstance(row, dict) - and isinstance(row.get("memory_id"), str) - and int(row.get("memory_at") or row.get("gmt_created") or 0) < cutoff_timestamp - ] - if not expired: - return deleted - removed_page = 0 - for row in expired: - result = await asyncio.to_thread(client.delete, row["memory_id"]) - if isinstance(result, dict) and result.get("success") is True: - count = int(result.get("deleted_count") or 0) - deleted += count - removed_page += count - if removed_page == 0: - return deleted - - async def _raw_retention_loop(self) -> None: - while True: - await asyncio.sleep(RAW_RETENTION_CHECK_SECONDS) - try: - # Its own task, so this pass's use of the read client ends - # with it instead of with this loop (see `sdk_read_handle`). - removed = await asyncio.create_task(self.prune_expired_raw()) - if removed: - logger.info("sdk_raw_retention_deleted count=%d", removed) - except asyncio.CancelledError: - raise - except Exception as error: - logger.warning( - "sdk_raw_retention_failed error_type=%s", - type(error).__name__, - ) + # MemHub keeps every raw the SDK stores, in every mode: there is no + # retention window (owner decision; the SDK itself never deletes raw). async def semantic_recall( self, @@ -233,6 +156,7 @@ class SdkMemoryRuntime: profile, api_key=secret, data_dir=self.data_dir, + mode=self.memory_mode(), ) async def decide_graph_width( @@ -249,6 +173,7 @@ class SdkMemoryRuntime: api_key=secret, data_dir=self.data_dir, decision=decision, + mode=self.memory_mode(), ) @asynccontextmanager @@ -387,12 +312,7 @@ class SdkMemoryRuntime: return None async def shutdown(self) -> None: - if self._raw_retention_task is None: - return - self._raw_retention_task.cancel() - with suppress(asyncio.CancelledError): - await self._raw_retention_task - self._raw_retention_task = None + return None def _agent_ids(raw: object) -> tuple[str, ...]: diff --git a/memhub/apps/core/src/one_memory/sdk_read_handle.py b/memhub/apps/core/src/one_memory/sdk_read_handle.py index 29cade6..e9d13fb 100644 --- a/memhub/apps/core/src/one_memory/sdk_read_handle.py +++ b/memhub/apps/core/src/one_memory/sdk_read_handle.py @@ -10,14 +10,20 @@ is closed once no task still uses it, as `SdkWriteCoordinator` does for writes (#218). A use is the task that asked for the client: callers keep their `client = await handle.client()` shape, and the use ends with that task (one HTTP request, one retention pass). + +The memory mode (lite / pro / ultra) is fixed per SDK client, so a mode change +rebuilds and retires the same way: handles are keyed by (profile, mode). The +mode never names a collection; it is a separate client parameter. """ from __future__ import annotations import asyncio +from collections.abc import Callable from pathlib import Path from typing import Any +from one_memory.hy_graph_admin import effective_memory_mode from one_memory.hy_sdk_runtime import ( BUILTIN_RETRIEVAL_PROFILE_ID, SdkClientHandle, @@ -40,12 +46,14 @@ class ActiveProfileReadHandle(SdkClientHandle): profiles: ModelProfileStore, secrets: SecretStore, data_dir: Path, + memory_mode: Callable[[], str] = effective_memory_mode, ) -> None: super().__init__(builtin_retrieval_snapshot(), api_key=None, data_dir=data_dir) self.profiles = profiles self.secrets = secrets self.data_dir = data_dir - self._handles: dict[tuple[str, int], SdkClientHandle] = {} + self.memory_mode = memory_mode + self._handles: dict[tuple[str, int, str], SdkClientHandle] = {} #: Tasks in flight per handle; a retired handle closes at zero. self._uses: dict[SdkClientHandle, int] = {} self._retired: set[SdkClientHandle] = set() @@ -53,7 +61,8 @@ class ActiveProfileReadHandle(SdkClientHandle): self._handle_lock = asyncio.Lock() async def client(self) -> Any: - identity = await asyncio.to_thread(self.profiles.active_identity) or _BUILTIN + profile = await asyncio.to_thread(self.profiles.active_identity) or _BUILTIN + identity = (*profile, self.memory_mode()) async with self._handle_lock: handle = self._handles.get(identity) if handle is None: @@ -95,10 +104,12 @@ class ActiveProfileReadHandle(SdkClientHandle): self._closing.add(closing) closing.add_done_callback(self._closing.discard) - def _build(self, identity: tuple[str, int]) -> SdkClientHandle: + def _build(self, key: tuple[str, int, str]) -> SdkClientHandle: + profile_id, revision, mode = key + identity = (profile_id, revision) if identity == _BUILTIN: return SdkClientHandle( - builtin_retrieval_snapshot(), api_key=None, data_dir=self.data_dir + builtin_retrieval_snapshot(), api_key=None, data_dir=self.data_dir, mode=mode ) profile = self.profiles.snapshot(*identity) view = retrieval_view(profile) @@ -111,4 +122,4 @@ class ActiveProfileReadHandle(SdkClientHandle): raise HySdkRuntimeError("secure_storage_unavailable") from None if secret is None: raise HySdkRuntimeError("model_secret_unavailable") - return SdkClientHandle(view, api_key=secret, data_dir=self.data_dir) + return SdkClientHandle(view, api_key=secret, data_dir=self.data_dir, mode=mode) diff --git a/memhub/apps/core/src/one_memory/sdk_write_processor.py b/memhub/apps/core/src/one_memory/sdk_write_processor.py index dd5d94a..ca98bf7 100644 --- a/memhub/apps/core/src/one_memory/sdk_write_processor.py +++ b/memhub/apps/core/src/one_memory/sdk_write_processor.py @@ -9,7 +9,8 @@ from dataclasses import dataclass from datetime import datetime from typing import Any -from one_memory.hy_sdk_runtime import SdkClientHandle +from one_memory.hy_graph_admin import effective_memory_mode +from one_memory.hy_sdk_runtime import SdkClientHandle, builtin_retrieval_snapshot from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord from one_memory.input_sanitization import sanitize_model_input from one_memory.model_profiles import ModelProfileSnapshot, ModelProfileStore @@ -31,8 +32,11 @@ class SdkWriteReceipt: class SdkWriteProcessor: """Turn one durable job into exactly one public SDK write call.""" - def __init__(self, client_handle: SdkClientHandle) -> None: + def __init__(self, client_handle: SdkClientHandle, *, lite: bool = False) -> None: self._client_handle = client_handle + #: lite makes no model call: `add_extracted` would still call one, so a + #: direct memory note is stored as one raw user turn through `add`. + self._lite = lite async def process_sdk_job( self, @@ -64,7 +68,7 @@ class SdkWriteProcessor: # waits and retries instead of being left `unknown`. raise RuntimeError("hy_memory_client_unavailable") from error - if extracted: + if extracted and not self._lite: contents = [sanitize_model_input(record.verbatim) for record in records] await before_hy_invocation() result = await asyncio.to_thread( @@ -81,12 +85,15 @@ class SdkWriteProcessor: else: messages = [ { - "role": record.role, + "role": "user" if extracted else record.role, "content": sanitize_model_input(record.verbatim), } for record in records - if record.content_kind is EvidenceContentKind.RAW_CONVERSATION - and record.role in _MESSAGE_ROLES + if extracted + or ( + record.content_kind is EvidenceContentKind.RAW_CONVERSATION + and record.role in _MESSAGE_ROLES + ) ] if not messages: raise RuntimeError("hy_memory_task_without_inputs") @@ -138,6 +145,15 @@ class SdkWriteCoordinator: from under a write still running on it: the SDK cleared its stores and the SharedRuntime closed mid-`add`, the job failed with AttributeError after HY was invoked, and it was left `unknown`. + + Handles are keyed by profile and memory mode, so a mode change retires + handles the same way. In lite no generation model is needed: without an + active profile the built-in local embedder writes raw turns. + + `exclusive_for_evolve(user_id)` keeps evolve and that user's writes + apart: it waits for the user's write in flight, and while it runs that + user's jobs are refused before the SDK is called + (`memory_evolve_in_progress`), so the drain puts them back. """ def __init__( @@ -148,13 +164,22 @@ class SdkWriteCoordinator: data_dir: Any, transport_factory: Callable[[ModelProfileSnapshot], Callable[[], Any] | None], sedimentation_enabled: Callable[[], bool], + memory_mode: Callable[[], str] = effective_memory_mode, ) -> None: self.profiles = profiles self.secrets = secrets self.data_dir = data_dir self.transport_factory = transport_factory self.sedimentation_enabled = sedimentation_enabled - self._handles: dict[tuple[str, int], SdkClientHandle] = {} + self.memory_mode = memory_mode + self._handles: dict[tuple[str, int, str], SdkClientHandle] = {} + #: Users evolve (or the graph backfill) is running for, the writes in + #: flight per user, and one exclusive run per user at a time. Writes + #: of one user still run side by side; only evolve waits for them. + self._evolving: set[str] = set() + self._writing: dict[str, int] = {} + self._writes_idle = asyncio.Condition() + self._exclusive: dict[str, asyncio.Lock] = {} #: Uses in flight per handle; a retired handle closes at zero. self._uses: dict[SdkClientHandle, int] = {} self._retired: set[SdkClientHandle] = set() @@ -169,16 +194,66 @@ class SdkWriteCoordinator: ) -> SdkWriteReceipt: if not self.sedimentation_enabled(): raise RuntimeError("model_sedimentation_disabled") - async with self._active_handle() as handle: - return await SdkWriteProcessor(handle).process_sdk_job( - records, - request_id=request_id, - before_hy_invocation=before_hy_invocation, - ) + user_id = records[0].user_id if records else "local" + if user_id in self._evolving: + # Before any SDK call: the drain returns the job to pending. + raise RuntimeError("memory_evolve_in_progress") + self._writing[user_id] = self._writing.get(user_id, 0) + 1 + try: + async with self._active_handle() as handle: + return await SdkWriteProcessor( + handle, lite=getattr(handle, "mode", None) == "lite" + ).process_sdk_job( + records, + request_id=request_id, + before_hy_invocation=before_hy_invocation, + ) + finally: + remaining = self._writing.get(user_id, 1) - 1 + if remaining > 0: + self._writing[user_id] = remaining + else: + self._writing.pop(user_id, None) + async with self._writes_idle: + self._writes_idle.notify_all() - async def inquire_write_outcome( - self, request_id: str - ) -> SdkWriteReceipt | None: + async def retire_stale_modes(self) -> None: + """After a mode change: retire handles of other modes, closing the idle ones. + + A handle still writing is closed when its last use ends (#218), so a + write that started in the old mode finishes in it. + """ + mode = self.memory_mode() + async with self._lock: + stale = {key: handle for key, handle in self._handles.items() if key[2] != mode} + for key in stale: + del self._handles[key] + self._retired.update(stale.values()) + unused = {value for value in self._retired if not self._uses.get(value)} + self._retired -= unused + await asyncio.gather(*(value.close() for value in unused)) + + def evolving(self, user_id: str) -> bool: + """Whether evolve holds this user, so their queued writes should wait.""" + return user_id in self._evolving + + @asynccontextmanager + async def exclusive_for_evolve(self, user_id: str) -> AsyncIterator[None]: + """Hold one user's writes off while evolve runs for them. + + New writes of the user are refused from here on; the ones already in + flight are waited for before the body runs. + """ + async with self._exclusive.setdefault(user_id, asyncio.Lock()): + self._evolving.add(user_id) + try: + async with self._writes_idle: + await self._writes_idle.wait_for(lambda: not self._writing.get(user_id)) + yield + finally: + self._evolving.discard(user_id) + + async def inquire_write_outcome(self, request_id: str) -> SdkWriteReceipt | None: async with self._active_handle() as handle: return await SdkWriteProcessor(handle).inquire_write_outcome(request_id) @@ -204,9 +279,11 @@ class SdkWriteCoordinator: @asynccontextmanager async def _active_handle(self) -> AsyncIterator[SdkClientHandle]: """Use the active profile's handle; retire the others, closing the unused.""" - identity, profile, secret = await asyncio.to_thread( - self._read_active_credentials + mode = self.memory_mode() + profile_identity, profile, secret = await asyncio.to_thread( + self._read_active_credentials, mode ) + identity = (*profile_identity, mode) async with self._lock: handle = self._handles.get(identity) if handle is None: @@ -215,6 +292,7 @@ class SdkWriteCoordinator: api_key=secret, data_dir=self.data_dir, http_transport_factory=self.transport_factory(profile), + mode=mode, ) self._handles[identity] = handle self._retired.update(value for key, value in self._handles.items() if key != identity) @@ -239,9 +317,14 @@ class SdkWriteCoordinator: def _read_active_credentials( self, + mode: str = "pro", ) -> tuple[tuple[str, int], ModelProfileSnapshot, str | None]: identity = self.profiles.active_identity() if identity is None: + if mode == "lite": + # lite stores raw turns with one embedding and calls no model. + builtin = builtin_retrieval_snapshot() + return (builtin.profile_id, builtin.revision), builtin, None raise RuntimeError("model_profile_not_active") profile = self.profiles.snapshot(*identity) try: diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.md index bbaf9c7..e4adde4 100644 --- a/memhub/apps/core/tests/TEST_INDEX.md +++ b/memhub/apps/core/tests/TEST_INDEX.md @@ -161,6 +161,7 @@ | `test_memory_evidence_freshness.py` | `contract` | 新鲜度拆除后：读路径恒为 current；证据修订 / 畸形 proof / segment ref 不再藏记忆或 503。 | | `test_memory_freshness_integration_review.py` | `integration` | Independent synthetic read-path review; no lifespan, model or user database. | | `test_memory_library_all_scenes.py` | `integration` | The v2 memory library across chat, working and coding memories. | +| `test_memory_mode_and_evolve.py` | `contract` | The memory mode (lite / pro / ultra), evolve and its nightly schedule. | | `test_memory_qa.py` | `contract` | 覆盖 search terms keep meaningful single chinese character、streamed answer emits recall then incremental grounded text、governance change during generation stops the old answer 等 14 个场景。 | | `test_memory_qa_api.py` | `integration` | 覆盖 ask stream endpoint emits recall deltas and final、ask stream endpoint requires the local token 场景。 | | `test_memory_qa_permissions.py` | `contract` | 覆盖 initial refusal precedes all secret access and network、invalid authorization result fails before secret access、authorization receives exact question and only outbound grounding 等 16 个场景。 | @@ -312,4 +313,4 @@ | `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. | | `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. | -共 307 个测试文件。 +共 308 个测试文件。 diff --git a/memhub/apps/core/tests/core_test_classification.py b/memhub/apps/core/tests/core_test_classification.py index 23d2315..226e871 100644 --- a/memhub/apps/core/tests/core_test_classification.py +++ b/memhub/apps/core/tests/core_test_classification.py @@ -14,8 +14,8 @@ from pathlib import Path import pytest PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live") -EXPECTED_TEST_FILE_COUNT = 307 -EXPECTED_TEST_FILES_SHA256 = "96a9ed11e358791723fb5ad827f72da2e9caeb673841119a5d9856a2d76fd5b1" +EXPECTED_TEST_FILE_COUNT = 308 +EXPECTED_TEST_FILES_SHA256 = "3aadf5953e41c01d4b60469f47242c00d2acef3c42550aa017255e31d216b589" # Pure transformations, validation models, parity checks and state machines. # This list is intentionally small: uncertain files default to contract. diff --git a/memhub/apps/core/tests/test_core_ready_before_recovery.py b/memhub/apps/core/tests/test_core_ready_before_recovery.py index 27e84a5..b324116 100644 --- a/memhub/apps/core/tests/test_core_ready_before_recovery.py +++ b/memhub/apps/core/tests/test_core_ready_before_recovery.py @@ -164,7 +164,7 @@ def test_metadata_repair_never_overlaps_an_sdk_client(tmp_path, monkeypatch): events.append("repair_finished") return MetadataRepairResult(None, 0, 0, 0) - def open_client(profile, *, api_key, data_dir, http_transport_factory): + def open_client(profile, *, api_key, data_dir, http_transport_factory, mode="pro"): events.append("client_opened") return ClientLease(object(), data_dir.resolve()) diff --git a/memhub/apps/core/tests/test_memory_mode_and_evolve.py b/memhub/apps/core/tests/test_memory_mode_and_evolve.py new file mode 100644 index 0000000..b84c102 --- /dev/null +++ b/memhub/apps/core/tests/test_memory_mode_and_evolve.py @@ -0,0 +1,861 @@ +"""The memory mode (lite / pro / ultra), evolve and its nightly schedule. + +Item 5: the mode is a saved setting (default pro, `ONE_MEMORY_GRAPH_STORE` +still forces ultra), it decides the mode every SDK client is built with, and +a change rebuilds the handles. lite needs no generation model, never calls +`add_extracted`, and keeps every raw. Item 6 (P1-3 / R7): `POST +/api/v2/evolve` is ultra only, one run per user at a time, never beside that +user's write, and loops while the SDK says `should_continue`; the schedule +runs nightly and catches up after sleep. R10: the status carries +`pipeline.mode` and the last evolve. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from httpx import ASGITransport, AsyncClient +from pydantic import SecretStr + +from one_memory import hy_sdk_runtime, sdk_write_processor +from one_memory.api import _sdk_client_handle, create_app, memory_mode_effects +from one_memory.config import Settings +from one_memory.hy_graph_admin import effective_memory_mode +from one_memory.imports.models import ( + EvidenceContentKind, + EvidenceInput, + EvidenceOrigin, + OriginMetadata, + WrittenBy, + materialize_write_input, +) +from one_memory.memory_evolution import ( + EvolveModeError, + EvolveScheduler, + EvolveService, + EvolveStateStore, + MemoryGraphBackfill, + evolve_due, +) +from one_memory.memory_server.config_state import ConfigStore, EvolveSchedule +from one_memory.sdk_memory_runtime import SdkMemoryRuntime +from one_memory.secrets import InMemorySecretStore + +TOKEN = "memory-mode-test" + + +# --- the setting and the client mode ----------------------------------------- + + +def test_mode_defaults_to_pro_and_the_env_still_forces_ultra(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("ONE_MEMORY_GRAPH_STORE", raising=False) + store = ConfigStore(tmp_path) + assert store.memory_mode() == "pro" + assert effective_memory_mode(store.memory_mode()) == "pro" + store.set_memory_mode("lite") + assert ConfigStore(tmp_path).memory_mode() == "lite" + assert json.loads((tmp_path / "config.json").read_text())["memory_mode"] == "lite" + monkeypatch.setenv("ONE_MEMORY_GRAPH_STORE", "kuzu") + assert effective_memory_mode("lite") == "ultra" + monkeypatch.setenv("ONE_MEMORY_GRAPH_STORE", "none") + assert effective_memory_mode("lite") == "lite" + with pytest.raises(ValueError): + store.set_memory_mode("max") # type: ignore[arg-type] + + +def test_an_untouched_install_writes_no_new_settings(tmp_path) -> None: + store = ConfigStore(tmp_path) + store.set_sync("cursor", True) + saved = json.loads((tmp_path / "config.json").read_text()) + assert "memory_mode" not in saved and "evolve_schedule" not in saved + + +def test_acquire_client_builds_the_sdk_client_in_the_given_mode(tmp_path, monkeypatch) -> None: + built: list[str] = [] + + class Client: + def __init__(self, *, config, mode, runtime, embedding_service): + built.append(mode) + + def close(self): + return None + + monkeypatch.setattr(hy_sdk_runtime, "HyMemoryClient", Client) + monkeypatch.setattr(hy_sdk_runtime, "_acquire_runtime", lambda config, key: object()) + monkeypatch.setattr(hy_sdk_runtime, "_release_runtime", lambda key: None) + monkeypatch.setattr(hy_sdk_runtime, "inspect_graph_width_mismatch", lambda config: None) + monkeypatch.delenv("ONE_MEMORY_GRAPH_STORE", raising=False) + profile = hy_sdk_runtime.builtin_retrieval_snapshot() + for mode in ("lite", "pro", "ultra"): + hy_sdk_runtime.acquire_client(profile, api_key=None, data_dir=tmp_path, mode=mode).close() + hy_sdk_runtime.acquire_client(profile, api_key=None, data_dir=tmp_path).close() + monkeypatch.setenv("ONE_MEMORY_GRAPH_STORE", "kuzu") + hy_sdk_runtime.acquire_client(profile, api_key=None, data_dir=tmp_path).close() + assert built == ["lite", "pro", "ultra", "pro", "ultra"] + + +def test_the_read_handle_rebuilds_on_a_mode_or_profile_change(monkeypatch, tmp_path) -> None: + """Reads follow the active profile's embedding (#255) and the memory mode.""" + from one_memory import sdk_read_handle + + modes = ["pro"] + made: list[_FakeHandle] = [] + + def handle_for(profile, *, api_key, data_dir, mode=None): + handle = _FakeHandle(mode or "pro", profile.profile_id) + made.append(handle) + return handle + + monkeypatch.setattr(sdk_read_handle, "SdkClientHandle", handle_for) + profiles = _Profiles(identity=None) + reads = sdk_read_handle.ActiveProfileReadHandle( + profiles=profiles, # type: ignore[arg-type] + secrets=None, # type: ignore[arg-type] + data_dir=tmp_path, + memory_mode=lambda: modes[0], + ) + + async def read(): + return await reads.client() + + async def scenario(): + first = await asyncio.create_task(read()) + assert (first.mode, first.profile_id) == ("pro", "builtin-retrieval") + modes[0] = "lite" + # A read still in flight on the old client keeps it open. + hold = asyncio.Event() + holding_client = asyncio.Event() + + async def holding(): + await reads.client() + holding_client.set() + await hold.wait() + + modes[0] = "pro" + holder = asyncio.create_task(holding()) + await asyncio.wait_for(holding_client.wait(), 5) + modes[0] = "lite" + try: + assert (await asyncio.create_task(read())).mode == "lite" + assert not made[0].closed + finally: + hold.set() + await holder + for _ in range(10): + await asyncio.sleep(0) + assert made[0].closed + assert [item.mode for item in made] == ["pro", "lite"] + await reads.close() + assert made[1].closed + + asyncio.run(scenario()) + + +class _FakeHandle: + def __init__(self, mode: str, profile_id: str = "p") -> None: + self.mode = mode + self.profile_id = profile_id + self.closed = False + + async def client(self): + return self + + async def close(self): + self.closed = True + + +class _Profiles: + def __init__(self, identity=("profile-a", 1)) -> None: + self.identity = identity + + def active_identity(self): + return self.identity + + def snapshot(self, profile_id, revision): + return SimpleNamespace(profile_id=profile_id, has_api_key=False) + + def resolve_snapshot_secret(self, profile_id, revision, secrets): + return None + + +def _coordinator(monkeypatch, modes, profiles=None): + opened: list[_FakeHandle] = [] + + def handle(profile, **kwargs): + value = _FakeHandle(kwargs.get("mode") or "pro", profile.profile_id) + opened.append(value) + return value + + monkeypatch.setattr(sdk_write_processor, "SdkClientHandle", handle) + coordinator = sdk_write_processor.SdkWriteCoordinator( + profiles=profiles or _Profiles(), # type: ignore[arg-type] + secrets=None, # type: ignore[arg-type] + data_dir=None, + transport_factory=lambda _profile: None, + sedimentation_enabled=lambda: True, + memory_mode=lambda: modes[0], + ) + return coordinator, opened + + +def test_a_mode_change_retires_idle_write_handles_and_keeps_a_busy_one(monkeypatch) -> None: + modes = ["pro"] + coordinator, opened = _coordinator(monkeypatch, modes) + + async def scenario(): + async with coordinator.use_client() as client: + assert client.mode == "pro" + modes[0] = "ultra" + await coordinator.retire_stale_modes() + # Still in use: closed only when this use ends (#218). + assert not opened[0].closed + assert opened[0].closed + async with coordinator.use_client() as client: + assert client.mode == "ultra" + modes[0] = "lite" + await coordinator.retire_stale_modes() + assert opened[1].closed + await coordinator.close() + + asyncio.run(scenario()) + + +def test_lite_needs_no_generation_model(monkeypatch) -> None: + modes = ["lite"] + coordinator, opened = _coordinator(monkeypatch, modes, _Profiles(identity=None)) + + async def scenario(): + async with coordinator.use_client() as client: + assert client.mode == "lite" + assert client.profile_id == hy_sdk_runtime.BUILTIN_RETRIEVAL_PROFILE_ID + modes[0] = "pro" + with pytest.raises(RuntimeError, match="model_profile_not_active"): + async with coordinator.use_client(): + pass + await coordinator.close() + + asyncio.run(scenario()) + + +def _record(text: str, *, extracted: bool, user_id: str = "local", role: str = "user"): + record = materialize_write_input( + EvidenceInput( + user_id=user_id, + source_id="one-memory-manual" if extracted else "hermes", + conversation_id="session-1", + message_id=str(uuid4()), + role=role, + written_by=WrittenBy.PERSON, + created_at=datetime(2026, 10, 1, tzinfo=UTC), + verbatim=text, + origin_meta={EvidenceOrigin.FILE: OriginMetadata()}, + ) + ) + if extracted: + record = record.model_copy(update={"content_kind": EvidenceContentKind.MEMORY_CONTENT}) + return record + + +class _WritingSdk: + def __init__(self) -> None: + self.calls: list[tuple[str, object]] = [] + + def add(self, messages, **kwargs): + self.calls.append(("add", messages)) + return {"success": True, "raw_ids": ["raw-1"]} + + def add_extracted(self, contents, **kwargs): + self.calls.append(("add_extracted", contents)) + return {"success": True, "memory_ids": ["m-1"]} + + +class _SdkHandle: + def __init__(self, sdk, mode="pro") -> None: + self.sdk = sdk + self.mode = mode + + async def client(self): + return self.sdk + + async def close(self): + return None + + +async def _noop() -> None: + return None + + +def test_a_note_in_lite_is_stored_as_raw_and_never_calls_add_extracted() -> None: + sdk = _WritingSdk() + note = (_record("I prefer green tea", extracted=True),) + + async def scenario(): + await sdk_write_processor.SdkWriteProcessor(_SdkHandle(sdk), lite=True).process_sdk_job( + note, request_id="r-lite", before_hy_invocation=_noop + ) + await sdk_write_processor.SdkWriteProcessor(_SdkHandle(sdk)).process_sdk_job( + note, request_id="r-pro", before_hy_invocation=_noop + ) + + asyncio.run(scenario()) + assert sdk.calls == [ + ("add", [{"role": "user", "content": "I prefer green tea"}]), + ("add_extracted", ["I prefer green tea"]), + ] + + +def test_a_lite_write_runs_through_the_coordinator_without_a_model(monkeypatch) -> None: + sdk = _WritingSdk() + coordinator = sdk_write_processor.SdkWriteCoordinator( + profiles=_Profiles(identity=None), # type: ignore[arg-type] + secrets=None, # type: ignore[arg-type] + data_dir=None, + transport_factory=lambda _profile: None, + sedimentation_enabled=lambda: True, + memory_mode=lambda: "lite", + ) + monkeypatch.setattr( + sdk_write_processor, "SdkClientHandle", lambda profile, **kwargs: _SdkHandle(sdk, "lite") + ) + turn = ( + _record("hello", extracted=False), + _record("hi there", extracted=False, role="assistant"), + ) + receipt = asyncio.run( + coordinator.process_sdk_job(turn, request_id="r-1", before_hy_invocation=_noop) + ) + assert receipt.raw_ids == ("raw-1",) + assert [name for name, _ in sdk.calls] == ["add"] + + +def test_no_raw_is_ever_deleted_in_any_mode() -> None: + """Owner decision: MemHub keeps every raw; there is no retention window.""" + from one_memory import sdk_memory_runtime + + for name in ("prune_expired_raw", "start_raw_retention", "_raw_retention_loop"): + assert not hasattr(SdkMemoryRuntime, name), name + assert not hasattr(sdk_memory_runtime, "RAW_RETENTION_DAYS") + + +def test_mode_switch_effects() -> None: + assert memory_mode_effects("lite", "pro") == ( + "earlier_raw_not_extracted", + "lite_search_stops", + ) + assert memory_mode_effects("ultra", "pro") == ("higher_mode_memories_hidden", "evolve_stopped") + assert memory_mode_effects("pro", "ultra", graph_backfill=True) == ( + "existing_memories_added_to_graph", + ) + assert memory_mode_effects("pro", "ultra", graph_backfill=False) == ( + "graph_backfill_already_running", + ) + assert memory_mode_effects("pro", "lite") == ("higher_mode_memories_hidden",) + assert memory_mode_effects("pro", "pro") == () + + +# --- evolve --------------------------------------------------------------------- + + +class _EvolveSdk: + """`evolve` answers `should_continue` for the first rounds, then stops.""" + + def __init__(self, continuing: int = 2, *, gate: asyncio.Event | None = None) -> None: + self.calls: list[dict] = [] + self.continuing = continuing + self.gate = gate + self.graph_builds: list[str] = [] + self.active = 0 + self.peak = 0 + + def evolve(self, user_id, agent_id="default_agent", max_batches=None): + self.active += 1 + self.peak = max(self.peak, self.active) + try: + self.calls.append( + {"user_id": user_id, "agent_id": agent_id, "max_batches": max_batches} + ) + more = len([call for call in self.calls if call["user_id"] == user_id]) <= ( + self.continuing + ) + return { + "success": True, + "request_id": f"evolve-{len(self.calls)}", + "tasks_processed": 2, + "results": {"schemas": ["private text"]}, + "cross_domain_sweeper": {"merged": 1}, + "should_continue": more, + "continuation_reason": "remaining_clusters" if more else None, + "remaining_clusters": 3 if more else 0, + "remaining_actionable_clusters": 1 if more else 0, + "remaining_fresh_facts": 0, + "cycle_status": "partial" if more else "complete", + "batch_failures": [], + "elapsed_ms": 5.0, + } + finally: + self.active -= 1 + + def build_memory_graph(self, user_id, *, agent_id=None, limit=10000, clear_first=False): + self.graph_builds.append(user_id) + return {"success": True, "memories_scanned": 4, "nodes": 4} + + +class _Coordinator: + """The real per-user exclusivity over a stand-in client.""" + + def __init__(self, sdk, monkeypatch) -> None: + self.inner, _ = _coordinator(monkeypatch, ["ultra"]) + self.sdk = sdk + + def exclusive_for_evolve(self, user_id): + return self.inner.exclusive_for_evolve(user_id) + + @asynccontextmanager + async def use_client(self): + yield self.sdk + + +def test_evolve_refuses_outside_ultra(tmp_path, monkeypatch) -> None: + service = EvolveService( + coordinator=_Coordinator(_EvolveSdk(), monkeypatch), + state=EvolveStateStore(tmp_path), + memory_mode=lambda: "pro", + ) + + async def scenario(): + with pytest.raises(EvolveModeError): + service.start("local") + + asyncio.run(scenario()) + + +def test_evolve_loops_while_should_continue_and_records_counts_only(tmp_path, monkeypatch) -> None: + sdk = _EvolveSdk(continuing=2) + state = EvolveStateStore(tmp_path) + service = EvolveService( + coordinator=_Coordinator(sdk, monkeypatch), state=state, memory_mode=lambda: "ultra" + ) + + async def scenario(): + started = service.start("wife", max_batches=2) + assert started.state == "running" + done = await service.wait(started.run_id) + assert done is not None + return done + + done = asyncio.run(scenario()) + assert done.state == "succeeded" and done.rounds == 3 and done.tasks_processed == 6 + assert done.cycle_status == "complete" and done.should_continue is False + assert {call["agent_id"] for call in sdk.calls} == {"all-apps"} + assert {call["max_batches"] for call in sdk.calls} == {2} + saved = (tmp_path / "evolve-state.json").read_text() + assert "private text" not in saved + assert json.loads(saved)["users"]["wife"]["last_result"]["rounds"] == 3 + + +def test_evolve_stops_at_the_round_cap(tmp_path, monkeypatch) -> None: + sdk = _EvolveSdk(continuing=100) + service = EvolveService( + coordinator=_Coordinator(sdk, monkeypatch), + state=EvolveStateStore(tmp_path), + memory_mode=lambda: "ultra", + max_rounds=4, + ) + + async def scenario(): + return await service.wait(service.start("local").run_id) + + done = asyncio.run(scenario()) + assert done is not None and done.rounds == 4 and done.capped is True + assert done.state == "succeeded" + + +def test_one_run_per_user_and_never_beside_that_users_write(tmp_path, monkeypatch) -> None: + sdk = _EvolveSdk(continuing=0) + coordinator = _Coordinator(sdk, monkeypatch) + service = EvolveService( + coordinator=coordinator, state=EvolveStateStore(tmp_path), memory_mode=lambda: "ultra" + ) + writing = asyncio.Event() + release = asyncio.Event() + + async def fake_write(self, records, *, request_id, before_hy_invocation): + writing.set() + await release.wait() + return sdk_write_processor.SdkWriteReceipt() + + monkeypatch.setattr(sdk_write_processor.SdkWriteProcessor, "process_sdk_job", fake_write) + record = (_record("hello", extracted=False, user_id="wife"),) + + async def scenario(): + inner = coordinator.inner + write = asyncio.create_task( + inner.process_sdk_job(record, request_id="w", before_hy_invocation=_noop) + ) + await writing.wait() + first = service.start("wife") + again = service.start("wife") + assert again.run_id == first.run_id + other = service.start("local") + await service.wait(other.run_id) + # The other user is not held; wife's run waits for wife's write. + assert sdk.calls and {call["user_id"] for call in sdk.calls} == {"local"} + assert inner.evolving("wife") + with pytest.raises(RuntimeError, match="memory_evolve_in_progress"): + await inner.process_sdk_job(record, request_id="w2", before_hy_invocation=_noop) + release.set() + await write + done = await service.wait(first.run_id) + assert done is not None and done.state == "succeeded" + assert not inner.evolving("wife") + assert sdk.peak == 1 + + asyncio.run(scenario()) + + +def test_a_failed_round_records_its_code(tmp_path, monkeypatch) -> None: + class Failing(_EvolveSdk): + def evolve(self, user_id, agent_id="default_agent", max_batches=None): + return { + "success": False, + "error_code": "schema_llm_failed", + "failure_kind": "llm_error", + "error_message": "the provider said something private", + } + + state = EvolveStateStore(tmp_path) + service = EvolveService( + coordinator=_Coordinator(Failing(), monkeypatch), state=state, memory_mode=lambda: "ultra" + ) + done = asyncio.run(service.wait(_start(service, "local"))) + assert done is not None and done.state == "failed" + assert done.error_code == "schema_llm_failed" and done.failure_kind == "llm_error" + assert state.user("local").last_error == "schema_llm_failed" + assert "private" not in (tmp_path / "evolve-state.json").read_text() + + +def _start(service: EvolveService, user_id: str) -> str: + async def begin() -> str: + return service.start(user_id).run_id + + loop = asyncio.new_event_loop() + try: + run_id = loop.run_until_complete(begin()) + loop.run_until_complete(service.wait(run_id)) + finally: + loop.close() + return run_id + + +# --- the nightly schedule ------------------------------------------------------- + +LOCAL = datetime.now().astimezone().tzinfo + + +def _at(day: int, hour: int, minute: int = 0) -> datetime: + return datetime(2026, 11, day, hour, minute, tzinfo=LOCAL) + + +def test_evolve_is_due_after_the_slot_once_a_day_and_catches_up_after_sleep() -> None: + schedule = EvolveSchedule() + anchor = _at(1, 12) + # Never ran: not at noon on the day ultra was enabled, then at 03:00 next day. + assert not evolve_due(_at(1, 13), schedule, None, anchor) + assert not evolve_due(_at(2, 2, 59), schedule, None, anchor) + assert evolve_due(_at(2, 3, 0), schedule, None, anchor) + # Ran at 03:00: not again that day, due again the next night. + assert not evolve_due(_at(2, 22), schedule, _at(2, 3, 1), anchor) + assert evolve_due(_at(3, 3, 5), schedule, _at(2, 3, 1), anchor) + # Asleep at 03:00: a run older than the slot is due on wake, the same day. + assert evolve_due(_at(3, 9), schedule, _at(2, 3, 1), anchor) + # Last run more than 24h ago is due even before today's slot. + assert evolve_due(_at(4, 1), schedule, _at(2, 3, 1), anchor) + assert not evolve_due(_at(3, 1), schedule, _at(2, 3, 1), anchor) + # Moved and turned off. + assert evolve_due(_at(2, 23, 30), EvolveSchedule(time="23:30"), _at(2, 3), anchor) + assert not evolve_due(_at(3, 9), EvolveSchedule(enabled=False), _at(1, 3), anchor) + + +def test_the_scheduler_runs_each_due_user_once_in_ultra_only(tmp_path, monkeypatch) -> None: + sdk = _EvolveSdk(continuing=0) + state = EvolveStateStore(tmp_path) + modes = ["pro"] + clock = [_at(1, 12)] + service = EvolveService( + coordinator=_Coordinator(sdk, monkeypatch), + state=state, + memory_mode=lambda: modes[0], + now=lambda: clock[0], + ) + schedule = [EvolveSchedule()] + + async def users(): + return ("local", "wife") + + scheduler = EvolveScheduler( + service=service, + schedule=lambda: schedule[0], + users_with_memories=users, + now=lambda: clock[0], + ) + + async def scenario(): + assert await scheduler.tick() == () # pro: nothing + modes[0] = "ultra" + assert await scheduler.tick() == () # anchored at noon + clock[0] = _at(2, 3, 2) + assert await scheduler.tick() == ("local", "wife") + assert await scheduler.tick() == () # once a night + clock[0] = _at(3, 8) # slept through 03:00 + assert await scheduler.tick() == ("local", "wife") + schedule[0] = EvolveSchedule(enabled=False) + clock[0] = _at(5, 8) + assert await scheduler.tick() == () + + asyncio.run(scenario()) + assert [call["user_id"] for call in sdk.calls] == ["local", "wife", "local", "wife"] + assert state.user("wife").last_result["trigger"] == "scheduled" + + +def test_the_graph_backfill_runs_once_per_user(tmp_path, monkeypatch) -> None: + sdk = _EvolveSdk() + state = EvolveStateStore(tmp_path) + backfill = MemoryGraphBackfill(coordinator=_Coordinator(sdk, monkeypatch), state=state) + + async def scenario(): + backfill.start(["local", "wife", "local"]) + await backfill.wait() + + asyncio.run(scenario()) + assert sdk.graph_builds == ["local", "wife"] + built = state.graph_build() + assert built.state == "succeeded" + assert {user: value.state for user, value in built.users.items()} == { + "local": "succeeded", + "wife": "succeeded", + } + + +# --- the HTTP surface ----------------------------------------------------------- + + +def application(tmp_path: Path): + settings = Settings( + data_dir=tmp_path / "data", + source_home=tmp_path, + api_token=SecretStr(TOKEN), + imports_enabled=False, + automatic_memory_processing=False, + live_sync_enabled=False, + ) + return create_app(settings, secret_store=InMemorySecretStore()) + + +def client_for(app) -> AsyncClient: + return AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) + + +class _ReadSdk(_EvolveSdk): + def memory_stats(self, *, user_id, statuses=None): + return {"total": 1, "scenes": {}} + + +def run_app(tmp_path, monkeypatch, scenario) -> None: + monkeypatch.delenv("ONE_MEMORY_GRAPH_STORE", raising=False) + + async def main() -> None: + app = application(tmp_path) + sdk = _ReadSdk(continuing=1) + + @asynccontextmanager + async def write_client(): + yield sdk + + async with app.router.lifespan_context(app): + monkeypatch.setattr(app.state.sdk_write_coordinator, "use_client", write_client) + async with client_for(app) as client: + await scenario(app, sdk, client) + + asyncio.run(main()) + + +@pytest.mark.fresh_sdk +def test_mode_api_get_put_and_its_effects(tmp_path, monkeypatch) -> None: + async def scenario(app, sdk, client): + current = (await client.get("/api/v1/settings/memory-mode")).json() + assert current["mode"] == current["configured_mode"] == "pro" + assert current["forced_by_env"] is False + assert app.state.sdk_write_coordinator.memory_mode() == "pro" + + lite = await client.put("/api/v1/settings/memory-mode", json={"mode": "lite"}) + assert lite.status_code == 200, lite.text + body = lite.json() + assert body["mode"] == "lite" and body["previous_mode"] == "pro" + assert body["effects"] == ["higher_mode_memories_hidden"] + assert app.state.sdk_write_coordinator.memory_mode() == "lite" + assert _sdk_client_handle(app).memory_mode() == "lite" + # lite claims queued work without a generation model. + from one_memory.api import _sedimentation_claim_enabled + + assert app.state.model_profile_store.active_identity() is None + app.state.history_import_gate_released = True + assert _sedimentation_claim_enabled(app, "one-memory-manual") + + ultra = await client.put("/api/v1/settings/memory-mode", json={"mode": "ultra"}) + assert ultra.status_code == 200, ultra.text + assert "existing_memories_added_to_graph" in ultra.json()["effects"] + await app.state.memory_graph_backfill.wait() + assert sdk.graph_builds == ["local"] + reread = (await client.get("/api/v1/settings/memory-mode")).json() + assert reread["graph_build"]["state"] == "succeeded" + + for bad in ({"mode": "max"}, {"mode": "pro", "x": 1}): + assert (await client.put("/api/v1/settings/memory-mode", json=bad)).status_code == 422 + assert ( + await client.get("/api/v1/settings/memory-mode", params={"user_id": "x"}) + ).status_code == 422 + + monkeypatch.setenv("ONE_MEMORY_GRAPH_STORE", "kuzu") + await client.put("/api/v1/settings/memory-mode", json={"mode": "pro"}) + forced = (await client.get("/api/v1/settings/memory-mode")).json() + assert forced["mode"] == "ultra" and forced["configured_mode"] == "pro" + assert forced["forced_by_env"] is True + + run_app(tmp_path, monkeypatch, scenario) + + +@pytest.mark.fresh_sdk +def test_evolve_api_is_ultra_only_async_and_scoped_per_user(tmp_path, monkeypatch) -> None: + async def scenario(app, sdk, client): + refused = await client.post("/api/v2/evolve", json={}) + assert refused.status_code == 409 + assert refused.json()["detail"] == "memory_mode_not_ultra" + + await client.put("/api/v1/settings/memory-mode", json={"mode": "ultra"}) + await app.state.memory_graph_backfill.wait() + started = await client.post( + "/api/v2/evolve", json={"user_id": "wife", "agent_id": "all-apps", "max_batches": 2} + ) + assert started.status_code == 202, started.text + run = started.json() + assert run["state"] == "running" and run["user_id"] == "wife" + await app.state.evolve_service.wait(run["run_id"]) + path = f"/api/v2/evolve/{run['run_id']}" + assert (await client.get(path)).status_code == 404 + done = (await client.get(path, params={"user_id": "wife"})).json() + assert done["state"] == "succeeded" and done["rounds"] == 2 + assert sdk.calls[0] == {"user_id": "wife", "agent_id": "all-apps", "max_batches": 2} + + overview = (await client.get("/api/v2/evolve", params={"user_id": "wife"})).json() + assert set(overview["users"]) == {"wife"} + assert overview["schedule"] == {"enabled": True, "time": "03:00"} + + processing = (await client.get("/api/v1/memory-processing")).json() + pipeline = processing["pipeline"] + assert pipeline["mode"] == "ultra" + assert pipeline["last_evolve_at"] is not None + assert pipeline["evolve"]["users"]["wife"]["last_result"]["state"] == "succeeded" + + for bad in ( + {"user_id": "a:b"}, + {"max_batches": 0}, + {"from_beginning": True}, + ): + assert (await client.post("/api/v2/evolve", json=bad)).status_code == 422 + + moved = await client.put( + "/api/v1/settings/evolve-schedule", json={"time": "04:30", "enabled": False} + ) + assert moved.json() == {"enabled": False, "time": "04:30"} + assert ( + await client.put("/api/v1/settings/evolve-schedule", json={"time": "25:00"}) + ).status_code == 422 + + run_app(tmp_path, monkeypatch, scenario) + + +@pytest.mark.fresh_sdk +def test_a_job_written_in_lite_stays_done_and_a_pending_job_takes_the_new_mode( + tmp_path, monkeypatch +) -> None: + """lite to pro reprocesses nothing: the mode is read when a job runs.""" + from one_memory.memory_server.durable_queue import DurableWriteDrain, execute_job + + modes = ["lite"] + seen: list[str] = [] + + class Sdk: + def __init__(self, mode): + self.mode = mode + + def add(self, messages, **kwargs): + seen.append(self.mode) + return {"success": True, "raw_ids": [f"raw-{len(seen)}"]} + + monkeypatch.setattr( + sdk_write_processor, + "SdkClientHandle", + lambda profile, **kwargs: _SdkHandle(Sdk(kwargs["mode"]), kwargs["mode"]), + ) + coordinator = sdk_write_processor.SdkWriteCoordinator( + profiles=_Profiles(), # type: ignore[arg-type] + secrets=None, # type: ignore[arg-type] + data_dir=None, + transport_factory=lambda _profile: None, + sedimentation_enabled=lambda: True, + memory_mode=lambda: modes[0], + ) + + async def never(_job_id): + return None + + async def drain_once(queue) -> int: + drain = DurableWriteDrain(queue, execute=never, lease=timedelta(seconds=30)) + claimed = drain.claim(limit=8) + for job_id in claimed: + await execute_job( + job_id, + queue=queue, + supervisor=coordinator, + evidence=None, # type: ignore[arg-type] + ) + return len(claimed) + + async def scenario(app, _sdk, client): + queue = app.state.memory_write_queue + for request_id in ("turn-lite", "turn-pro"): + written = await client.post( + "/api/v2/memory-writes", + json={ + "request_id": request_id, + "session_id": f"s-{request_id}", + "source_id": "one-memory-manual", + "messages": [ + {"role": "user", "content": request_id}, + {"role": "assistant", "content": "ok"}, + ], + }, + ) + assert written.status_code == 202, written.text + if request_id == "turn-lite": + assert await drain_once(queue) == 1 + modes[0] = "pro" + await coordinator.retire_stale_modes() + assert await drain_once(queue) == 1 + assert await drain_once(queue) == 0 + assert queue.get("turn-lite").state == queue.get("turn-pro").state == "completed" + assert seen == ["lite", "pro"] + await coordinator.close() + + run_app(tmp_path, monkeypatch, scenario) diff --git a/memhub/apps/core/tests/test_sdk_data_plane_migration.py b/memhub/apps/core/tests/test_sdk_data_plane_migration.py index 89de51e..3b46953 100644 --- a/memhub/apps/core/tests/test_sdk_data_plane_migration.py +++ b/memhub/apps/core/tests/test_sdk_data_plane_migration.py @@ -6,7 +6,7 @@ import io import json import sqlite3 from contextlib import closing -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from pathlib import Path import pytest @@ -30,7 +30,6 @@ from one_memory.sdk_data_plane_migration import ( MemoryAuthority, SdkDataPlaneMigrationStore, ) -from one_memory.sdk_memory_runtime import SdkMemoryRuntime from one_memory.sdk_migration_bundle import create_sdk_migration_bundle from one_memory.sdk_migration_runner import ( PREPARED_PHASE, @@ -1350,57 +1349,6 @@ def test_raw_http_dto_uses_the_unified_memory_list_shape(): assert page.raw[0].custom == {"role": "user"} -def test_sdk_raw_retention_keeps_only_the_latest_three_days(): - now = datetime(2026, 10, 23, tzinfo=UTC) - - class Client: - def __init__(self): - self.rows = [ - { - "memory_id": "old", - "memory_at": int((now - timedelta(days=4)).timestamp()), - }, - { - "memory_id": "recent", - "memory_at": int((now - timedelta(days=2)).timestamp()), - }, - ] - - def list_memories(self, **_kwargs): - return { - "vdb": { - "memories": list(self.rows), - "total": len(self.rows), - "limit": 500, - "offset": 0, - } - } - - def delete(self, memory_id): - before = len(self.rows) - self.rows = [row for row in self.rows if row["memory_id"] != memory_id] - return { - "success": True, - "deleted_count": before - len(self.rows), - } - - client = Client() - - class Handle: - async def client(self): - return client - - runtime = SdkMemoryRuntime( - coordinator=None, - read_handle=Handle(), - write_queue=None, - extraction_settings=None, - data_dir=Path("."), - ) - assert asyncio.run(runtime.prune_expired_raw(now=now)) == 1 - assert [row["memory_id"] for row in client.rows] == ["recent"] - - # The UI asks which memory data plane is authoritative and must get the # answer the API gates use. diff --git a/memhub/apps/core/tests/test_search_without_profile.py b/memhub/apps/core/tests/test_search_without_profile.py index 864316a..ae45adc 100644 --- a/memhub/apps/core/tests/test_search_without_profile.py +++ b/memhub/apps/core/tests/test_search_without_profile.py @@ -140,7 +140,7 @@ class _Recorder: self.opened: list[tuple[str, int | None, str | None]] = [] self.closed: list[str] = [] - def __call__(self, profile, *, api_key, data_dir, http_transport_factory=None): + def __call__(self, profile, *, api_key, data_dir, http_transport_factory=None, mode="pro"): from one_memory.hy_sdk_runtime import build_sdk_config config = build_sdk_config(profile, api_key=api_key, data_dir=data_dir) diff --git a/memhub/docs/ARCHITECTURE.md b/memhub/docs/ARCHITECTURE.md index ab6a6c8..a514655 100644 --- a/memhub/docs/ARCHITECTURE.md +++ b/memhub/docs/ARCHITECTURE.md @@ -130,7 +130,7 @@ This exception does not extend to VDB, Coding, prompt, reconcile, or lifecycle. - Storage backends：仅 SDK config/env（Chroma/FAISS/Qdrant、Kuzu/Neo4j、cache/history sqlite|mysql、coding sqlite 等）。 - Queue：pending `memory_write_jobs` 只表示尚未调用 SDK 的待处理请求，因此没有 SDK raw/memory ID。worker 单次调用 SDK `add()`；SDK 内部完成 raw 写入与 extraction/reconciliation，成功后 MemHub 仅将返回的 raw ID / memory IDs 记作 job outcome。 -- Raw retention：SDK L1 raw 只保留最近三天，由 Runtime 的 `prune_expired_raw` 执行；每个完整 turn 只保存真人 user 与 assistant，tool 不进入 raw。高层 Chat/Coding/Working memory 不受该窗口影响。 +- Raw retention：MemHub 不删除任何 SDK L1 raw，任何模式都全部保留（SDK 自身也不删 raw）；每个完整 turn 只保存真人 user 与 assistant，tool 不进入 raw。记忆模式（lite / pro / ultra）与 evolve 见 `docs/contracts/memory-mode-evolve-v1.md`。 - Recall：Host Bridge / Core 先做宿主安装凭据与 tool grant，再调 **SDK search/get**。当前产品没有逐 memory 的 A-harness→B-harness 可见性开关，默认允许跨 harness 搜索；未来若增加来源限制，直接透传 SDK `source` / `include_sources` / `exclude_sources`，不在 MemHub 建 memory ACL。 - Context wire：cutover marker 提交前只使用 legacy Context v1；提交后 Host Bridge 只使用 SDK-native Context v2。两种 schema 原样校验、原样返回，不把 SDK item 补造成 v1 的 `revision`、`evidence_ids`、`paths` 或 `adoption`，已选择的 v2 路径失败时也不回退 legacy 权威。 - Ask wire：marker 前保留 Governance Ask v1；marker 后 Web / Host Bridge 只调用 SDK Ask v2。v2 的引用只含 SDK 原生 `memory_id/content/layer/status/source/gmt_created`，生成过程中重复 `get` 校验同一 SDK 快照，不补造 revision、Evidence 或 scopes，SDK 失败不回退 v1。 diff --git a/memhub/docs/cli/EXPERIENCE.md b/memhub/docs/cli/EXPERIENCE.md index a2233a4..b57bd7f 100644 --- a/memhub/docs/cli/EXPERIENCE.md +++ b/memhub/docs/cli/EXPERIENCE.md @@ -152,6 +152,20 @@ node apps/cli/dist/one-memory.cjs profile --versions --user-id wife --json 用于找出覆盖了正确值的错误版本（例如 `用户` / `The user`），再用 `memories delete --yes` 删除；对应 `GET /api/v2/user-profile/versions`。 +### 记忆模式与进化（`settings mode` / `evolve`） + +```bash +node apps/cli/dist/one-memory.cjs settings mode # 当前模式 +node apps/cli/dist/one-memory.cjs settings mode ultra # 只显示切换计划，不改动 +node apps/cli/dist/one-memory.cjs settings mode ultra --yes # 切换 +node apps/cli/dist/one-memory.cjs evolve --user-id wife --wait # 立即进化（仅 ultra） +``` + +- 切换不迁移数据：已处理的对话不会重做，排队中的和新的按新模式处理；正在写的一批按旧模式写完。 +- 任何模式都保留全部原始对话。 +- `status --json` 与 `progress --json` 输出 `pipeline.mode`、每晚进化时间和每个用户的上次进化。 +- 契约见 `docs/contracts/memory-mode-evolve-v1.md`。 + ## 8. 召回（recall） `recall` 两条路径（raw / memory）此前已真机验证。文本输出会标明每条是「原始对话摘录」 diff --git a/memhub/docs/contracts/memory-mode-evolve-v1.md b/memhub/docs/contracts/memory-mode-evolve-v1.md new file mode 100644 index 0000000..5e09d3e --- /dev/null +++ b/memhub/docs/contracts/memory-mode-evolve-v1.md @@ -0,0 +1,346 @@ +# Memory mode, evolve and the nightly schedule v1 + +Status: Core + CLI implemented (PR B of the SDK-surface batch). The App +controls are built separately against this contract. + +All routes need the Runtime bearer token. Unknown query parameters and body +fields answer `422 {"detail": "request_validation_failed"}`, like every +memory route. `user_id` is optional (default `local`) and must match +`^[A-Za-z0-9._-]{1,64}$`. The App never sends it. + +## 1. What the modes are + +| Mode | SDK client | What happens to a conversation | What search reads | +|---|---|---|---| +| `lite` | `mode="lite"` | Stored as one raw turn with one embedding. No model call; no generation model needs to be configured. | Raw turns only (the SDK forces `L1_RAW`). | +| `pro` (default) | `mode="pro"` | Stored as raw, then a model extracts memories (System 1). The raw becomes SHADOW once extraction succeeds. | Extracted memories (facts, profile, intentions). | +| `ultra` | `mode="ultra"` | Same as pro, and new facts are also projected into the knowledge graph. Every night, evolve runs System 2 for every user. | Same as pro, plus the graph-backed lanes below. | + +The product difference between pro and ultra is the nightly evolve. MemHub ultra +uses SDK `mode="ultra"` because `client.evolve` refuses any other mode. Compared +with pro, SDK ultra (hy-memory 2.0.9) also turns on the following, so the owner +can judge their cost: + +- **The graph store.** Kuzu is always opened in ultra. Client init fails if the + graph cannot open, so `PUT ultra` answers `409 graph_store_unavailable` when + `kuzu` is missing. In pro it opens only when `MEMORY_ENABLE_GRAPH` is set, and + MemHub does not set it. Writes project new L2 facts into the graph. +- **The schema lane (L6_SCHEMA).** The search's schema channel runs only when + a graph store exists, so in MemHub only in ultra. It returns the schemas that + evolve wrote (`schema_limit`, `schema_min_score`). +- **Graph-aware ranking on the normal lane.** With a graph store, search adds + graph-entity (MENTIONS) boosts when the query names known entities and no + source filter is set. The heavier graph stages need the graph *and* their + own env flag, and every flag defaults off, so none of them run in MemHub: + graph rerank and episode expansion (`MEMORY_GRAPH_RERANK_ENABLED`), topic + graph entry, same-series bundle, hierarchy bundle. +- **The profile.** `get_user_profile` appends the balanced catalog and the + specific supplement from the graph, when the SDK's balanced-catalog flag is + on. + +`ONE_MEMORY_GRAPH_STORE` (development only) still forces ultra whatever the +setting says. Any value other than empty or `none` does this. The API reports +it as `forced_by_env: true`, with `configured_mode` showing the saved choice. + +**Raw is never deleted.** In every mode MemHub keeps every raw turn the SDK +stored, and the SDK never deletes raw itself. There is no retention window. + +## 2. Switching mode: what changes and what does not + +The SDK migrates nothing when the mode changes. MemHub applies these rules, +and they are the owner's: + +- The mode is read **when a queued job runs**, not when it was queued. + - A job that already completed, in any mode, stays completed. It is never + reclaimed or reprocessed. + - Jobs still waiting (pending, assembling, waiting) and new input run in + the new mode. +- **A batch in flight finishes in the mode it started in.** Its SDK client is + retired only when its last use ends (the #218 use count). The next claim + uses the new mode. +- **Search follows the new mode at once.** The read client is rebuilt on the + next request (the active-profile read handle is keyed by profile and mode); the client it replaces closes once no request still uses it. +- **lite → pro/ultra:** + - what lite stored stays raw and is not extracted; + - lite's raw-only search stops at once; + - nothing is re-ingested or backfilled. +- **pro/ultra → lite:** memories made by the higher mode stay on disk, but lite + search does not serve them. Nothing is deleted. +- **ultra → pro/lite:** the nightly evolve stops. +- **→ ultra:** + - every known user's existing facts are added to the graph once + (`build_memory_graph(user_id, limit=10000, clear_first=False)`); + - this runs in the background, one user at a time, never beside that + user's write; + - progress is reported in `graph_build`. +- Vector collection names never depend on the mode. The mode is a separate + client parameter, not part of the embedding-space identity. + +## 3. `GET /api/v1/settings/memory-mode` + +No parameters. `200` response: + +```json +{ + "mode": "pro", + "configured_mode": "pro", + "forced_by_env": false, + "modes": ["lite", "pro", "ultra"], + "graph_build": { + "state": "idle", + "started_at": null, + "finished_at": null, + "users": {} + } +} +``` + +- `mode` is what SDK clients are built with now; `configured_mode` is the + saved setting. +- `graph_build.state` is `idle | running | succeeded | failed`. Each entry in + `graph_build.users[]` is + `{state: pending|running|succeeded|failed, memories_scanned, nodes, error_code}`. + +## 4. `PUT /api/v1/settings/memory-mode` + +Body: `{"mode": "lite" | "pro" | "ultra"}`. `200` response: the GET shape, +plus these fields: + +```json +{ + "previous_mode": "pro", + "effects": ["existing_memories_added_to_graph"] +} +``` + +`effects` is empty when the effective mode did not change. That happens when +the mode was already set, or when `ONE_MEMORY_GRAPH_STORE` forces ultra; the +saved setting still changes in that case. Otherwise `effects` lists these, in +this order: + +| Effect | When | Meaning | +|---|---|---| +| `earlier_raw_not_extracted` | from lite | What lite stored stays raw; it is not extracted. | +| `lite_search_stops` | from lite | lite's raw-only search ends now; search reads memories. | +| `higher_mode_memories_hidden` | to a lower mode | Memories of the higher mode stay stored but are not shown. | +| `evolve_stopped` | from ultra | The nightly evolve stops. | +| `existing_memories_added_to_graph` | to ultra | The one-time graph backfill started (see `graph_build`). | +| `graph_backfill_already_running` | to ultra | A backfill from an earlier switch is still running; no second one started. | + +Errors: + +- `409 {"detail": "graph_store_unavailable"}`: `ultra` was asked for, but the + graph store package is not installed. Nothing changed. +- `422 request_validation_failed`: an unknown mode or an extra field. + +Show the planned effects before calling PUT. Clients can compute them with the +table above: the CLI (`memhub settings mode ` without `--yes`) and Core +(`memory_mode_effects`) use the same rule. + +## 5. Evolve + +### `POST /api/v2/evolve` → `202` + +Body (every field optional): + +```json +{"user_id": "local", "agent_id": "all-apps", "max_batches": 3} +``` + +- `agent_id` is 1..128 characters. The default is `all-apps`, the agent + MemHub writes every memory under. +- `max_batches` is 1..1000. The default is the SDK's own. + +The response is the run receipt below, with `state: "running"`. The run goes +on in the background. + +- **One run per user.** Asking while that user's run is going answers the same + receipt, again `202`. +- **Never beside that user's writes.** The run waits for the user's write in + flight. While it runs, that user's queued jobs wait: they are returned to + pending before any SDK call. Other users are not held. +- **Loops while the SDK answers `should_continue: true`**, up to 20 rounds. A + run stopped by that cap has `capped: true`; start it again to continue. + +Errors: + +- `409 {"detail": "memory_mode_not_ultra"}`: the effective mode is not ultra. +- `422`: a malformed `user_id`, `max_batches` outside its range, or an extra + field. + +### Run receipt + +```json +{ + "run_id": "uuid", + "user_id": "local", + "agent_id": "all-apps", + "max_batches": null, + "trigger": "manual", + "state": "succeeded", + "started_at": "2026-11-01T03:00:00Z", + "finished_at": "2026-11-01T03:00:42Z", + "rounds": 2, + "tasks_processed": 5, + "batch_failures": 0, + "cycle_status": "complete", + "should_continue": false, + "continuation_reason": null, + "remaining_clusters": 0, + "remaining_actionable_clusters": 0, + "remaining_fresh_facts": 0, + "capped": false, + "elapsed_ms": 41873.2, + "error_code": null, + "failure_kind": null, + "request_id": "sdk-request-id", + "results": {}, + "cross_domain_sweeper": {} +} +``` + +- `trigger` is `manual` or `scheduled`. +- `state` is `running`, `succeeded` or `failed`. +- `cycle_status` is the SDK's `partial` or `complete`. +- `error_code` and `failure_kind` are stable codes on failure. Examples: + `memory_evolve_cancelled` (the Runtime stopped mid-run), + `hy_memory_evolve_rejected` (the SDK answered `success: false` without a + code), or the SDK's own `error_code`. Provider messages never appear. +- `results` and `cross_domain_sweeper` are the last round's SDK bodies. They + live in memory only and are never written to disk or diagnostics. + +### `GET /api/v2/evolve/{run_id}?user_id=` → receipt + +Returns `404 {"detail": "evolve_run_not_found"}` in two cases: + +- the run belongs to another user (answered exactly like a missing run); +- the Runtime restarted since the run. Receipts live in memory, and the last + 64 are kept. + +### `GET /api/v2/evolve?user_id=` → overview + +```json +{ + "mode": "ultra", + "schedule": {"enabled": true, "time": "03:00"}, + "users": { + "local": { + "last_run_at": "2026-11-01T03:00:00Z", + "last_result": {"state": "succeeded", "rounds": 2, "...": "receipt without results"}, + "last_error": null + } + }, + "running": [] +} +``` + +- Without `user_id` the overview covers every user; with it, that one user. +- `users` survives restarts. It is kept in `/evolve-state.json`, + counts and codes only. + +## 6. Nightly schedule + +### `GET /api/v1/settings/evolve-schedule` + +Returns `{"enabled": true, "time": "03:00"}`. `time` is `HH:MM`, 24-hour, in +the machine's local time. + +### `PUT /api/v1/settings/evolve-schedule` + +Body: `{"enabled"?: bool, "time"?: "HH:MM"}`. An absent field keeps its +value. The response is the saved schedule. A bad time answers `422`. + +How the schedule runs: + +- It runs only in ultra. +- It checks every 5 minutes. A user's run is due when either holds: + - today's slot has passed, and the user's last run is older than that slot; + - the user's last run is more than 24 hours old. + + The second rule is how a machine that slept through 03:00 catches up when + it wakes. +- A user who never ran is measured from the first time the schedule ran in + ultra, not from the epoch. So switching to ultra at noon does not evolve + everyone at once; their first run is at the next slot. +- Users run one after another. + +The schedule runs every user that has memories. The SDK cannot list users, so +Core takes them from its own records: + +- `local`; +- every user holding a source grant (a Hermes profile); +- every user whose turns or notes passed the write queue. + +A user is kept when `memory_stats` reports any memory or fact for them. + +## 7. Status fields (R10) + +`GET /api/v1/memory-processing` has a new optional field, `pipeline`: + +```json +{ + "pipeline": { + "mode": "ultra", + "configured_mode": "ultra", + "forced_by_env": false, + "evolve": {"...": "the overview of section 5"}, + "last_evolve_at": "2026-11-01T03:00:00Z", + "graph_build": {"...": "as in section 3"} + } +} +``` + +`memhub status --json` adds a top-level projection of it: + +```json +{ + "pipeline": { + "mode": "ultra", + "configured_mode": "ultra", + "forced_by_env": false, + "last_evolve_at": "...", + "evolve_schedule": {"enabled": true, "time": "03:00"}, + "evolve_running": 0, + "evolve_users": { + "local": {"last_run_at": "...", "last_error": null, "state": "succeeded"} + }, + "graph_build": "succeeded" + } +} +``` + +`memhub progress --json` carries the same `pipeline`. Both are `null` against +an older Core. + +## 8. CLI + +```bash +memhub settings mode # show the current mode +memhub settings mode lite # print the plan, change nothing +memhub settings mode lite --yes # switch +memhub evolve [--user-id ] [--agent-id ] [--max-batches ] [--wait] [--json] +``` + +## 9. Copy the UI can reuse + +| Key | zh-CN | en | es | +|---|---|---|---| +| title | 记忆模式 | Memory mode | Modo de memoria | +| lite | lite：不调用模型，保留全部原始对话，只在原始对话里检索。 | lite: no model calls. Every original conversation is kept, and search reads only those conversations. | lite: sin llamadas al modelo. Se conservan todas las conversaciones originales y la búsqueda solo las consulta a ellas. | +| pro | pro：由模型从对话中提取记忆（System 1）。 | pro: a model extracts memories from your conversations (System 1). | pro: un modelo extrae memorias de tus conversaciones (System 1). | +| ultra | ultra：在 pro 之上每晚自动整合记忆（System 2，evolve），并使用知识图谱。 | ultra: pro plus nightly memory consolidation (System 2, evolve), using a knowledge graph. | ultra: pro más una consolidación nocturna de las memorias (System 2, evolve), con un grafo de conocimiento. | +| confirm | 切换到 {mode} 前请确认： | Before switching to {mode}, please confirm: | Antes de cambiar a {mode}, confirma lo siguiente: | +| earlier_raw_not_extracted | lite 下已处理完的对话不会再被提取；只有尚未处理的和之后的新对话会由模型整理成记忆。 | Conversations already handled in lite are not extracted again; only conversations still waiting, and new ones, become memories. | Las conversaciones ya procesadas en lite no se vuelven a extraer; solo las pendientes y las nuevas se convierten en memorias. | +| lite_search_stops | 只在原始对话里的检索会立即停止，改为检索记忆。 | Search over original conversations stops now; search reads memories instead. | La búsqueda en las conversaciones originales se detiene ahora; se buscará en las memorias. | +| higher_mode_memories_hidden | 更高模式生成的记忆仍然保存，但在这个模式下不会显示。 | Memories created by a higher mode stay stored but are not shown in this mode. | Las memorias creadas por un modo superior se conservan, pero no se muestran en este modo. | +| evolve_stopped | 每晚的自动整合（System 2）会停止。 | Nightly consolidation (System 2) stops. | La consolidación nocturna (System 2) se detiene. | +| existing_memories_added_to_graph | 已有记忆会在后台加入知识图谱。 | Your existing memories are added to the knowledge graph in the background. | Tus memorias actuales se añaden al grafo de conocimiento en segundo plano. | +| graph_backfill_already_running | 知识图谱正在补建中。 | The knowledge graph is still being built. | El grafo de conocimiento todavía se está construyendo. | +| raw_kept | 原始对话全部保留，任何模式都不删除。 | Every original conversation is kept, in every mode. | Se conservan todas las conversaciones originales, en cualquier modo. | +| graph_build_running | 正在把已有记忆加入知识图谱… | Adding existing memories to the knowledge graph… | Añadiendo las memorias actuales al grafo de conocimiento… | +| forced_by_env | 开发环境变量 ONE_MEMORY_GRAPH_STORE 正在强制使用 ultra。 | The development variable ONE_MEMORY_GRAPH_STORE is forcing ultra. | La variable de desarrollo ONE_MEMORY_GRAPH_STORE está forzando ultra. | +| graph_store_unavailable | 这个安装缺少知识图谱组件，无法切换到 ultra。 | This installation lacks the knowledge-graph component, so it cannot switch to ultra. | A esta instalación le falta el componente del grafo de conocimiento; no puede cambiar a ultra. | +| schedule | 每天 {time} 自动整合 | Consolidates daily at {time} | Consolida cada día a las {time} | +| last_evolve | 上次整合：{time} | Last consolidation: {time} | Última consolidación: {time} | +| never_evolved | 尚未整合 | Not consolidated yet | Aún no consolidado | +| evolve_not_ultra | 记忆整合只在 ultra 模式下可用。 | Consolidation is available in ultra mode only. | La consolidación solo está disponible en el modo ultra. | diff --git a/memhub/docs/contracts/sdk-data-plane-migration-v1.md b/memhub/docs/contracts/sdk-data-plane-migration-v1.md index 83aec8f..451d9b2 100644 --- a/memhub/docs/contracts/sdk-data-plane-migration-v1.md +++ b/memhub/docs/contracts/sdk-data-plane-migration-v1.md @@ -26,8 +26,7 @@ and no raw turn, and does not compare the mirror with the SDK: cannot be trusted to decide what the SDK should contain; - the product has no user editing of memories in practice, so the mirror holds no user intent the SDK lacks; -- SDK L1 raw is retained for three days by the Runtime's own - `prune_expired_raw`, not by the migration. +- SDK L1 raw is kept: neither the migration nor the Runtime deletes raw. "Already complete" holds only if the new Runtime opens the same stores. Builds before #145 named a profile's vector collections `one_memory_<12 hex>…`; #145