feat(memhub): draw the memory map from SDK memories on SDK authority

The default memory view is the map, and on SDK authority it had nothing
to draw: its list and graph read only legacy authority. It now reads SDK
memories (`/api/v2/memory-library`) and `/api/v2/memories/graph` and
draws them with the same map model, so the map stays the default view.
Opening a memory from the map goes to the SDK library, which reads,
edits and deletes it through v2; governed detail is refused there.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -4,6 +4,12 @@ Hy Memory and MemHub are versioned and released independently.
 
 ## Unreleased
 
+- Fixed legacy-to-SDK startup migration to adopt native Coding Experience and
+  Working Fact IDs instead of importing them as Chat layers. Raw migration now
+  rebuilds bounded user/assistant turns from the latest three days, excludes
+  tools, and enforces the same rolling retention after cutover.
+- Paused host capture writes with a live Desktop migration fence; Cursor
+  History catch-up resumes from its existing checkpoint after cutover.
 - Removed MemHub's private `hy_memory_bridge` implementation. MemHub now calls
   public `HyMemoryClient` methods at runtime; prompt/reconcile/response
   monkey-patches and private VDB/Coding runtime access were deleted. Kuzu Graph

memhub/MEMORY.md

@@ -13,6 +13,8 @@
    接口先补 SDK。Runtime writers 停止后的 App 启动迁移可直接操作底层存储，
    但不得把迁移专用方法加入公共 Client。
 3. MemHub **不存 memory 实体**；**可以存 `memory_id`** 做组织/回执。原文进入 **SDK raw**。
+4. SDK raw 只保留滚动三天；一条 raw 对应一个完整 turn，只含真人 user 与
+   assistant，tool 不持久化。提取后的高层 memory 不受该窗口影响。
 
 生产入口通过共享 Runtime 租约取得公开 `HyMemoryClient`；MemHub 只在调用前执行
 模型配置、密钥和 harness/model 门禁，不维护第二套 memory 正文权威。

memhub/apps/cli/src/cli-logs.test.ts

@@ -1,6 +1,7 @@
-import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import { inflateRawSync } from "node:zlib";
 import { DiagnosticLogger } from "@one-memory/runtime-control";
 import { afterEach, expect, it, vi } from "vitest";
 
@@ -29,6 +30,27 @@ async function fixture() {
   return dataDir;
 }
 
+function readZipEntry(archive: Buffer, expected: string): string {
+  let offset = 0;
+  while (archive.readUInt32LE(offset) === 0x04034b50) {
+    const method = archive.readUInt16LE(offset + 8);
+    const compressedSize = archive.readUInt32LE(offset + 18);
+    const nameLength = archive.readUInt16LE(offset + 26);
+    const extraLength = archive.readUInt16LE(offset + 28);
+    const nameStart = offset + 30;
+    const dataStart = nameStart + nameLength + extraLength;
+    const name = archive.subarray(nameStart, nameStart + nameLength).toString();
+    const compressed = archive.subarray(dataStart, dataStart + compressedSize);
+    if (name === expected) {
+      if (method === 0) return compressed.toString();
+      if (method === 8) return inflateRawSync(compressed).toString();
+      throw new Error("unsupported zip compression");
+    }
+    offset = dataStart + compressedSize;
+  }
+  throw new Error(`missing ${expected}`);
+}
+
 it("prints matching diagnostic events as NDJSON without starting Runtime", async () => {
   const dataDir = await fixture();
   let stdout = "";
@@ -74,6 +96,25 @@ it("exports one support zip to the requested downloads directory", async () => {
   });
   expect(receipt.path).toMatch(/MemHub-日志-3800-\d{8}-\d{4}\.zip$/);
   expect(await readdir(downloads)).toEqual([path.basename(receipt.path)]);
+  const archive = await readFile(receipt.path);
+  expect(JSON.parse(readZipEntry(archive, "summary.json")).commit).toBeNull();
+});
+
+it("fails closed for an explicitly supplied invalid commit", async () => {
+  const dataDir = await fixture();
+  const downloads = await mkdtemp(path.join(os.tmpdir(), "om-cli-downloads-"));
+  roots.push(downloads);
+  vi.spyOn(process.stdout, "write").mockImplementation(() => true);
+
+  await expect(
+    run(["logs", "export", "--json"], {
+      ONE_MEMORY_DATA_DIR: dataDir,
+      ONE_MEMORY_EXPORT_DIR: downloads,
+      ONE_MEMORY_BUILD_NUMBER: "3800",
+      ONE_MEMORY_COMMIT: "unknown",
+    }),
+  ).rejects.toThrow("diagnostic_commit_invalid");
+  expect(await readdir(downloads)).toEqual([]);
 });
 
 it("never emits a secret from a malformed manually written event", async () => {

memhub/apps/cli/src/cli.test.ts

@@ -57,6 +57,11 @@ const runtimeMocks = vi.hoisted(() => ({
   })),
   uninstallUserService: vi.fn(async (..._args: unknown[]) => "removed"),
   restartUserService: vi.fn(async () => undefined),
+  runSdkMigration: vi.fn(
+    async (): Promise<{ state: string; code?: string }> => ({
+      state: "not_required",
+    }),
+  ),
   rollbackLocalInstall: vi.fn(async () => undefined),
   planCliLink: vi.fn(async ({ home, installRoot }) => ({
     schema_version: 1,
@@ -126,6 +131,7 @@ vi.mock("@one-memory/runtime-control", async (importOriginal) => {
     installUserService: runtimeMocks.installUserService,
     uninstallUserService: runtimeMocks.uninstallUserService,
     restartUserService: runtimeMocks.restartUserService,
+    runSdkMigration: runtimeMocks.runSdkMigration,
     rollbackLocalInstall: runtimeMocks.rollbackLocalInstall,
     planCliLink: runtimeMocks.planCliLink,
     installCliLink: runtimeMocks.installCliLink,
@@ -889,6 +895,108 @@ describe("public One Memory CLI", () => {
       expect(runtimeMocks.restartUserService).toHaveBeenCalledTimes(2);
     });
 
+    const upgradeArguments = [
+      "upgrade",
+      "--source",
+      "/tmp/dist",
+      "--home",
+      "/tmp/home",
+      "--install-root",
+      "/tmp/install",
+      "--data-dir",
+      "/tmp/custom data",
+      "--yes",
+    ];
+    const presentCliLink = {
+      schema_version: 1 as const,
+      platform: "linux" as const,
+      home: "/tmp/home",
+      install_root: "/tmp/install",
+      bin_directory: "/tmp/bin",
+      status: "present" as const,
+      blocked_reason: null,
+      link_path: "/tmp/bin/one-memory",
+      target_path: "/tmp/install/current/bin/one-memory",
+    };
+
+    it("migrates the home with the new Core before restarting the Runtime", async () => {
+      vi.clearAllMocks();
+      runtimeMocks.planCliLink.mockResolvedValueOnce(presentCliLink);
+      const order: string[] = [];
+      runtimeMocks.runSdkMigration.mockImplementationOnce(async () => {
+        order.push("migrate");
+        return { state: "cutover" };
+      });
+      runtimeMocks.restartUserService.mockImplementationOnce(async () => {
+        order.push("restart");
+      });
+      const output = captureStdout();
+      let code: number;
+      try {
+        code = await run(
+          upgradeArguments,
+          {},
+          { waitForRuntimeReady: async () => true },
+        );
+      } finally {
+        output.restore();
+      }
+      expect(code!).toBe(0);
+      expect(order).toEqual(["migrate", "restart"]);
+      expect(runtimeMocks.runSdkMigration).toHaveBeenCalledWith(
+        expect.objectContaining({ dataDir: "/tmp/custom data" }),
+      );
+    });
+
+    it("rolls back to the previous version when the migration does not finish", async () => {
+      vi.clearAllMocks();
+      runtimeMocks.planCliLink.mockResolvedValueOnce(presentCliLink);
+      runtimeMocks.runSdkMigration.mockResolvedValueOnce({
+        state: "deferred",
+        code: "sdk_migration_disk_space_insufficient",
+      });
+      const streams = captureStreams();
+      let code: number;
+      try {
+        code = await run(
+          upgradeArguments,
+          {},
+          { waitForRuntimeReady: async () => true },
+        );
+      } finally {
+        streams.restore();
+      }
+      expect(code!).toBe(1);
+      expect(JSON.parse(streams.stderr())).toMatchObject({
+        command: "upgrade",
+        code: "sdk_migration_disk_space_insufficient",
+      });
+      expect(runtimeMocks.rollbackLocalInstall).toHaveBeenCalledOnce();
+    });
+
+    it("keeps the new version when a migrated home is slow to come up", async () => {
+      vi.clearAllMocks();
+      runtimeMocks.planCliLink.mockResolvedValueOnce(presentCliLink);
+      runtimeMocks.runSdkMigration.mockResolvedValueOnce({ state: "cutover" });
+      const streams = captureStreams();
+      let code: number;
+      try {
+        code = await run(
+          upgradeArguments,
+          {},
+          { waitForRuntimeReady: async () => false },
+        );
+      } finally {
+        streams.restore();
+      }
+      expect(code!).toBe(1);
+      expect(JSON.parse(streams.stderr())).toMatchObject({
+        command: "upgrade",
+        code: "runtime_not_ready_after_sdk_migration",
+      });
+      expect(runtimeMocks.rollbackLocalInstall).not.toHaveBeenCalled();
+    });
+
     it("reports success once the upgraded Runtime is healthy", async () => {
       vi.clearAllMocks();
       runtimeMocks.planCliLink.mockResolvedValueOnce({

memhub/apps/cli/src/cli.ts

@@ -39,7 +39,9 @@ import {
   rollbackCliLink,
   rollbackLocalInstall,
   runRuntimeCli,
+  runSdkMigration,
   runtimeInstallLayout,
+  type SdkMigrationResult,
   stopRuntimeProcess,
 } from "@one-memory/runtime-control";
 import axios from "axios";
@@ -354,6 +356,9 @@ export interface RunControls {
   /** Wait for a restarted Runtime to become healthy after `upgrade`.
    * Injectable so the upgrade tests do not spend real seconds polling. */
   waitForRuntimeReady?: (dataDir: string) => Promise<boolean>;
+  /** Run the SDK data migration before install/upgrade starts the new
+   * Runtime. Injectable so the tests do not spawn a real Core. */
+  migrateSdkData?: typeof migrateSdkDataBeforeStart;
   restartManagedRuntime?: typeof restartManagedService;
   stopRuntime?: typeof stopRuntimeProcess;
   ensureRuntime?: typeof ensureRuntimeProcess;
@@ -1048,15 +1053,21 @@ async function runCommand(
         entry: "cli",
       }));
       const target = path.join(downloads, diagnosticZipFilename(build));
+      const environmentCommit = environment.ONE_MEMORY_COMMIT?.trim();
+      const embeddedCommit =
+        typeof __ONE_MEMORY_COMMIT__ === "string"
+          ? __ONE_MEMORY_COMMIT__.trim()
+          : "";
+      const commit =
+        environmentCommit ||
+        (embeddedCommit && embeddedCommit !== "unknown"
+          ? embeddedCommit
+          : undefined);
       const receipt = await exportDiagnosticZip({
         dataDir: config.dataDir,
         target,
         productVersion: build,
-        commit:
-          environment.ONE_MEMORY_COMMIT?.trim() ||
-          (typeof __ONE_MEMORY_COMMIT__ === "string"
-            ? __ONE_MEMORY_COMMIT__
-            : "unknown"),
+        ...(commit === undefined ? {} : { commit }),
         coreSha256: await sha256File(config.coreCommand).catch(() => undefined),
         home: os.homedir(),
         crashReportsDir: path.join(
@@ -3143,6 +3154,7 @@ async function runCommand(
       null;
     let installReceiptWritten = false;
     let serviceApplied = false;
+    let migrated = false;
     let previousOwner:
       | Awaited<ReturnType<typeof claimCliServiceOwnership>>
       | undefined;
@@ -3153,6 +3165,23 @@ async function runCommand(
       }
       shellPathReceipt = await applyShellPath(shellPath);
       previousOwner = await claimCliServiceOwnership(layout.dataDir);
+      // `install` over an existing home (reinstall, App-to-CLI takeover) must
+      // migrate it too, before the new Runtime starts serving it.
+      const migration = await (
+        controls.migrateSdkData ?? migrateSdkDataBeforeStart
+      )({
+        coreCommand: layout.coreCommand,
+        dataDir: layout.dataDir,
+        runtimeBuild: String(plan.build_number ?? plan.version),
+        asJson: true,
+      });
+      if (migration.state === "deferred") {
+        throw new SdkMigrationDeferred(
+          migration.code ?? "sdk_migration_incomplete",
+        );
+      }
+      migrated =
+        migration.state === "cutover" || migration.state === "complete";
       await applyPreparedServiceInstall(preparedService);
       serviceApplied = true;
       // The receipt exists so a later command can find the data dir without
@@ -3204,6 +3233,13 @@ async function runCommand(
           );
         }
       };
+      if (migrated) {
+        // An older Runtime refuses a migrated home: keep this install.
+        process.stderr.write(
+          `${JSON.stringify(notReadyAfterMigration("install"), null, 2)}\n`,
+        );
+        return EXIT_FAILED;
+      }
       if (serviceApplied) {
         await rollbackPreparedServiceInstall(
           preparedService,
@@ -3213,7 +3249,13 @@ async function runCommand(
         await rollbackFilesAndOwner();
       }
       process.stderr.write(
-        `${JSON.stringify(serviceErrorDocument("install", error), null, 2)}\n`,
+        `${JSON.stringify(
+          error instanceof SdkMigrationDeferred
+            ? sdkMigrationFailure("install", error.code)
+            : serviceErrorDocument("install", error),
+          null,
+          2,
+        )}\n`,
       );
       return EXIT_FAILED;
     }
@@ -3575,6 +3617,7 @@ async function runCommand(
     let previousOwner:
       | Awaited<ReturnType<typeof claimCliServiceOwnership>>
       | undefined;
+    let migrated = false;
     const rollbackUpgrade = async (
       dataDir: string | undefined,
     ): Promise<void> => {
@@ -3606,6 +3649,25 @@ async function runCommand(
         memhubReceipts.push(await installManagedCliLink(link));
       }
       previousOwner = await claimCliServiceOwnership(layout.dataDir);
+      const migration = await (
+        controls.migrateSdkData ?? migrateSdkDataBeforeStart
+      )({
+        coreCommand: layout.coreCommand,
+        dataDir: parsed.dataDir ?? layout.dataDir,
+        runtimeBuild: String(plan.build_number ?? plan.version),
+        asJson,
+      });
+      if (migration.state === "deferred") {
+        await rollbackUpgrade(parsed.dataDir);
+        process.stderr.write(
+          `${JSON.stringify(sdkMigrationFailure("upgrade", migration.code ?? "sdk_migration_incomplete"), null, 2)}\n`,
+        );
+        return EXIT_FAILED;
+      }
+      // An older Runtime refuses a migrated home, so from here a failure
+      // keeps the new version instead of rolling back to one that cannot start.
+      migrated =
+        migration.state === "cutover" || migration.state === "complete";
       await restartUserService(layout, serviceOptions);
       // A restart that returns without error only means launchd/systemd
       // accepted the command. O-22: the first `upgrade` implementation
@@ -3617,6 +3679,12 @@ async function runCommand(
       const ready = await (controls.waitForRuntimeReady ?? waitForRuntimeReady)(
         dataDir,
       );
+      if (!ready && migrated) {
+        process.stderr.write(
+          `${JSON.stringify(notReadyAfterMigration("upgrade"), null, 2)}\n`,
+        );
+        return EXIT_FAILED;
+      }
       if (!ready) {
         await rollbackUpgrade(parsed.dataDir);
         process.stderr.write(
@@ -3644,9 +3712,9 @@ async function runCommand(
         });
       }
     } catch (error) {
-      await rollbackUpgrade(parsed.dataDir);
+      if (!migrated) await rollbackUpgrade(parsed.dataDir);
       process.stderr.write(
-        `${JSON.stringify(serviceErrorDocument("upgrade", error), null, 2)}\n`,
+        `${JSON.stringify(migrated ? notReadyAfterMigration("upgrade") : serviceErrorDocument("upgrade", error), null, 2)}\n`,
       );
       return EXIT_FAILED;
     }
@@ -4014,6 +4082,79 @@ export function runtimeIsReady(
   );
 }
 
+const SDK_MIGRATION_STEPS: Record<string, string> = {
+  bundle: "正在备份记忆数据，数据较多时需要几分钟……",
+  prepare: "正在整理待处理的写入任务……",
+  cutover: "正在切换记忆数据源……",
+  cleanup: "正在清理旧数据表……",
+};
+
+/**
+ * Migrate a home written by an older release before the new Runtime serves it.
+ *
+ * Runs the new Runtime's own Core with the Runtime stopped. Progress goes to
+ * stderr so stdout stays one result document.
+ */
+async function migrateSdkDataBeforeStart(options: {
+  coreCommand: string;
+  dataDir: string;
+  runtimeBuild: string;
+  asJson: boolean;
+}): Promise<SdkMigrationResult> {
+  return runSdkMigration({
+    dataDir: options.dataDir,
+    core: { command: options.coreCommand },
+    runtimeBuild: options.runtimeBuild,
+    onProgress: (step) => {
+      process.stderr.write(
+        options.asJson
+          ? `${JSON.stringify({ schema_version: 1, product: PRODUCT, event: "sdk_migration.progress", step })}\n`
+          : `${SDK_MIGRATION_STEPS[step] ?? step}\n`,
+      );
+    },
+  });
+}
+
+class SdkMigrationDeferred extends Error {
+  constructor(readonly code: string) {
+    super(code);
+  }
+}
+
+function notReadyAfterMigration(command: "install" | "upgrade") {
+  return {
+    schema_version: 1,
+    product: PRODUCT,
+    command,
+    status: "error",
+    code: "runtime_not_ready_after_sdk_migration",
+    message:
+      "记忆数据已迁移到新版本，但新版本 Runtime 没有在限定时间内就绪。旧版本无法打开已迁移的数据，所以没有回滚。请运行 memhub restart，或 memhub doctor 查看原因。",
+  };
+}
+
+const SDK_MIGRATION_FAILURES: Record<string, string> = {
+  sdk_migration_disk_space_insufficient:
+    "磁盘空间不足，记忆数据迁移没有完成，已回滚到上一版本。请腾出与记忆数据同等大小的空间后重试；原有记忆数据不会丢失。",
+  sdk_migration_memory_unreadable:
+    "核对发现新版本还读不到你的全部记忆，没有切换数据源，已回滚到上一版本。请把这个错误码反馈给我们；原有记忆数据不会丢失。",
+  sdk_migration_model_profile_required:
+    "迁移需要先有可用的模型配置，用来核对新版本能否读回记忆。已回滚到上一版本；请先用 memhub models setup 完成配置后再升级。",
+};
+
+function sdkMigrationFailure(command: "install" | "upgrade", code: string) {
+  return {
+    schema_version: 1,
+    product: PRODUCT,
+    command,
+    status: "error",
+    code,
+    message:
+      SDK_MIGRATION_FAILURES[code] ??
+      "记忆数据迁移没有完成，已回滚到上一版本。请重试；如果仍然失败，请把这个错误码反馈给我们。原有记忆数据不会丢失。",
+  };
+}
+
 async function waitForRuntimeReady(
   dataDir: string,
   timeoutMs = 45_000,
@@ -4730,10 +4871,10 @@ async function runMaintenanceRestore(options: {
   const instanceId = randomUUID();
   const child = spawn(options.coreCommand, ["serve", "--maintenance"], {
     env: {
-      // Do not pass the caller's Runtime tuning variables (for example
-      // retrieval-index flags) into a maintenance process: a restore must
-      // start the product's minimal maintenance configuration, not inherit
-      // an unrelated experiment from the shell.
+      // Do not pass the caller's Runtime tuning variables into a
+      // maintenance process: a restore must start the product's minimal
+      // maintenance configuration, not inherit an unrelated experiment from
+      // the shell.
       PATH: process.env.PATH,
       HOME: process.env.HOME,
       LANG: process.env.LANG,

memhub/apps/cli/src/log-report.test.ts

@@ -1,4 +1,4 @@
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, rm, symlink } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { DiagnosticLogger } from "@one-memory/runtime-control";
@@ -41,11 +41,15 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
     session_id: "11111111-1111-4111-8111-111111111111",
     details: {
       count: 1,
+      reason: "/Users/me/private-query.txt",
+      status: "sk-live-secret-token",
+      source_id: "credential.private",
       prompt: "secret-token-123",
       authorization: "Bearer secret-token-123",
     } as unknown as Record<string, string | number | boolean | null>,
     safe_stack: [
       "packages/runtime-control/src/example.ts:42:7",
+      "packages/runtime/sk-live-secret-token.ts:1:1",
       "/Users/me/private.py:1",
       "C:\\Users\\me\\private.py:1",
     ],
@@ -56,6 +60,8 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
     "packages/runtime-control/src/example.ts:42:7",
   ]);
   expect(JSON.stringify(redacted)).not.toContain("secret-token-123");
+  expect(JSON.stringify(redacted)).not.toContain("sk-live-secret-token");
+  expect(JSON.stringify(redacted)).not.toContain("credential.private");
   expect(JSON.stringify(redacted)).not.toContain("/Users/me");
   expect(JSON.stringify(redacted)).not.toContain("C:\\\\Users");
 });
@@ -132,3 +138,27 @@ it("follows a newly appended event without stopping Runtime", async () => {
   });
   expect(seen).toEqual(["service.ready", "core.failed"]);
 });
+
+it("fails follow mode when the diagnostic directory is unsafe", async () => {
+  const dataDir = await mkdtemp(
+    path.join(os.tmpdir(), "om-cli-follow-unsafe-"),
+  );
+  const external = await mkdtemp(
+    path.join(os.tmpdir(), "om-cli-follow-external-"),
+  );
+  roots.push(dataDir, external);
+  await symlink(external, path.join(dataDir, "logs"));
+  const query = parseLogQuery(["--follow"]);
+  expect(query).not.toBeNull();
+  if (!query) throw new Error("query_missing");
+  await expect(
+    followLogs({
+      dataDir,
+      query,
+      signal: new AbortController().signal,
+      maxEvents: 1,
+      emit: () => undefined,
+      wait: async () => undefined,
+    }),
+  ).rejects.toThrow("log_directory_unreadable");
+});

memhub/apps/cli/src/log-report.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 import type { DiagnosticEventV1 } from "@one-memory/runtime-control";
 import {
   diagnosticLogFiles,
+  projectDiagnosticEventForSafeOutput,
   readDiagnosticEvents,
 } from "@one-memory/runtime-control";
 
@@ -24,54 +25,6 @@ export interface LogQuery {
 
 const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
 const NAME = /^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$/;
-const SAFE_DETAIL_KEYS = new Set([
-  "action",
-  "attempt",
-  "count",
-  "method",
-  "mode",
-  "phase",
-  "platform",
-  "reason",
-  "result_count",
-  "source_id",
-  "spawned",
-  "status",
-  "version",
-  "parent_session_id",
-  "invalid_lines",
-  "truncated",
-  "runtime_state",
-  "core_state",
-  "broker_state",
-  "instance_id",
-  "open_file_count",
-  "memory_bytes",
-  "thread_count",
-  "database_error_count",
-  "file_count",
-  "sqlite_file_count",
-  "byte_file_count",
-  "total_bytes",
-  "copied_bytes",
-  "bytes_per_second",
-  "stored_count",
-  "revised_count",
-  "skipped_count",
-  "error_count",
-  "pending_count",
-  "prepared_count",
-  "queue_depth",
-  "in_flight_count",
-  "call_count",
-  "created_count",
-  "updated_count",
-  "superseded_count",
-  "deleted_count",
-  "operation_kind",
-  "file_kind",
-]);
-
 export function parseLogQuery(arguments_: readonly string[]): LogQuery | null {
   if (arguments_.filter((item) => item === "--json").length > 1) return null;
   const values: Partial<LogQuery> = { errorsOnly: false, follow: false };
@@ -237,63 +190,13 @@ export function renderLogText(
   return `${lines.join("\n")}\n`;
 }
 
-/** Defence in depth for the CLI read surface. `DiagnosticLogger` already
- * allow-lists keys, but a future writer, a manually edited log, or a malformed
- * line must not turn `memhub logs` into an exfiltration path. */
+/** Defence in depth for the CLI read surface. Local writers accept arbitrary
+ * bounded scalar keys, so every outward read surface must reuse the same
+ * field-and-value projection as the support-bundle exporter. */
 export function redactDiagnosticEvent(
   event: DiagnosticEventV1,
 ): DiagnosticEventV1 {
-  const redacted: DiagnosticEventV1 = {
-    schema_version: 1,
-    timestamp: event.timestamp,
-    level: event.level,
-    service: event.service,
-    component: event.component,
-    event: event.event,
-    outcome: event.outcome,
-    session_id: event.session_id,
-    ...(event.trace_id === undefined ? {} : { trace_id: event.trace_id }),
-    ...(event.incident_id === undefined
-      ? {}
-      : { incident_id: event.incident_id }),
-    ...(event.instance_id === undefined
-      ? {}
-      : { instance_id: event.instance_id }),
-    ...(event.operation_id === undefined
-      ? {}
-      : { operation_id: event.operation_id }),
-    ...(event.request_id === undefined ? {} : { request_id: event.request_id }),
-    ...(event.job_id === undefined ? {} : { job_id: event.job_id }),
-    ...(event.task_id === undefined ? {} : { task_id: event.task_id }),
-    ...(event.duration_ms === undefined
-      ? {}
-      : { duration_ms: event.duration_ms }),
-    ...(event.state_before === undefined
-      ? {}
-      : { state_before: event.state_before }),
-    ...(event.state_after === undefined
-      ? {}
-      : { state_after: event.state_after }),
-    ...(event.error_code === undefined ? {} : { error_code: event.error_code }),
-    ...(event.error_type === undefined ? {} : { error_type: event.error_type }),
-    ...(event.safe_stack === undefined
-      ? {}
-      : {
-          safe_stack: event.safe_stack.filter(
-            (frame) =>
-              !frame.startsWith("/") &&
-              !/^[A-Za-z]:\\/.test(frame) &&
-              !frame.includes(".."),
-          ),
-        }),
-  };
-  const details = Object.fromEntries(
-    Object.entries(event.details ?? {}).filter(([key]) =>
-      SAFE_DETAIL_KEYS.has(key),
-    ),
-  );
-  if (Object.keys(details).length > 0) redacted.details = details;
-  return redacted;
+  return projectDiagnosticEventForSafeOutput(event);
 }
 
 export async function followLogs(options: {
@@ -315,6 +218,7 @@ export async function followLogs(options: {
       ...options.query,
       tail: 5_000,
     });
+    if (report.failure) throw new Error(report.failure);
     for (const event of report.events) {
       const identity = JSON.stringify(event);
       if (seen.has(identity)) continue;

memhub/apps/core/pyproject.toml

@@ -13,7 +13,8 @@ dependencies = [
     "filelock>=3.32.4",
     # Public SDK version validated against MemHub's permission and recall contracts.
     "hy-memory==2.0.5",
-    # Retrieval's derived vector index directly consumes these locked versions.
+    # chromadb arrives through the HY SDK; locked here so an unrelated upgrade
+    # cannot move the SDK's store. Built-in embedding consumes numpy directly.
     "chromadb==1.5.9",
     "numpy==2.5.2",
     "httpx>=0.28.1",

memhub/apps/core/src/one_memory/api.py

@@ -841,8 +841,6 @@ def create_app(
     secret_store: SecretStore | None = None,
 ) -> FastAPI:
     runtime_settings = settings or load_settings()
-    if runtime_settings.retrieval_index_enabled and not manage_runtime:
-        raise ValueError("retrieval_runtime_ownership_required")
 
     @asynccontextmanager
     async def lifespan(_: FastAPI) -> AsyncIterator[None]:
@@ -998,9 +996,7 @@ def create_app(
                 app.state.recall_target = GovernedRecallTarget(
                     app.state.memory_governance_store,
                     filter_memories=app.state.source_read_access.memories,
-                    semantic_search=lambda query: _sdk_memory_runtime(
-                        app
-                    ).semantic_recall(query),
+                    semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
                     # Read per request rather than cached: an import that lands
                     # while the app is open should make its workspace reachable
                     # without a restart. Affordable only because the read is served
@@ -1102,9 +1098,7 @@ def create_app(
                 authorize_generation=lambda profile, question, grounding: (
                     _authorize_answer_generation(app, profile, question, grounding)
                 ),
-                semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(
-                    query
-                ),
+                semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
                 resolve_workspace=lambda workspace_path: resolve_workspace_scopes(
                     workspace_path,
                     evidence_store.known_workspaces(user_id="local"),
@@ -1128,21 +1122,21 @@ def create_app(
                 profiles=app.state.model_profile_store,
                 secrets=app.state.secret_store,
                 data_dir=runtime_settings.data_dir,
-                transport_factory=lambda profile: lambda: AuditedModelTransport(
-                    app.state.model_request_audit_store,
-                    profile_id=profile.profile_id,
-                    profile_revision=profile.revision,
-                    provider_kind=str(profile.provider_kind),
-                    model=profile.model,
-                    inner=(
-                        BudgetedModelTransport(model_request_budget)
-                        if model_request_budget is not None
-                        else None
-                    ),
+                transport_factory=lambda profile: (
+                    lambda: AuditedModelTransport(
+                        app.state.model_request_audit_store,
+                        profile_id=profile.profile_id,
+                        profile_revision=profile.revision,
+                        provider_kind=str(profile.provider_kind),
+                        model=profile.model,
+                        inner=(
+                            BudgetedModelTransport(model_request_budget)
+                            if model_request_budget is not None
+                            else None
+                        ),
+                    )
                 ),
-                sedimentation_enabled=lambda: _config_store(
-                    app
-                ).sedimentation_enabled(),
+                sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(),
             )
             app.state.sdk_memory_runtime = SdkMemoryRuntime(
                 coordinator=app.state.sdk_write_coordinator,
@@ -1151,6 +1145,8 @@ def create_app(
                 extraction_settings=app.state.extraction_settings_store,
                 data_dir=runtime_settings.data_dir,
             )
+            if app.state.memory_authority is MemoryAuthority.SDK:
+                app.state.sdk_memory_runtime.start_raw_retention()
             # C2 owns the single production budget (handle 5s + reserve 250ms,
             # total 5.25s, lease 10s, renew 1s, claim 250ms). The composition
             # root only consumes it: no local formula, and the 250ms claim
@@ -1165,10 +1161,8 @@ def create_app(
                     exclude_error_codes=NON_RETRYABLE_POST_WRITE_CODES,
                 )
                 for job in unknown:
-                    receipt = (
-                        await app.state.sdk_write_coordinator.inquire_write_outcome(
-                            job.request_id
-                        )
+                    receipt = await app.state.sdk_write_coordinator.inquire_write_outcome(
+                        job.request_id
                     )
                     if receipt is None:
                         continue
@@ -1434,6 +1428,14 @@ def create_app(
             app.state.memory_backfill = None
             app.state.memory_visibility_task = None
             app.state.deletion_sync = None
+            if app.state.memory_authority is MemoryAuthority.LEGACY:
+                _start_backfill(app)
+                _start_visibility_recompute(app)
+                app.state.deletion_sync = watched_task(
+                    asyncio.get_running_loop(),
+                    lambda: _keep_legacy_deletions_in_sync(app),
+                    name="one-memory-deletion-sync",
+                )
             # Both times the loop was blocked here it was found from outside,
             # by hand, days later. This says so at the time, and says what was
             # holding it.
@@ -1452,22 +1454,7 @@ def create_app(
             app.state.resource_diagnostics = watched_task(
                 asyncio.get_running_loop(), _emit_resource_snapshots
             )
-            app.state.retrieval_runtime = None
             try:
-                if runtime_settings.retrieval_index_enabled:
-                    from one_memory.retrieval.index_provisioning import provision_retrieval_index
-                    from one_memory.retrieval.runtime import ExistingRetrievalRuntime
-
-                    # The runtime only resumes an index someone initialized. Turning
-                    # the feature on is that someone; otherwise enabling it would
-                    # leave every search on the lexical lane with nothing to say why.
-                    await asyncio.to_thread(provision_retrieval_index, runtime_settings.data_dir)
-                    app.state.retrieval_runtime = ExistingRetrievalRuntime(
-                        runtime_settings.data_dir,
-                        minimum_free_bytes=runtime_settings.retrieval_minimum_free_bytes,
-                        service_levels=app.state.service_levels,
-                    )
-                    await app.state.retrieval_runtime.start()
                 # Startup writes are done. Only now let the derived-content
                 # rebuild take the single write checkout: it is a long
                 # background job, and on a large library it would otherwise
@@ -1494,15 +1481,6 @@ def create_app(
                     outcome="started",
                     instance_id=runtime_settings.instance_id,
                 )
-                retrieval_stop_error: BaseException | None = None
-                if app.state.retrieval_runtime is not None:
-                    try:
-                        await app.state.retrieval_runtime.stop()
-                    except (Exception, asyncio.CancelledError) as error:
-                        # Preserve failure while cleaning other owned services.
-                        # Experimental: an exceptional live-thread failure still
-                        # needs an outer ownership fence before production enablement.
-                        retrieval_stop_error = error
                 app.state.main_database_report.cancel()
                 app.state.loop_heartbeat.cancel()
                 with suppress(asyncio.CancelledError):
@@ -1560,8 +1538,6 @@ def create_app(
                 # Waited for off the loop: this is the task whose whole point
                 # is that the loop never waits on a disk.
                 await asyncio.to_thread(flush_diagnostics)
-                if retrieval_stop_error is not None:
-                    raise retrieval_stop_error
 
     app = FastAPI(
         title="MemHub Core",
@@ -1608,9 +1584,7 @@ def create_app(
         )
         if getattr(
             app.state, "memory_authority", MemoryAuthority.LEGACY
-        ) is MemoryAuthority.SDK and request.url.path.startswith(
-            legacy_paths
-        ):
+        ) is MemoryAuthority.SDK and request.url.path.startswith(legacy_paths):
             return JSONResponse(
                 status_code=409,
                 content={"detail": "sdk_memory_authority_required"},
@@ -1732,7 +1706,7 @@ def create_app(
                     trace_id=trace_id,
                     instance_id=runtime_settings.instance_id,
                     duration_ms=elapsed_ms(started),
-                    details={**diagnostic_details, "status": str(response.status_code)},
+                    details={**diagnostic_details, "status_code": response.status_code},
                 )
             return response
         except Exception as error:
@@ -3128,10 +3102,10 @@ def create_app(
         store = getattr(app.state, "sdk_data_plane_migrations", None)
         if store is None:
             store = SdkDataPlaneMigrationStore(_evidence_store(app).database_path)
-        marker = store.cutover()
+        authority, migration_id = store.authority_state()
         return MemoryAuthorityResponse(
-            authority="sdk" if marker is not None else "legacy",
-            migration_id=marker.migration_id if marker is not None else None,
+            authority="sdk" if authority is MemoryAuthority.SDK else "legacy",
+            migration_id=migration_id,
         )
 
     @app.get(
@@ -3157,6 +3131,130 @@ def create_app(
         except Exception as error:
             raise _sdk_http_error(error, "hy_memory_list_failed") from None
 
+    @app.get(
+        "/api/v2/memories/graph",
+        response_model=MemoryGraphResponse,
+        dependencies=protected,
+    )
+    async def sdk_memory_graph(
+        limit: int = Query(default=160, ge=1, le=300),
+    ) -> MemoryGraphResponse:
+        """The memory map on SDK authority: SDK memories plus the Kuzu overlay.
+
+        Memories come from the public client; only the overlay reads Kuzu
+        (`hy_graph_projection`, the visualization exception). Shape matches
+        the legacy map so the same view draws both.
+        """
+        try:
+            client = await _sdk_client_handle(app).client()
+            raw = await asyncio.to_thread(
+                client.list_memories, user_id="local", limit=limit, offset=0, order="desc"
+            )
+            page = HySdkMemoryPage.from_sdk(raw, limit=limit, offset=0)
+        except Exception as error:
+            raise _sdk_http_error(error, "hy_memory_list_failed") from None
+        vdb = raw.get("vdb") if isinstance(raw, dict) else None
+        total = vdb.get("total") if isinstance(vdb, dict) else None
+        memories = [memory for memory in page.memories if memory.layer != "l1_raw"]
+        catalog = await asyncio.to_thread(_source_catalog, app)
+        nodes: dict[str, MemoryGraphNode] = {}
+        edges: list[MemoryGraphEdge] = []
+        sessions: dict[str, list[str]] = {}
+        for memory in memories:
+            memory_node_id = f"memory:{memory.memory_id}"
+            nodes[memory_node_id] = MemoryGraphNode(
+                node_id=memory_node_id,
+                node_type="memory",
+                label=memory.content,
+                memory_id=memory.memory_id,
+                kind=memory.layer,
+                status=memory.status,
+            )
+            if memory.session_id:
+                sessions.setdefault(memory.session_id, []).append(memory_node_id)
+            if memory.source:
+                source_node_id = f"source:{memory.source}"
+                registration = catalog.get(memory.source)
+                nodes[source_node_id] = MemoryGraphNode(
+                    node_id=source_node_id,
+                    node_type="source",
+                    label=(
+                        registration.descriptor.display_name if registration else memory.source
+                    ),
+                    source_id=memory.source,
+                )
+                edges.append(
+                    MemoryGraphEdge(
+                        edge_id=f"{source_node_id}->{memory_node_id}",
+                        source_node_id=source_node_id,
+                        target_node_id=memory_node_id,
+                        relation="evidence_source",
+                        origin="provenance",
+                    )
+                )
+        for _session_id, node_ids in sorted(sessions.items()):
+            for left, right in zip(node_ids, node_ids[1:], strict=False):
+                edges.append(
+                    MemoryGraphEdge(
+                        edge_id=f"{left}->{right}:same_conversation",
+                        source_node_id=left,
+                        target_node_id=right,
+                        relation="same_conversation",
+                        origin="same_conversation",
+                    )
+                )
+        hy_graph = None
+        # Aggregate nodes carry no per-source provenance yet, so a revoked
+        # source withholds the overlay, as on the legacy map.
+        if not await asyncio.to_thread(_read_access(app).has_revoked_sources):
+            try:
+                hy_graph = await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
+            except HySdkError:
+                hy_graph = None
+        if hy_graph is not None:
+            for node in hy_graph.nodes:
+                node_id = f"memory:{node.node_id}"
+                if node_id in nodes:
+                    continue
+                node_id = f"hy:{node.node_id}"
+                nodes.setdefault(
+                    node_id,
+                    MemoryGraphNode(
+                        node_id=node_id,
+                        node_type="topic" if node.layer == "TOPIC" else "schema",
+                        label=node.content,
+                        confidence=node.confidence,
+                    ),
+                )
+            for edge in hy_graph.edges:
+                source_node_id, target_node_id = (
+                    f"memory:{edge.source_node_id}"
+                    if f"memory:{edge.source_node_id}" in nodes
+                    else f"hy:{edge.source_node_id}",
+                    f"memory:{edge.target_node_id}"
+                    if f"memory:{edge.target_node_id}" in nodes
+                    else f"hy:{edge.target_node_id}",
+                )
+                if source_node_id not in nodes or target_node_id not in nodes:
+                    continue
+                edges.append(
+                    MemoryGraphEdge(
+                        edge_id=f"{source_node_id}->{target_node_id}:{edge.relation}",
+                        source_node_id=source_node_id,
+                        target_node_id=target_node_id,
+                        relation=edge.relation,
+                        weight=edge.weight,
+                        origin="hy_relation",
+                    )
+                )
+        return MemoryGraphResponse(
+            total_memories=total if isinstance(total, int) else len(memories),
+            projected_memories=len(memories),
+            nodes=tuple(nodes.values()),
+            edges=tuple(edges),
+            hy_graph_available=hy_graph is not None and hy_graph.consulted,
+        )
+
     @app.get(
         "/api/v2/user-profile",
         response_model=SdkUserProfileResponse,
@@ -3781,9 +3879,7 @@ def create_app(
             all_memories, projected, nodes, edges = await asyncio.to_thread(read_governed_part)
             try:
                 hy_graph = (
-                    None
-                    if revoked
-                    else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
+                    None if revoked else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
                 )
             except HySdkError:
                 hy_graph = None
@@ -3892,9 +3988,7 @@ def create_app(
         active_queue, in_flight_count = supervisor.active_depths()
         planned, current = supervisor.manual_batch_progress()
         sdk_counts = (
-            supervisor.status_counts()
-            if app.state.memory_authority is MemoryAuthority.SDK
-            else {}
+            supervisor.status_counts() if app.state.memory_authority is MemoryAuthority.SDK else {}
         )
         # `running` counts rows the store still marks running, which includes
         # work a worker has already given up on. Reporting that as "what is
@@ -3915,9 +4009,7 @@ def create_app(
             ),
             recent_completions=depths.recent_completions,
             running=sdk_counts.get("running", running_rows),
-            stranded_running=(
-                0 if sdk_counts else max(0, running_rows - in_flight_count)
-            ),
+            stranded_running=(0 if sdk_counts else max(0, running_rows - in_flight_count)),
             unexecuted_policy_failures=store.unexecuted_policy_failures(),
             waiting_for_model=sdk_counts.get(
                 "waiting_for_model",
@@ -4238,7 +4330,7 @@ def create_app(
             if action == "delete":
                 deletion = _memory_store(app).deletion(memory_id, user_id="local")
                 if deletion is None or deletion.status is not DeletionStatus.COMPLETED:
-                    await _sdk_memory_runtime(app).finish_deletion(memory_id)
+                    await _finish_deletion(app, memory_id)
             return memory
         except KeyError:
             raise HTTPException(status_code=404, detail="memory_not_found") from None
@@ -4488,9 +4580,8 @@ class _LazyASGIApp:
     `create_app()` must not run at import time. The managed `serve()` entry
     imports `create_app` from this module before it can construct its own app
     with `manage_runtime=True`; an eager module-level app was therefore built
-    first with the default `manage_runtime=False`. With retrieval index enabled
-    that default app hit `retrieval_runtime_ownership_required` and killed the
-    process before the managed app could bind a port.
+    first with the default `manage_runtime=False`, doing startup work for an
+    app that never serves.
     """
 
     def __init__(self) -> None:
@@ -4936,6 +5027,54 @@ def _secret_store_detail(error: SecretStoreUnavailableError) -> str:
     return str(getattr(error, "code", None) or SecretStoreErrorCode.STORE_UNAVAILABLE)
 
 
+async def _finish_deletion(app: FastAPI, memory_id: str) -> None:
+    """Delete from the SDK and, on legacy authority, settle the deletion row.
+
+    Without the settle the legacy row stays `pending` for good, and the person
+    is shown a deletion that never finishes.
+    """
+    if app.state.memory_authority is MemoryAuthority.SDK:
+        await _sdk_memory_runtime(app).finish_deletion(memory_id)
+        return
+    await _work_legacy_deletions(app, memory_id=memory_id)
+
+
+async def _work_legacy_deletions(app: FastAPI, *, memory_id: str | None = None) -> None:
+    store = _memory_store(app)
+    runtime = _sdk_memory_runtime(app)
+    claims = await asyncio.to_thread(store.claim_deletions, user_id="local", memory_id=memory_id)
+    for claim in claims:
+        try:
+            await runtime.finish_deletion(claim.memory_id)
+        except HySdkError as error:
+            code: str | None = error.code
+        except Exception as error:  # noqa: BLE001 - recorded so it waits its turn
+            logger.warning("memory_deletion_unexpected_failure: %s", type(error).__name__)
+            code = "hy_memory_delete_unexpected"
+        else:
+            code = None
+        await asyncio.to_thread(store.settle_deletion, claim, user_id="local", error_code=code)
+
+
+_LEGACY_DELETION_SYNC_IDLE_SECONDS = 60.0
+
+
+async def _keep_legacy_deletions_in_sync(app: FastAPI) -> None:
+    """Retry legacy deletions the SDK did not take; nothing else asks again."""
+    store = _memory_store(app)
+    while True:
+        wait = _LEGACY_DELETION_SYNC_IDLE_SECONDS
+        try:
+            await _work_legacy_deletions(app)
+            due = await asyncio.to_thread(store.next_deletion_due, user_id="local")
+        except Exception as error:  # noqa: BLE001 - reported, then retried
+            report_exception("memory.deletion_sync", "memory_deletion_sync.failed", error)
+        else:
+            if due is not None:
+                wait = min(wait, max(0.0, (due - store.clock()).total_seconds()))
+        await asyncio.sleep(wait)
+
+
 def _memory_store(app: FastAPI) -> MemoryGovernanceStore:
     return cast(MemoryGovernanceStore, app.state.memory_governance_store)
 
@@ -5284,7 +5423,7 @@ def _report_visibility_failure(phase: str, error: BaseException) -> None:
         event="memory_visibility.background_failed",
         outcome="failed",
         error_type=type(error).__name__,
-        details={"phase": phase, "reason": str(error)[:200]},
+        details={"phase": phase},
     )
 
 

memhub/apps/core/src/one_memory/cli.py

@@ -1,5 +1,3 @@
-import asyncio
-import hashlib
 import os
 import signal
 import socket
@@ -31,13 +29,51 @@ def _end_by(seconds: str | None) -> None:
 if __name__ == "__main__" and sys.argv[1:2] == ["capture"]:
     _end_by(os.environ.get("ONE_MEMORY_CAPTURE_DEADLINE_S"))
 
+import errno  # noqa: E402
 import json  # noqa: E402
-from collections.abc import Sequence  # noqa: E402
+import re  # noqa: E402
+from collections.abc import Iterator, Sequence  # noqa: E402
 from typing import TextIO  # noqa: E402
 
 from one_memory.config import load_settings  # noqa: E402
 from one_memory.runtime_client import RuntimeStatus, inspect_runtime  # noqa: E402
 
+# The startup dialog can only act on the name it is given: running out of disk
+# and a damaged backup are different next steps. These are the names that reach
+# it.
+_SAFE_MIGRATION_CODE = re.compile(r"^[a-z][a-z0-9_]{0,95}$")
+_OUT_OF_SPACE = frozenset({errno.ENOSPC, getattr(errno, "EDQUOT", errno.ENOSPC)})
+
+
+def _exception_chain(error: BaseException) -> Iterator[BaseException]:
+    seen: set[int] = set()
+    current: BaseException | None = error
+    while current is not None and id(current) not in seen:
+        seen.add(id(current))
+        yield current
+        current = current.__cause__ or current.__context__
+
+
+def migration_failure_code(
+    error: BaseException, *, fallback: str = "sdk_migration_preflight_failed"
+) -> str:
+    """Name a migration failure; never guess a code from free text.
+
+    Errs toward the caller's generic code: an exception this does not recognize
+    keeps its message out of the document Desktop reads.
+    """
+    chain = list(_exception_chain(error))
+    for item in chain:
+        if isinstance(item, OSError) and item.errno in _OUT_OF_SPACE:
+            return "sdk_migration_disk_space_insufficient"
+    for item in chain:
+        code = getattr(item, "code", "")
+        if not isinstance(code, str) or not _SAFE_MIGRATION_CODE.match(code):
+            continue
+        if code.startswith(("sdk_data_plane_", "sdk_migration_")):
+            return code
+    return fallback
+
 
 def main() -> None:
     if sys.argv[1:] == ["--keychain-read"]:
@@ -77,16 +113,10 @@ def run(arguments: Sequence[str], *, output: TextIO | None = None) -> int:
     command, *options = arguments
     if command == "restore":
         return restore(options, stream)
-    if command == "migration-bundle":
-        return migration_bundle(options, stream)
     if command == "sdk-migrate":
         return sdk_migrate(options, stream)
-    if command == "sdk-cutover":
-        return sdk_cutover(options, stream)
     if command == "sdk-migration-status":
         return sdk_migration_status(options, stream)
-    if command == "sdk-cleanup":
-        return sdk_cleanup(options, stream)
     if command == "boundary" and not options:
         from one_memory.workbuddy_boundary import write_hook
 
@@ -107,11 +137,8 @@ def run(arguments: Sequence[str], *, output: TextIO | None = None) -> int:
         stream.write(
             "Usage: one-memory [serve [--maintenance|--reader]|status [--json]|doctor [--json]"
             "|restore --source <managed-backup> --yes [--json]"
-            "|migration-bundle --bundle-id <id> --runtime-build <build> --yes [--json]"
-            "|sdk-migrate --migration-id <id> --bundle-id <id> --yes [--json]"
-            "|sdk-cutover --migration-id <id> --bundle-id <id> --yes [--json]"
+            "|sdk-migrate --runtime-build <build> --yes [--json]"
             "|sdk-migration-status [--json]"
-            "|sdk-cleanup --yes [--json]"
             "|boundary [--peek|--ack TOKEN]|capture SOURCE]\n"
         )
         return 2
@@ -253,19 +280,20 @@ def _write_restore(
         stream.write(f"{text}\n")
 
 
-def migration_bundle(options: Sequence[str], stream: TextIO) -> int:
-    """Create a complete management + SDK bundle while Runtime is stopped."""
-    bundle_id: str | None = None
+def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
+    """Bring a stopped home to SDK authority in one resumable command.
+
+    Backs up the home, makes queue work independent of the legacy tables,
+    commits SDK authority and drops those tables. The SDK already holds every
+    memory and raw turn, so nothing is copied. With `--json`, each step is one
+    progress line and the last line is the result.
+    """
     runtime_build: str | None = None
     confirmed = False
     as_json = False
     index = 0
     while index < len(options):
         option = options[index]
-        if option == "--bundle-id" and index + 1 < len(options):
-            bundle_id = options[index + 1]
-            index += 2
-            continue
         if option == "--runtime-build" and index + 1 < len(options):
             runtime_build = options[index + 1]
             index += 2
@@ -280,116 +308,8 @@ def migration_bundle(options: Sequence[str], stream: TextIO) -> int:
             continue
         stream.write(f"one-memory: 参数无效：{option}\n")
         return 2
-    if not bundle_id or not runtime_build:
-        stream.write("one-memory: migration-bundle 需要 --bundle-id <id> --runtime-build <build>\n")
-        return 2
-
-    settings = load_settings()
-    status = inspect_runtime(settings.data_dir)
-    if status.state != "stopped":
-        document = {
-            "schema_version": 1,
-            "state": "blocked",
-            "code": "sdk_migration_bundle_runtime_running",
-            "runtime_state": status.state,
-        }
-        _write_restore(stream, as_json, document, "Runtime 仍在运行；不能创建迁移备份。")
-        return 4
-    if not confirmed:
-        document = {
-            "schema_version": 1,
-            "state": "preview",
-            "bundle_id": bundle_id,
-            "requires_confirmation": True,
-        }
-        _write_restore(stream, as_json, document, "预览通过；加 --yes 后创建完整迁移备份。")
-        return 4
-
-    from one_memory.sdk_migration_bundle import (
-        MigrationBundleRejected,
-        create_sdk_migration_bundle,
-        inspect_sdk_migration_bundle,
-    )
-
-    try:
-        result = create_sdk_migration_bundle(
-            data_dir=settings.data_dir,
-            bundle_id=bundle_id,
-            runtime_build=runtime_build,
-            runtime_stopped=True,
-        )
-    except MigrationBundleRejected as error:
-        if error.code == "sdk_migration_bundle_exists":
-            try:
-                manifest = inspect_sdk_migration_bundle(
-                    data_dir=settings.data_dir,
-                    bundle_id=bundle_id,
-                )
-            except MigrationBundleRejected:
-                pass
-            else:
-                document = {
-                    "schema_version": 1,
-                    "state": "reused",
-                    "bundle_id": bundle_id,
-                    "file_count": manifest.get("file_count"),
-                    "total_bytes": manifest.get("total_bytes"),
-                }
-                _write_restore(
-                    stream,
-                    as_json,
-                    document,
-                    f"已验证并复用迁移备份：{bundle_id}",
-                )
-                return 0
-        document = {
-            "schema_version": 1,
-            "state": "failed",
-            "code": error.code,
-        }
-        _write_restore(stream, as_json, document, f"迁移备份失败：{error.code}")
-        return 1
-    document = {
-        "schema_version": 1,
-        "state": "created",
-        "bundle_id": result.bundle_id,
-        "bundle_path": str(result.bundle_path),
-        "file_count": result.file_count,
-        "total_bytes": result.total_bytes,
-    }
-    _write_restore(stream, as_json, document, f"完整迁移备份已创建：{result.bundle_path}")
-    return 0
-
-
-def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
-    """Backfill legacy memory/raw into SDK while keeping legacy authority."""
-    migration_id: str | None = None
-    bundle_id: str | None = None
-    confirmed = False
-    as_json = False
-    index = 0
-    while index < len(options):
-        option = options[index]
-        if option == "--migration-id" and index + 1 < len(options):
-            migration_id = options[index + 1]
-            index += 2
-            continue
-        if option == "--bundle-id" and index + 1 < len(options):
-            bundle_id = options[index + 1]
-            index += 2
-            continue
-        if option == "--yes":
-            confirmed = True
-            index += 1
-            continue
-        if option == "--json":
-            as_json = True
-            index += 1
-            continue
-        stream.write(f"one-memory: 参数无效：{option}\n")
-        return 2
-    if not migration_id or not bundle_id:
-        stream.write("one-memory: sdk-migrate 需要 --migration-id <id> --bundle-id <id>\n")
+    if not runtime_build:
+        stream.write("one-memory: sdk-migrate 需要 --runtime-build <build>\n")
         return 2
 
     settings = load_settings()
@@ -404,271 +324,59 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
         _write_restore(stream, as_json, document, "Runtime 仍在运行；不能迁移。")
         return 4
     if not confirmed:
-        document = {
-            "schema_version": 1,
-            "state": "preview",
-            "migration_id": migration_id,
-            "bundle_id": bundle_id,
-            "requires_confirmation": True,
-            "commits_cutover": False,
-        }
-        _write_restore(
-            stream,
-            as_json,
-            document,
-            "预览通过；加 --yes 后执行可续跑 backfill，但不会切换 authority。",
-        )
-        return 4
-
-    from one_memory.database_access import close_main_databases
-    from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE, HY_MEMORY_VERSION
-    from one_memory.hy_sdk_migration import HySdkMigration
-    from one_memory.hy_sdk_runtime import acquire_client
-    from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
-    from one_memory.memory_server.migrations import MIGRATIONS
-    from one_memory.model_profiles import ModelProfileStore
-    from one_memory.sdk_migration_bundle import (
-        MigrationBundleRejected,
-        inspect_sdk_migration_bundle,
-    )
-    from one_memory.sdk_migration_runner import (
-        BackfillResult,
-        prepare_sdk_migration_staging,
-        run_sdk_data_plane_backfill,
-        sdk_migration_staging,
-    )
-    from one_memory.secrets import KeyringSecretStore, SecretStoreUnavailableError
-
-    database = settings.data_dir / "one-memory.sqlite3"
-    try:
-        manifest = inspect_sdk_migration_bundle(
-            data_dir=settings.data_dir,
-            bundle_id=bundle_id,
-        )
-        asyncio.run(migrate(SqliteMigrationStore(database), MIGRATIONS))
-        close_main_databases()
-        profiles = ModelProfileStore(database)
-        identity = profiles.active_identity()
-        if identity is None:
-            raise RuntimeError("model_profile_not_active")
-        profile = profiles.snapshot(*identity)
-        secrets = KeyringSecretStore()
-        secret = profiles.resolve_snapshot_secret(*identity, secrets)
-        if profile.has_api_key and secret is None:
-            raise RuntimeError("model_secret_unavailable")
-    except MigrationBundleRejected as error:
-        code = error.code
-    except SecretStoreUnavailableError:
-        code = "secure_storage_unavailable"
-    except Exception as error:
-        code = (
-            str(error)
-            if str(error)
-            in {
-                "model_profile_not_active",
-                "model_secret_unavailable",
-            }
-            else "sdk_migration_preflight_failed"
-        )
-    else:
-        namespace_fingerprint = hashlib.sha256(HY_MEMORY_NAMESPACE.encode()).hexdigest()
-        backend_material = json.dumps(
-            {
-                "namespace": HY_MEMORY_NAMESPACE,
-                "embedding_provider": str(profile.embedding_provider_kind or ""),
-                "embedding_model": profile.embedding_model,
-                "embedding_dimensions": profile.embedding_dimensions,
-                "vector": "chroma",
-                "cache": "sqlite",
-                "history": "sqlite",
-                "coding": "sqlite",
-            },
-            sort_keys=True,
-            separators=(",", ":"),
-        )
-        backend_fingerprint = hashlib.sha256(backend_material.encode()).hexdigest()
-        staging = sdk_migration_staging(
-            data_dir=settings.data_dir,
-            migration_id=migration_id,
-            namespace=HY_MEMORY_NAMESPACE,
-        )
-        try:
-            staging_data_dir = prepare_sdk_migration_staging(
-                paths=staging,
-                database_path=database,
-            )
-        except Exception:
-            code = "sdk_migration_staging_failed"
-            staging_data_dir = None
-
-        async def run_backfill() -> BackfillResult:
-            assert staging_data_dir is not None
-            lease = await asyncio.to_thread(
-                acquire_client,
-                profile,
-                api_key=secret,
-                data_dir=staging_data_dir,
-            )
-            try:
-                return await run_sdk_data_plane_backfill(
-                    database_path=database,
-                    client=HySdkMigration(lease.client),
-                    migration_id=migration_id,
-                    bundle_id=bundle_id,
-                    source_runtime_build=str(manifest.get("runtime_build") or ""),
-                    target_sdk_version=HY_MEMORY_VERSION,
-                    source_schema_version=MIGRATIONS[-1].version,
-                    namespace_fingerprint=namespace_fingerprint,
-                    backend_fingerprint=backend_fingerprint,
-                )
-            finally:
-                await asyncio.to_thread(lease.close)
-
-        try:
-            result = (
-                asyncio.run(run_backfill())
-                if staging_data_dir is not None
-                else None
-            )
-        except Exception:
-            code = "sdk_migration_backfill_failed"
-        else:
-            if result is None:
-                document = {"schema_version": 1, "state": "failed", "code": code}
-                _write_restore(stream, as_json, document, f"SDK backfill 失败：{code}")
-                return 1
-            document = {
-                "schema_version": 1,
-                "state": result.state,
-                "phase": result.phase,
-                "migration_id": result.migration_id,
-                "counts": result.counts,
-                "memory_items": result.memory_items,
-                "raw_items": result.raw_items,
-                "commits_cutover": False,
-            }
-            _write_restore(
-                stream,
-                as_json,
-                document,
-                "SDK backfill 已完成；authority 尚未切换。",
-            )
-            return 0 if result.state == "ready" else 4
-
-    document = {"schema_version": 1, "state": "failed", "code": code}
-    _write_restore(stream, as_json, document, f"SDK backfill 失败：{code}")
-    return 1
-
-
-def sdk_cutover(options: Sequence[str], stream: TextIO) -> int:
-    """Commit SDK authority only after offline reconciliation and work drain."""
-    migration_id: str | None = None
-    bundle_id: str | None = None
-    confirmed = False
-    as_json = False
-    index = 0
-    while index < len(options):
-        option = options[index]
-        if option == "--migration-id" and index + 1 < len(options):
-            migration_id = options[index + 1]
-            index += 2
-            continue
-        if option == "--bundle-id" and index + 1 < len(options):
-            bundle_id = options[index + 1]
-            index += 2
-            continue
-        if option == "--yes":
-            confirmed = True
-            index += 1
-            continue
-        if option == "--json":
-            as_json = True
-            index += 1
-            continue
-        stream.write(f"one-memory: 参数无效：{option}\n")
-        return 2
-    if not migration_id or not bundle_id:
-        stream.write("one-memory: sdk-cutover 需要 --migration-id <id> --bundle-id <id>\n")
-        return 2
-    settings = load_settings()
-    status = inspect_runtime(settings.data_dir)
-    if status.state != "stopped":
-        document = {
-            "schema_version": 1,
-            "state": "blocked",
-            "code": "sdk_cutover_runtime_running",
-            "runtime_state": status.state,
-        }
-        _write_restore(stream, as_json, document, "Runtime 仍在运行；不能切换 authority。")
-        return 4
-    if not confirmed:
-        document = {
-            "schema_version": 1,
-            "state": "preview",
-            "migration_id": migration_id,
-            "bundle_id": bundle_id,
-            "requires_confirmation": True,
-        }
-        _write_restore(stream, as_json, document, "预览通过；加 --yes 后提交 SDK authority。")
+        document = {"schema_version": 1, "state": "preview", "requires_confirmation": True}
+        _write_restore(stream, as_json, document, "预览通过；加 --yes 后执行迁移。")
         return 4
 
     from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE, HY_MEMORY_VERSION
-    from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
-    from one_memory.sdk_migration_bundle import MigrationBundleRejected
-    from one_memory.sdk_migration_runner import (
-        activate_sdk_migration_staging,
-        commit_reconciled_cutover,
-        resume_legacy_cleanup,
-        sdk_migration_staging,
-    )
+    from one_memory.sdk_migration_runner import run_startup_migration
 
-    try:
-        database_path = settings.data_dir / "one-memory.sqlite3"
-        staging = sdk_migration_staging(
-            data_dir=settings.data_dir,
-            migration_id=migration_id,
-            namespace=HY_MEMORY_NAMESPACE,
-        )
-        marker = activate_sdk_migration_staging(
-            paths=staging,
-            database_path=database_path,
-            commit=lambda: commit_reconciled_cutover(
-                data_dir=settings.data_dir,
-                database_path=database_path,
-                migration_id=migration_id,
-                bundle_id=bundle_id,
-                sdk_version=HY_MEMORY_VERSION,
-            ),
-        )
-        resume_legacy_cleanup(
-            data_dir=settings.data_dir,
-            database_path=database_path,
-        )
-    except (DataPlaneMigrationError, MigrationBundleRejected, OSError) as error:
-        code = getattr(error, "code", str(error))
-        document = {
-            "schema_version": 1,
-            "state": "blocked",
-            "code": code,
-            "migration_id": migration_id,
-            "bundle_id": bundle_id,
-        }
-        _write_restore(stream, as_json, document, f"SDK authority 未切换：{code}")
-        return 4
-    document = {
-        "schema_version": 1,
-        "state": "cutover",
-        "migration_id": marker.migration_id,
-        "bundle_id": marker.backup_bundle_id,
-        "sdk_version": marker.sdk_version,
-        "completed_at": marker.completed_at,
+    step_names = {
+        "bundle": "正在备份记忆数据……",
+        "prepare": "正在整理待处理的写入任务……",
+        "cutover": "正在切换数据源……",
+        "vectors": "正在接回旧版本的向量数据……",
+        "verify": "正在用新版本核对记忆能否读回……",
+        "cleanup": "正在删除旧表……",
     }
-    _write_restore(stream, as_json, document, "SDK authority 已切换。")
+
+    def progress(step: str) -> None:
+        if as_json:
+            stream.write(
+                json.dumps({"schema_version": 1, "state": "running", "step": step}) + "\n"
+            )
+        else:
+            stream.write(f"{step_names.get(step, step)}\n")
+        stream.flush()
+
+    try:
+        result = run_startup_migration(
+            data_dir=settings.data_dir,
+            runtime_build=runtime_build,
+            sdk_version=HY_MEMORY_VERSION,
+            namespace=HY_MEMORY_NAMESPACE,
+            progress=progress,
+        )
+    except Exception as error:
+        from one_memory.sdk_migration_verify import MemoryReadbackFailed
+
+        code = migration_failure_code(error)
+        failed: dict[str, object] = {"schema_version": 1, "state": "failed", "code": code}
+        if isinstance(error, MemoryReadbackFailed):
+            failed["readback"] = error.report.as_document()
+        _write_restore(stream, as_json, failed, f"迁移未完成：{code}")
+        return 1
+    messages = {
+        "not_required": "不需要迁移。",
+        "complete": "迁移已完成；遗留的旧表或向量已处理。",
+        "cutover": "迁移完成：记忆数据已以 SDK 为准。",
+    }
+    _write_restore(stream, as_json, result.as_document(), messages[result.state])
     return 0
 
 
 def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
-    """Read-only startup probe used before taking the Runtime maintenance fence."""
+    """Read-only probe: does this home need `sdk-migrate`?"""
     if list(options) not in ([], ["--json"]):
         stream.write("one-memory: sdk-migration-status 只接受 --json\n")
         return 2
@@ -676,82 +384,40 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
     settings = load_settings()
     database_path = settings.data_dir / "one-memory.sqlite3"
     if not database_path.is_file():
-        document = {
+        document: dict[str, object] = {
             "schema_version": 1,
-            "authority": "legacy",
-            "state": "empty",
-            "migration_id": None,
+            "authority": "sdk",
+            "state": "not_required",
         }
     else:
-        import sqlite3
-
-        from one_memory.database_access import main_database
         from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
         from one_memory.sdk_data_plane_migration import (
             MemoryAuthority,
             SdkDataPlaneMigrationStore,
         )
-        from one_memory.sdk_migration_runner import sdk_migration_staging
+        from one_memory.sdk_migration_runner import abandoned_staging_present
+        from one_memory.sdk_migration_verify import read_active_profile
+        from one_memory.sdk_vector_reattach import legacy_vector_collections
         from one_memory.target_schema import legacy_tables_present
 
-        migration_store = SdkDataPlaneMigrationStore(database_path)
-        marker = migration_store.cutover()
-        authority = migration_store.authority()
-        stat = database_path.stat()
-        generation = hashlib.sha256(f"{stat.st_size}:{stat.st_mtime_ns}".encode()).hexdigest()[:16]
+        authority = SdkDataPlaneMigrationStore(database_path).authority()
         if authority is MemoryAuthority.SDK:
-            legacy_tables = legacy_tables_present(database_path)
-            staging_cleanup = False
-            if marker is not None:
-                staging = sdk_migration_staging(
+            profile = read_active_profile(database_path)
+            leftovers = bool(
+                legacy_tables_present(database_path)
+                or abandoned_staging_present(
                     data_dir=settings.data_dir,
-                    migration_id=marker.migration_id,
                     namespace=HY_MEMORY_NAMESPACE,
                 )
-                staging_cleanup = (
-                    staging.protective_root.exists()
-                    or staging.staging_home.exists()
+                or (
+                    profile is not None
+                    and legacy_vector_collections(settings.data_dir, profile)
                 )
-            document = {
-                "schema_version": 1,
-                "authority": "sdk",
-                "state": (
-                    "cleanup_required"
-                    if legacy_tables or staging_cleanup
-                    else "complete"
-                ),
-                "migration_id": marker.migration_id if marker else None,
-                "generation": generation,
-                "legacy_tables": list(legacy_tables),
-            }
+            )
+            state = "required" if leftovers else "not_required"
         else:
-            with main_database(database_path).read(row_factory=sqlite3.Row) as database:
-                tables = {
-                    str(row["name"])
-                    for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
-                }
-                counts = {
-                    "memory": (
-                        int(
-                            database.execute("SELECT COUNT(*) FROM governed_memories").fetchone()[0]
-                        )
-                        if "governed_memories" in tables
-                        else 0
-                    ),
-                    "raw": (
-                        int(database.execute("SELECT COUNT(*) FROM evidence").fetchone()[0])
-                        if "evidence" in tables
-                        else 0
-                    ),
-                }
-            document = {
-                "schema_version": 1,
-                "authority": "legacy",
-                "state": "required" if any(counts.values()) else "empty",
-                "migration_id": None,
-                "counts": counts,
-                "generation": generation,
-            }
+            state = "required"
+        document = {"schema_version": 1, "authority": authority.value, "state": state}
     if as_json:
         stream.write(f"{json.dumps(document, ensure_ascii=False)}\n")
     else:
@@ -759,65 +425,6 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
     return 0
 
 
-def sdk_cleanup(options: Sequence[str], stream: TextIO) -> int:
-    if set(options) - {"--yes", "--json"} or options.count("--yes") > 1:
-        stream.write("one-memory: sdk-cleanup 只接受 --yes [--json]\n")
-        return 2
-    confirmed = "--yes" in options
-    as_json = "--json" in options
-    settings = load_settings()
-    status = inspect_runtime(settings.data_dir)
-    if status.state != "stopped":
-        document = {
-            "schema_version": 1,
-            "state": "blocked",
-            "code": "sdk_cleanup_runtime_running",
-        }
-        _write_restore(stream, as_json, document, "Runtime 仍在运行；不能清理旧表。")
-        return 4
-    if not confirmed:
-        _write_restore(
-            stream,
-            as_json,
-            {
-                "schema_version": 1,
-                "state": "preview",
-                "requires_confirmation": True,
-            },
-            "加 --yes 后删除已迁移旧表。",
-        )
-        return 4
-    from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
-    from one_memory.sdk_migration_bundle import MigrationBundleRejected
-    from one_memory.sdk_migration_runner import resume_legacy_cleanup
-
-    try:
-        dropped = resume_legacy_cleanup(
-            data_dir=settings.data_dir,
-            database_path=settings.data_dir / "one-memory.sqlite3",
-        )
-    except (DataPlaneMigrationError, MigrationBundleRejected) as error:
-        code = getattr(error, "code", str(error))
-        _write_restore(
-            stream,
-            as_json,
-            {"schema_version": 1, "state": "blocked", "code": code},
-            f"旧表清理未完成：{code}",
-        )
-        return 4
-    _write_restore(
-        stream,
-        as_json,
-        {
-            "schema_version": 1,
-            "state": "clean",
-            "dropped_tables": list(dropped),
-        },
-        "旧表已删除。",
-    )
-    return 0
-
-
 def _status_text(command: str, status: RuntimeStatus) -> str:
     if status.state == "running":
         location = f"{status.host}:{status.port}"

memhub/apps/core/src/one_memory/config.py

@@ -23,10 +23,6 @@ class Settings(BaseSettings):
     live_sync_enabled: bool = True
     automatic_memory_processing: bool = True
     imports_enabled: bool = True
-    # Experimental and unreleased: ownership/error shutdown still requires acceptance.
-    # Resume only explicitly initialized generations; never migrate at startup.
-    retrieval_index_enabled: bool = False
-    retrieval_minimum_free_bytes: int = Field(default=256 * 1024 * 1024, ge=0)
     # Bounded metadata only: query, original text and answers are never stored.
     call_event_retention_days: int | None = Field(default=30, ge=1, le=365)
     call_event_max_events: int | None = Field(default=10_000, ge=1, le=100_000)

memhub/apps/core/src/one_memory/context_routes.py

@@ -38,7 +38,7 @@ from one_memory.context_models import (
     SdkMemoryContextItem,
     SdkRawContextItem,
 )
-from one_memory.context_raw import revalidate_raw_context
+from one_memory.context_raw import read_raw_context, revalidate_raw_context
 from one_memory.context_search import search_context
 from one_memory.diagnostic_log import diagnostic_event, diagnostic_exception
 from one_memory.external_memory_access import external_access
@@ -50,7 +50,6 @@ from one_memory.hy_sdk_types import (
     parse_sdk_search,
 )
 from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval.raw_lane import read_raw_indexed_context
 from one_memory.semantic_deadline import hy_wait_budget, search_within
 
 CALL_EVENT_RESPONSE_BUDGET_MS = 250
@@ -99,12 +98,11 @@ def context_router(dependencies: Sequence[Depends]) -> APIRouter:
             read = asyncio.create_task(
                 in_read_pool(
                     raw_read_pool(),
-                    read_raw_indexed_context,
+                    read_raw_context,
                     database,
                     value,
                     source_ids=authority.raw_sources,
                     deadline=lane_end,
-                    index=getattr(request.app.state, "retrieval_runtime", None),
                 )
             )
             done, _ = await asyncio.wait({read}, timeout=max(0, lane_end - time.monotonic()))

memhub/apps/core/src/one_memory/context_v2_coordinator.py

@@ -22,7 +22,7 @@ from dataclasses import dataclass
 from datetime import UTC, datetime
 from functools import partial
 from pathlib import Path
-from typing import Any, Literal, Protocol, cast
+from typing import Any, Literal, cast
 
 from one_memory.background_work import interactive_read_pool
 from one_memory.context_expand import ExpansionAuthority, ExpansionRefSigner
@@ -36,23 +36,8 @@ from one_memory.retrieval.models import (
     RawEvidence,
     RetrievalPath,
     RetrievalResponse,
-    TimeRange,
 )
 from one_memory.retrieval.query_plan import plan_local_query
-from one_memory.retrieval.raw_candidates import RawCandidateBatch
-from one_memory.retrieval.raw_evidence import materialize_raw_candidates
-
-
-class RawIndexLookup(Protocol):
-    def lookup_raw(
-        self,
-        query: str,
-        *,
-        source_ids: frozenset[str],
-        deadline: float,
-        time_range: TimeRange | None = None,
-    ) -> tuple[RawCandidateBatch, bool]: ...
-
 
 ExpandRefs = tuple[ExpansionRefSigner, ExpansionAuthority]
 
@@ -106,11 +91,6 @@ class LaneResult:
     coverage_limited: bool = False
 
 
-def _semantic_only_raw(item: RawEvidence) -> bool:
-    """Semantic overlap alone is a candidate, not a confirmed recall item."""
-    return set(item.matched_by) == {"semantic"}
-
-
 @dataclass(frozen=True)
 class StageTimings:
     raw_ms: float = 0.0
@@ -191,12 +171,6 @@ def _managed(
     return selected, truncated
 
 
-def _group_missing_requirements(item: RawEvidence | GovernedEvidence) -> list[str]:
-    if isinstance(item, RawEvidence) and _semantic_only_raw(item):
-        return ["relevance_confirmation"]
-    return []
-
-
 def _path(
     status: str,
     reason: str | None,
@@ -296,8 +270,7 @@ def read_lexical_raw_lane(
         ("lexical", "object") if plan.object_keys else ("lexical",),
         truncated,
         # Starvation means: this lane matched, then dropped the item whole. An
-        # ordinary no-match must stay empty, and an unconfigured semantic lane
-        # is not starvation.
+        # ordinary no-match must stay empty.
         starved and not items,
         probe_saturated,
     )
@@ -564,97 +537,6 @@ def warm_raw_read_path(
         return False
 
 
-def read_semantic_raw_lane(
-    index: RawIndexLookup | None,
-    database_path: Path,
-    query: CoordinatorQuery,
-    *,
-    source_ids: frozenset[str],
-    deadline: float,
-) -> LaneResult:
-    if not source_ids:
-        return LaneResult((), "unavailable", "permission_required", (), False)
-    # No derived index is the no-model baseline, not a failure: the lexical lane
-    # must still answer. Semantic is genuinely unsearched in that state, so it
-    # must not claim a channel or a coverage result it never produced.
-    if index is None:
-        return LaneResult((), "not_searched", None, (), False)
-    try:
-        batch, unpublished = index.lookup_raw(
-            query.query,
-            source_ids=source_ids,
-            deadline=deadline,
-            time_range=plan_local_query(query.query).time_range,
-        )
-    except Exception:
-        return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
-    if not isinstance(batch, RawCandidateBatch):
-        return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
-    if batch.reason == "not_authorized":
-        return LaneResult((), "unavailable", "permission_required", ("semantic",), False)
-    if batch.reason == "deadline":
-        return LaneResult((), "unavailable", "timeout_budget", ("semantic",), True)
-    if batch.reason in {"vector_unavailable", "index_unavailable"}:
-        return LaneResult((), "unavailable", batch.reason, ("semantic",), False)
-    store, generation = _index_state(index, database_path)
-    from one_memory.retrieval.index_store import RetrievalIndexStore
-
-    store = cast("RetrievalIndexStore | None", store)
-    if store is None or generation is None:
-        return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
-    try:
-        materialized = materialize_raw_candidates(
-            store,
-            batch,
-            generation_id=generation,
-            source_ids=source_ids,
-            deadline=deadline,
-            character_budget=query.character_budget,
-        )
-    except Exception:
-        return LaneResult((), "unavailable", "index_unavailable", ("semantic",), False)
-    if materialized.reason == "deadline":
-        return LaneResult((), "unavailable", "timeout_budget", ("semantic",), True)
-    if materialized.reason in {"index_unavailable", "not_authorized"}:
-        return LaneResult((), "unavailable", materialized.reason, ("semantic",), False)
-    complete = materialized.complete and not unpublished
-    if materialized.reason == "evidence_changed":
-        status: Literal["ok", "empty", "unavailable", "not_searched"] = "unavailable"
-        reason = "evidence_changed"
-    elif not materialized.items and not complete:
-        status, reason = "unavailable", "index_incomplete"
-    elif materialized.reason == "character_budget":
-        status, reason = "ok", None
-    else:
-        status, reason = ("ok" if materialized.items else "empty"), None
-    return LaneResult(
-        tuple(materialized.items),
-        status,
-        reason,
-        ("semantic",),
-        materialized.reason == "character_budget",
-    )
-
-
-def _index_state(index: RawIndexLookup, database_path: Path) -> tuple[object | None, str | None]:
-    store = getattr(index, "_store", None)
-    generation = None
-    snapshot = getattr(index, "snapshot", None)
-    if snapshot is not None:
-        try:
-            generation = snapshot().generation_id
-        except Exception:
-            generation = None
-    if store is None:
-        try:
-            from one_memory.retrieval.index_store import RetrievalIndexStore
-
-            store = RetrievalIndexStore(database_path)
-        except Exception:
-            store = None
-    return store, generation
-
-
 def read_memory_lane(
     database_path: Path,
     query: CoordinatorQuery,
@@ -857,7 +739,6 @@ def coordinate_context_v2(
     query: CoordinatorQuery,
     *,
     database_path: Path,
-    index: RawIndexLookup | None = None,
     deadline: float,
     expand_refs: ExpandRefs | None = None,
 ) -> RetrievalResponse:
@@ -871,62 +752,29 @@ def coordinate_context_v2(
         lexical = read_lexical_raw_lane(
             database_path, query, source_ids=authority.raw_sources, deadline=deadline
         )
-        semantic = read_semantic_raw_lane(
-            index, database_path, query, source_ids=authority.raw_sources, deadline=deadline
-        )
         timings = StageTimings(raw_ms=(time.monotonic() - raw_started) * 1000)
         combined: dict[tuple[str, str], RawEvidence] = {}
-        for item in (*lexical.items, *semantic.items):
-            if not isinstance(item, RawEvidence):
-                continue
-            identity = _identity(item)
-            previous = combined.get(identity)
-            if previous is None or _semantic_only_raw(previous):
-                combined[identity] = item
-        # Semantic similarity is a candidate signal, not a verified recall
-        # result. Keep semantic-only candidates visible in coverage, but do not
-        # release their bodies for host injection unless the item is
-        # independently confirmed by the lexical/object lane. A global distance
-        # threshold is deliberately not used: the measured positive and
-        # negative distributions overlap.
-        semantic_only = {_identity(item) for item in combined.values() if _semantic_only_raw(item)}
-        confirmed = {_identity(item) for item in lexical.items if isinstance(item, RawEvidence)}
-        deferred_raw = semantic_only - confirmed
-        if deferred_raw:
-            combined = {
-                identity: item
-                for identity, item in combined.items()
-                if identity not in deferred_raw
-            }
-        channels = tuple(dict.fromkeys((*lexical.channels, *semantic.channels)))
+        for item in lexical.items:
+            if isinstance(item, RawEvidence):
+                combined.setdefault(_identity(item), item)
         if lexical.status == "unavailable":
             status: Literal["ok", "empty", "unavailable", "not_searched", "degraded"] = (
                 "unavailable"
             )
             reason = lexical.reason
-        elif semantic.status == "unavailable":
-            status = "unavailable"
-            reason = semantic.reason
-        elif combined:
-            status, reason = "ok", semantic.reason if semantic.status != "ok" else None
-        elif deferred_raw:
-            status, reason = "degraded", "relevance_confirmation"
-        elif lexical.status == "empty":
-            status, reason = ("empty" if semantic.status != "not_searched" else "ok"), None
+        elif combined or lexical.status == "empty":
+            status, reason = "ok", None
         else:
             status, reason = "empty", None
-        truncated = lexical.truncated or semantic.truncated
         raw = LaneResult(
             tuple(combined.values()),
             status,
             reason,
-            channels,
-            truncated,
+            lexical.channels,
+            lexical.truncated,
             lexical.starved,
             lexical.coverage_limited,
         )
-        if deferred_raw:
-            object.__setattr__(raw, "coverage_limited", True)
     elif "raw" in query.kinds and not authority.raw_allowed:
         raw = LaneResult((), "unavailable", "permission_required", (), False)
     if "memory" in query.kinds:
@@ -995,9 +843,7 @@ def coordinate_context_v2(
             return _path("unavailable", raw.reason, raw.channels, "not_searched", False, 0)
         if dropped and not final_raw:
             return _path("unavailable", "permission_required", raw.channels, "unavailable", True, 0)
-        # Lexical-only is complete when the lexical lane actually answered. An
-        # unconfigured semantic lane is a capability absence, not proof that
-        # less than the authorized range was searched.
+        # The raw lane is complete when the lexical lane actually answered.
         complete = (
             raw.status in {"ok", "empty"}
             and not dropped
@@ -1022,9 +868,7 @@ def coordinate_context_v2(
                 "count_budget"
                 if truncated
                 else "coverage_budget"
-                if raw.coverage_limited and not deferred_raw
-                else "relevance_confirmation"
-                if deferred_raw
+                if raw.coverage_limited
                 else "character_budget"
             )
         return _path(
@@ -1112,7 +956,7 @@ def coordinate_context_v2(
                 "group_id": f"group-{index}",
                 "answer_target": query.intent,
                 "decision_status": "missing_confirmation",
-                "missing_requirements": _group_missing_requirements(item),
+                "missing_requirements": [],
                 "object_refs": _object_refs(item, plan.object_keys),
                 "items": [group_item.model_dump(mode="json") for group_item in (item,)],
             }
@@ -1185,7 +1029,6 @@ async def search_context_v2(
     query: CoordinatorQuery,
     *,
     database_path: Path,
-    index: RawIndexLookup | None = None,
     deadline: float,
     expand_refs: ExpandRefs | None = None,
 ) -> RetrievalResponse:
@@ -1198,7 +1041,6 @@ async def search_context_v2(
             authority,
             query,
             database_path=database_path,
-            index=index,
             deadline=deadline,
             expand_refs=expand_refs,
         ),

memhub/apps/core/src/one_memory/diagnostic_log.py

@@ -20,8 +20,10 @@ from __future__ import annotations
 import atexit
 import json
 import logging
+import math
 import os
 import re
+import stat
 import sys
 import threading
 import time
@@ -72,80 +74,30 @@ class _Refused(OSError):
     """The path is not one we are willing to write to, and never will be."""
 
 
-DiagnosticLevel = Literal["info", "warn", "error"]
+DiagnosticLevel = Literal["debug", "info", "warn", "error"]
 _IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
 _EVENT = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$")
-_DETAIL_KEYS = frozenset(
-    {
-        "action",
-        "attempt",
-        "count",
-        "method",
-        "mode",
-        "phase",
-        "platform",
-        "reason",
-        "result_count",
-        "source_id",
-        "spawned",
-        "status",
-        "version",
-        "parent_session_id",
-        "invalid_lines",
-        "truncated",
-        "open_file_count",
-        "memory_bytes",
-        "thread_count",
-        "database_error_count",
-        "route_path",
-        "sqlite_errorcode",
-        "sqlite_errorname",
-        "file_count",
-        "sqlite_file_count",
-        "byte_file_count",
-        "total_bytes",
-        "copied_bytes",
-        "bytes_per_second",
-        "stored_count",
-        "revised_count",
-        "skipped_count",
-        "error_count",
-        "pending_count",
-        "prepared_count",
-        "queue_depth",
-        "in_flight_count",
-        "call_count",
-        "created_count",
-        "updated_count",
-        "superseded_count",
-        "deleted_count",
-        "operation_kind",
-        "index_complete",
-        "lexical_hits",
-        "semantic_hits",
-        "generation",
-        "timed_out",
-        "failure_site",
-        "error_class",
-        "file_kind",
-        "file_size",
-        "file_mtime_ns",
-        "file_inode",
-        "cursor_new",
-        "cursor_old",
-        "cursor_read_to_end",
-        "cursor_rewind_branch",
-        "summed_ms",
-        "model_ms",
-        "accounted_ms",
-        "unaccounted_ms",
-        "conversation_hash",
-        # The sedimentation switch: a bounded boolean, the same shape as
-        # `spawned` and `index_complete`. Left out of this list it did not
-        # drop a log line, it turned the switch's own request into a 500.
-        "enabled",
-    }
+_DIAGNOSTIC_LOG_NAME = re.compile(
+    r"^diagnostic-(runtime|core|host-bridge|desktop|cli)\.jsonl(?:\.\d+)?$"
 )
+_DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
+_CODE_FRAME = re.compile(
+    r"^(?:(?:apps|packages|scripts)/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:\d+"
+    r"|one_memory/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:[A-Za-z_][A-Za-z0-9_]*"
+    r"|[A-Za-z0-9_][A-Za-z0-9_.-]*\.py:\d+:[A-Za-z_][A-Za-z0-9_]*)$"
+)
+_SHORT_CREDENTIAL_TOKEN = re.compile(
+    r"(?<![A-Za-z0-9])([A-Za-z0-9]{24,47})(?:[_-])?(?![A-Za-z0-9])"
+)
+_REFUSED_DETAIL_KEYS = frozenset({"__proto__", "prototype", "constructor"})
+_MAX_DETAIL_KEYS = 32
+_MAX_SAFE_INTEGER = (1 << 53) - 1
+_EXPORT_SECRET_MARKER = re.compile(
+    r"(?:^|[._:-])(?:authorization|credential|password|prompt|query|secret|token)(?:$|[._:-])",
+    re.IGNORECASE,
+)
+_TRACE_STRING_DETAIL_KEYS = frozenset({"phase", "mode", "status", "reason", "failure_site"})
+_TRACE_NUMBER_DETAIL_KEYS = frozenset({"count"})
 _trace_id: ContextVar[str | None] = ContextVar("one_memory_trace_id", default=None)
 _active: DiagnosticLog | None = None
 _database_errors: Counter[tuple[str, str]] = Counter()
@@ -264,7 +216,12 @@ class DiagnosticLog:
         }
         value.update({key: _identifier(item) for key, item in identifiers.items() if item})
         if duration_ms is not None:
-            if not isinstance(duration_ms, int) or duration_ms < 0:
+            if (
+                not isinstance(duration_ms, int)
+                or isinstance(duration_ms, bool)
+                or duration_ms < 0
+                or duration_ms > _MAX_SAFE_INTEGER
+            ):
                 raise ValueError("diagnostic_duration_invalid")
             value["duration_ms"] = duration_ms
         for field_name, item in {
@@ -276,7 +233,9 @@ class DiagnosticLog:
             if item is not None:
                 value[field_name] = _identifier(item)
         if safe_stack:
-            value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack[:256]]
+            if len(safe_stack) > 16:
+                raise ValueError("diagnostic_stack_invalid")
+            value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack]
         if details:
             value["details"] = _details(details)
         self._submit(_serialize(value))
@@ -289,6 +248,8 @@ class DiagnosticLog:
         event: str,
         outcome: str,
     ) -> dict[str, object]:
+        if level not in ("debug", "info", "warn", "error"):
+            raise ValueError("diagnostic_level_invalid")
         return {
             "schema_version": 1,
             "timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
@@ -651,24 +612,17 @@ def _drain_diagnostics_at_exit() -> None:
 
 
 def diagnostic_event(**values: object) -> None:
-    active = _active
-    if active is None:
-        return
-    if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
-        return
     try:
+        active = _active
+        if active is None:
+            return
+        if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
+            return
         active.write(**values)  # type: ignore[arg-type]
-    except OSError:
-        # Diagnostics must not become a new product outage when a disk is full
-        # or the log directory is temporarily unavailable.
-        return
-    except ValueError:
-        # Same promise, same reason, for a rejected payload instead of a failed
-        # write. An unlisted `details` key once turned every external recall
-        # into a 500 and then killed the process, and the sedimentation switch
-        # into a permanent failure. The payload itself is still wrong and is
-        # still caught in CI by tests/test_diagnostic_payload_keys.py; what
-        # must not happen is the caller paying for it.
+    except Exception:
+        # Diagnostics are best-effort: validation, serialization, logging, and
+        # filesystem failures must never become a product outage. BaseException
+        # deliberately remains visible so process-control signals still work.
         return
 
 
@@ -691,46 +645,95 @@ def read_trace_events(
     if (request_id is None) == (trace_id is None):
         raise ValueError("exactly_one_trace_identity_required")
     key, expected = ("request_id", request_id) if request_id is not None else ("trace_id", trace_id)
-    if expected is None or not _IDENTIFIER.fullmatch(expected):
+    if expected is None or not _export_identifier(expected):
         raise ValueError("invalid_trace_identity")
     selected: list[dict[str, object]] = []
-    allowed_details = {"phase", "mode", "status", "reason", "failure_site", "count"}
-    files = sorted(
-        (data_dir / "logs").glob("diagnostic-*.jsonl*"),
-        key=lambda path: path.stat().st_mtime,
-    )
-    for path in files:
-        if path.is_symlink() or not path.is_file():
+    directory = data_dir / "logs"
+    try:
+        directory_info = directory.lstat()
+    except FileNotFoundError:
+        return ()
+    if stat.S_ISLNK(directory_info.st_mode) or not stat.S_ISDIR(directory_info.st_mode):
+        raise ValueError("diagnostic_directory_unsafe")
+    files: list[tuple[Path, float]] = []
+    for path in directory.iterdir():
+        if not _DIAGNOSTIC_LOG_NAME.fullmatch(path.name):
             continue
-        for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        try:
+            info = path.lstat()
+        except OSError:
+            continue
+        if stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode):
+            files.append((path, info.st_mtime))
+    for path, _mtime in sorted(files, key=lambda item: item[1]):
+        for line in _read_regular_file_no_follow(path).splitlines():
             try:
                 event = json.loads(line)
             except (TypeError, json.JSONDecodeError):
                 continue
             if not isinstance(event, dict) or event.get(key) != expected:
                 continue
-            details = event.get("details")
-            projected = {
-                name: event[name]
-                for name in (
-                    "timestamp",
-                    "component",
-                    "event",
-                    "outcome",
-                    "duration_ms",
-                    "error_code",
-                    "error_type",
-                )
-                if name in event
+            timestamp = event.get("timestamp")
+            if not isinstance(timestamp, str):
+                continue
+            try:
+                datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+            except ValueError:
+                continue
+            component = event.get("component")
+            event_name = event.get("event")
+            outcome = event.get("outcome")
+            projected: dict[str, object] = {
+                "timestamp": timestamp,
+                "component": (
+                    component
+                    if isinstance(component, str) and _export_identifier(component)
+                    else "diagnostic.redacted"
+                ),
+                "event": (
+                    event_name
+                    if isinstance(event_name, str) and _export_identifier(event_name)
+                    else "diagnostic.redacted"
+                ),
+                "outcome": (
+                    outcome
+                    if isinstance(outcome, str) and _export_identifier(outcome)
+                    else "redacted"
+                ),
             }
+            duration_ms = event.get("duration_ms")
+            if (
+                isinstance(duration_ms, int)
+                and not isinstance(duration_ms, bool)
+                and 0 <= duration_ms <= _MAX_SAFE_INTEGER
+            ):
+                projected["duration_ms"] = duration_ms
+            for name in ("error_code", "error_type"):
+                value = event.get(name)
+                if isinstance(value, str) and _export_identifier(value):
+                    projected[name] = value
+            details = event.get("details")
             if isinstance(details, dict):
-                projected["details"] = {
-                    name: value for name, value in details.items() if name in allowed_details
-                }
+                safe_details = _trace_details(details)
+                if safe_details:
+                    projected["details"] = safe_details
             selected.append(projected)
     return tuple(selected[-max(1, min(limit, 500)) :])
 
 
+def _read_regular_file_no_follow(path: Path) -> str:
+    descriptor = os.open(path, os.O_RDONLY | _O_NOFOLLOW)
+    try:
+        if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+            raise ValueError("diagnostic_file_unsafe")
+        with os.fdopen(descriptor, encoding="utf-8", errors="replace") as handle:
+            descriptor = -1
+            return handle.read()
+    finally:
+        if descriptor >= 0:
+            os.close(descriptor)
+
+
 def current_trace_id() -> str | None:
     return _trace_id.get()
 
@@ -837,8 +840,77 @@ def diagnostic_exception(error: BaseException) -> dict[str, object]:
     }
 
 
+def _contains_credential_material(value: str) -> bool:
+    return bool(
+        re.search(
+            r"(?:^|[^A-Za-z0-9])(?:sk-|sk_|omhg_|ghp_|github_pat_)[A-Za-z0-9._-]+",
+            value,
+            re.IGNORECASE,
+        )
+        or re.search(r"(?:^|[^A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]+", value, re.IGNORECASE)
+        or re.search(r"(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])", value)
+        or re.search(r"(?:^|[^A-Za-z0-9])AKID[A-Za-z0-9]{12,}", value, re.IGNORECASE)
+        or re.search(r"(?:^|[^A-Za-z0-9])AIza[A-Za-z0-9_-]{20,}", value)
+        or re.search(r"(?:^|[^A-Za-z0-9])GOCSPX-[A-Za-z0-9_-]+", value)
+        or any(
+            _is_conservative_short_credential(match.group(1))
+            for match in _SHORT_CREDENTIAL_TOKEN.finditer(value)
+        )
+        or re.search(
+            r"\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b",
+            value,
+        )
+        or any(
+            len(candidate) >= 48
+            and any(character.isalpha() for character in candidate)
+            and any(character.isdigit() for character in candidate)
+            for candidate in re.findall(r"\b[A-Za-z0-9._-]{48,}\b", value)
+        )
+    )
+
+
+def _is_conservative_short_credential(value: str) -> bool:
+    return bool(
+        re.search(r"[A-Z]", value)
+        and re.search(r"[a-z]", value)
+        and re.search(r"\d", value)
+        and not re.fullmatch(r"[A-Fa-f0-9]+", value)
+    )
+
+
+def _export_identifier(value: str) -> bool:
+    return bool(
+        _IDENTIFIER.fullmatch(value)
+        and not _EXPORT_SECRET_MARKER.search(value)
+        and not _contains_credential_material(value)
+    )
+
+
+def _trace_details(values: Mapping[object, object]) -> dict[str, str | int | float]:
+    result: dict[str, str | int | float] = {}
+    for key, value in values.items():
+        if (
+            key in _TRACE_STRING_DETAIL_KEYS
+            and isinstance(value, str)
+            and _export_identifier(value)
+        ):
+            result[str(key)] = value
+        elif key in _TRACE_NUMBER_DETAIL_KEYS and isinstance(value, (int, float)):
+            if isinstance(value, bool):
+                continue
+            safe_integer = isinstance(value, int) and abs(value) <= _MAX_SAFE_INTEGER
+            safe_float = (
+                isinstance(value, float)
+                and math.isfinite(value)
+                and (not value.is_integer() or abs(value) <= _MAX_SAFE_INTEGER)
+            )
+            if safe_integer or safe_float:
+                result[str(key)] = value
+    return result
+
+
 def _serialize(value: Mapping[str, object]) -> str:
-    return json.dumps(value, ensure_ascii=True, separators=(",", ":")) + "\n"
+    return json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")) + "\n"
 
 
 def _uuid(value: str | None) -> str | None:
@@ -864,24 +936,47 @@ def _event_name(value: str) -> str:
 
 
 def _safe_stack_line(value: str) -> str:
-    if not value or len(value) > 256 or "\n" in value or "\r" in value:
+    if (
+        not value
+        or len(value) > 256
+        or any(character in value for character in "\0\r\n")
+        or _contains_credential_material(value)
+    ):
         raise ValueError("diagnostic_stack_invalid")
-    if Path(value.split(":", 1)[0]).is_absolute() or "file://" in value:
-        raise ValueError("diagnostic_stack_invalid")
-    return value
+    if _CODE_FRAME.fullmatch(value):
+        return value
+    if value.startswith("thread|"):
+        frames = value.removeprefix("thread|").split(" > ")
+        if frames and all(_CODE_FRAME.fullmatch(frame) for frame in frames):
+            return value
+    raise ValueError("diagnostic_stack_invalid")
 
 
 def _details(
     values: Mapping[str, str | int | float | bool | None],
 ) -> dict[str, str | int | float | bool | None]:
+    if len(values) > _MAX_DETAIL_KEYS:
+        raise ValueError("diagnostic_detail_count_invalid")
     result: dict[str, str | int | float | bool | None] = {}
     for key, value in values.items():
-        if key not in _DETAIL_KEYS:
+        if (
+            not isinstance(key, str)
+            or not _DETAIL_KEY.fullmatch(key)
+            or key in _REFUSED_DETAIL_KEYS
+        ):
             raise ValueError("diagnostic_detail_key_invalid")
         if isinstance(value, str):
-            if not value or len(value) > 256 or "\n" in value or "\r" in value:
+            if not value or len(value) > 256 or any(character in value for character in "\0\r\n"):
                 raise ValueError("diagnostic_detail_value_invalid")
-        elif value is not None and not isinstance(value, (int, float, bool)):
+        elif isinstance(value, float):
+            if not math.isfinite(value) or (value.is_integer() and abs(value) > _MAX_SAFE_INTEGER):
+                raise ValueError("diagnostic_detail_value_invalid")
+        elif isinstance(value, bool) or value is None:
+            pass
+        elif isinstance(value, int):
+            if abs(value) > _MAX_SAFE_INTEGER:
+                raise ValueError("diagnostic_detail_value_invalid")
+        else:
             raise ValueError("diagnostic_detail_value_invalid")
         result[key] = value
     return result

memhub/apps/core/src/one_memory/exception_reporting.py

@@ -3,8 +3,8 @@
 Every site that catches `Exception` broadly must call `report_exception`
 before (or instead of) continuing, so a failure never looks like success
 and a stack is always on disk. Content (conversation text, keys, SQL
-parameters) never reaches the log: only exception type, a truncated
-message, and file:line:function frames.
+parameters, and exception messages) never reaches the diagnostic log: only
+exception type, a repeat count, and file:line:function frames.
 """
 
 from __future__ import annotations
@@ -67,7 +67,7 @@ def report_exception(
         event=event,
         outcome="failed",
         **payload,
-        details={"reason": _safe_message(error), "count": _RECENT[key]},
+        details={"count": _RECENT[key]},
     )
     if full:
         _logger.error(

memhub/apps/core/src/one_memory/hy_sdk_migration.py

@@ -1,18 +1,13 @@
-"""Privileged offline access to the HY SDK data plane during App migration.
+"""Offline snapshot and restore of the HY SDK data plane for the App migration.
 
-Runtime product code must use public ``HyMemoryClient`` methods.  This module
-is the deliberately isolated exception: App startup has stopped all writers,
-so the one-time legacy migration may access SDK stores directly without
-turning migration mechanics into public Client APIs.
+Runtime product code must use public ``HyMemoryClient`` methods.  The startup
+migration bundle is the isolated exception: all writers are stopped, so it may
+copy and restore the SDK-owned local stores as files.
 """
 
 from __future__ import annotations
 
-import asyncio
-from collections.abc import Callable, Mapping
-from datetime import datetime
 from pathlib import Path
-from typing import Any, Protocol
 
 from hy_memory import (  # type: ignore[import-untyped]
     DataPlaneRestore,
@@ -20,380 +15,6 @@ from hy_memory import (  # type: ignore[import-untyped]
     restore_local_data_plane,
     snapshot_local_data_plane,
 )
-from hy_memory.models.memory import (  # type: ignore[import-untyped]
-    MemoryLayer,
-    MemoryNode,
-    MemoryStatus,
-    SourceType,
-)
-from hy_memory.utils.source_filter import normalize_source  # type: ignore[import-untyped]
-
-from one_memory.hy_sdk_types import (
-    HySdkMemoryPage,
-    HySdkMemoryRecord,
-    HySdkMigrationError,
-    HySdkRawPage,
-    HySdkRawRecord,
-)
-
-
-class MigrationClient(Protocol):
-    _loop_thread: Any
-    _vector_store: Any
-    _embed_service: Any
-    _graph_store: Any
-
-    def list_memories(
-        self,
-        *,
-        user_id: str,
-        agent_id: str | None = None,
-        statuses: list[str] | None = None,
-        limit: int = 100,
-        offset: int = 0,
-        order: str = "desc",
-    ) -> Mapping[str, object]: ...
-
-    def delete(self, memory_id: str) -> Mapping[str, object]: ...
-
-    def get_write_outcome(self, request_id: str) -> Mapping[str, object] | None: ...
-
-
-class HySdkMigration:
-    """Typed offline adapter over one initialized SDK data plane."""
-
-    def __init__(self, client: MigrationClient) -> None:
-        self.client = client
-        self._loop_thread = client._loop_thread
-        self._vector_store = client._vector_store
-        self._embed_service = client._embed_service
-        self._graph_store = client._graph_store
-
-    async def list_memories(
-        self,
-        *,
-        user_id: str,
-        agent_id: str | None = None,
-        statuses: tuple[str, ...] = ("active",),
-        limit: int = 100,
-        offset: int = 0,
-        order: str = "desc",
-    ) -> HySdkMemoryPage:
-        raw = await _sdk_call(
-            "hy_sdk_memory_list_failed",
-            self.client.list_memories,
-            user_id=user_id,
-            agent_id=agent_id,
-            statuses=list(statuses),
-            limit=limit,
-            offset=offset,
-            order=order,
-        )
-        return _memory_page(raw, limit=limit, offset=offset)
-
-    async def write_legacy_memory(
-        self,
-        *,
-        content: str,
-        memory_id: str,
-        user_id: str,
-        agent_id: str,
-        session_id: str,
-        source: str,
-        layer: str,
-        status: str,
-        owner: str | None,
-        tags: tuple[str, ...],
-        supersedes: tuple[str, ...],
-        superseded_by: tuple[str, ...],
-        is_latest: bool,
-        memory_at: datetime | None,
-        metadata: Mapping[str, object],
-        request_id: str,
-    ) -> dict[str, object]:
-        raw = await _sdk_call(
-            "hy_sdk_memory_import_failed",
-            self._loop_thread.run,
-            self._write_legacy_memory(
-                content=content,
-                memory_id=memory_id,
-                user_id=user_id,
-                agent_id=agent_id,
-                session_id=session_id,
-                source=source,
-                layer=layer,
-                status=status,
-                owner=owner,
-                tags=tags,
-                supersedes=supersedes,
-                superseded_by=superseded_by,
-                is_latest=is_latest,
-                memory_at=memory_at,
-                metadata=metadata,
-                request_id=request_id,
-            ),
-        )
-        return _successful_result(raw, "hy_sdk_memory_import_failed")
-
-    async def _write_legacy_memory(
-        self,
-        *,
-        content: str,
-        memory_id: str,
-        user_id: str,
-        agent_id: str,
-        session_id: str,
-        source: str,
-        layer: str,
-        status: str,
-        owner: str | None,
-        tags: tuple[str, ...],
-        supersedes: tuple[str, ...],
-        superseded_by: tuple[str, ...],
-        is_latest: bool,
-        memory_at: datetime | None,
-        metadata: Mapping[str, object],
-        request_id: str,
-    ) -> dict[str, object]:
-        if not content or not memory_id or not user_id:
-            raise ValueError("content, memory_id and user_id are required")
-        normalized_layer = MemoryLayer.from_string(layer)
-        if normalized_layer is MemoryLayer.L1_RAW:
-            raise ValueError("raw content must use the raw migration lane")
-        normalized_status = MemoryStatus(status.lower())
-        normalized_source = normalize_source(source)
-        existing = await self._vector_store.get_by_id(memory_id)
-        if existing is not None:
-            identity = (
-                existing.content == content,
-                existing.user_id == user_id,
-                existing.agent_id == agent_id,
-                existing.session_id == session_id,
-                existing.source == normalized_source,
-                existing.layer == normalized_layer,
-                existing.status == normalized_status,
-                existing.owner == owner,
-                list(existing.tags or []) == list(tags),
-                list(existing.supersedes or []) == list(supersedes),
-                list(existing.superseded_by or []) == list(superseded_by),
-                existing.is_latest == is_latest,
-            )
-            if not all(identity):
-                raise ValueError("memory_id is already bound to different memory content")
-            await self._project_graph(existing)
-            return {
-                "success": True,
-                "memory_id": memory_id,
-                "memory_ids": [memory_id],
-                "request_id": request_id,
-                "replayed": True,
-            }
-        custom = dict(metadata)
-        custom["migration_request_id"] = request_id
-        embedding = await self._embed_service.embed(content)
-        node = MemoryNode(
-            node_id=memory_id,
-            user_id=user_id,
-            agent_id=agent_id,
-            session_id=session_id,
-            source=normalized_source,
-            layer=normalized_layer,
-            content=content,
-            source_type=SourceType.EXPLICIT,
-            status=normalized_status,
-            owner=owner,
-            tags=list(tags),
-            supersedes=list(supersedes),
-            superseded_by=list(superseded_by),
-            is_latest=is_latest,
-            embedding=embedding,
-            memory_at=memory_at,
-            custom=custom,
-        )
-        stored_id = await self._vector_store.upsert(node)
-        if stored_id != memory_id:
-            raise RuntimeError("exact migration changed the requested memory identity")
-        await self._project_graph(node)
-        return {
-            "success": True,
-            "memory_id": memory_id,
-            "memory_ids": [memory_id],
-            "request_id": request_id,
-        }
-
-    async def _project_graph(self, node: MemoryNode) -> None:
-        if self._graph_store is not None:
-            await self._graph_store.upsert_memory_node(node)
-
-    async def read_raw_inventory(
-        self,
-        *,
-        user_id: str,
-        agent_id: str | None = None,
-        session_id: str | None = None,
-        source: str | None = None,
-        limit: int = 100,
-        offset: int = 0,
-        order: str = "desc",
-    ) -> HySdkRawPage:
-        raw = await _sdk_call(
-            "hy_sdk_raw_list_failed",
-            self._loop_thread.run,
-            self._read_raw_inventory(
-                user_id=user_id,
-                agent_id=agent_id,
-                session_id=session_id,
-                source=source,
-                limit=limit,
-                offset=offset,
-                order=order,
-            ),
-        )
-        return _raw_page(raw, limit=limit, offset=offset)
-
-    async def _read_raw_inventory(
-        self,
-        *,
-        user_id: str,
-        agent_id: str | None,
-        session_id: str | None,
-        source: str | None,
-        limit: int,
-        offset: int,
-        order: str,
-    ) -> dict[str, object]:
-        if not user_id:
-            raise ValueError("user_id is required")
-        nodes = await self._vector_store.list_by_user(
-            user_id=user_id,
-            agent_id=agent_id,
-            status_filter=[MemoryStatus.ACTIVE],
-            layers=[MemoryLayer.L1_RAW],
-            include_sources=[source] if source else None,
-            limit=100_000,
-            only_latest=False,
-            with_vectors=False,
-            allow_raw_layer=True,
-        )
-        if session_id:
-            nodes = [node for node in nodes if node.session_id == session_id]
-        nodes.sort(
-            key=_node_timestamp,
-            reverse=order.lower() != "asc",
-        )
-        return {
-            "raw": [_raw_node(node) for node in nodes[offset : offset + limit]],
-            "total": len(nodes),
-            "limit": limit,
-            "offset": offset,
-        }
-
-    async def write_legacy_raw(
-        self,
-        *,
-        content: str,
-        raw_id: str,
-        user_id: str,
-        agent_id: str,
-        session_id: str,
-        source: str,
-        memory_at: datetime | None,
-        metadata: Mapping[str, object],
-        request_id: str,
-    ) -> dict[str, object]:
-        raw = await _sdk_call(
-            "hy_sdk_raw_import_failed",
-            self._loop_thread.run,
-            self._write_legacy_raw(
-                content=content,
-                raw_id=raw_id,
-                user_id=user_id,
-                agent_id=agent_id,
-                session_id=session_id,
-                source=source,
-                memory_at=memory_at,
-                metadata=metadata,
-                request_id=request_id,
-            ),
-        )
-        return _successful_result(raw, "hy_sdk_raw_import_failed")
-
-    async def _write_legacy_raw(
-        self,
-        *,
-        content: str,
-        raw_id: str,
-        user_id: str,
-        agent_id: str,
-        session_id: str,
-        source: str,
-        memory_at: datetime | None,
-        metadata: Mapping[str, object],
-        request_id: str,
-    ) -> dict[str, object]:
-        if not content or not raw_id or not user_id:
-            raise ValueError("content, raw_id and user_id are required")
-        normalized_source = normalize_source(source)
-        existing = await self._vector_store.get_by_id(raw_id)
-        if existing is not None:
-            identity = (
-                existing.layer is MemoryLayer.L1_RAW,
-                existing.content == content,
-                existing.user_id == user_id,
-                existing.agent_id == agent_id,
-                existing.session_id == session_id,
-                existing.source == normalized_source,
-            )
-            if not all(identity):
-                raise ValueError("raw_id is already bound to different source content")
-            return {
-                "success": True,
-                "raw_id": raw_id,
-                "request_id": request_id,
-                "replayed": True,
-            }
-        custom = dict(metadata)
-        custom["migration_request_id"] = request_id
-        embedding = await self._embed_service.embed(content)
-        node = MemoryNode(
-            node_id=raw_id,
-            user_id=user_id,
-            agent_id=agent_id,
-            session_id=session_id,
-            source=normalized_source,
-            layer=MemoryLayer.L1_RAW,
-            content=content,
-            source_type=SourceType.EXPLICIT,
-            status=MemoryStatus.ACTIVE,
-            is_latest=True,
-            embedding=embedding,
-            memory_at=memory_at,
-            custom=custom,
-        )
-        stored_id = await self._vector_store.upsert(node)
-        if stored_id != raw_id:
-            raise RuntimeError("exact migration changed the requested raw identity")
-        return {"success": True, "raw_id": raw_id, "request_id": request_id}
-
-    async def delete(self, memory_id: str) -> dict[str, object]:
-        raw = await _sdk_call(
-            "hy_sdk_memory_delete_failed",
-            self.client.delete,
-            memory_id,
-        )
-        return _successful_result(raw, "hy_sdk_memory_delete_failed")
-
-    async def get_write_outcome(self, request_id: str) -> dict[str, object] | None:
-        raw = await _sdk_call(
-            "hy_sdk_write_outcome_failed",
-            self.client.get_write_outcome,
-            request_id,
-        )
-        if raw is None:
-            return None
-        if not isinstance(raw, Mapping):
-            raise HySdkMigrationError("hy_sdk_write_outcome_shape_invalid")
-        return dict(raw)
 
 
 def snapshot_data_plane(
@@ -424,176 +45,3 @@ def restore_data_plane(
         writers_stopped=writers_stopped,
         protective_root=protective_root,
     )
-
-
-async def _sdk_call[T](
-    code: str,
-    call: Callable[..., T],
-    *args: object,
-    **kwargs: object,
-) -> T:
-    pending = asyncio.create_task(asyncio.to_thread(call, *args, **kwargs))
-    try:
-        return await asyncio.shield(pending)
-    except asyncio.CancelledError:
-        await _finish_cancelled_call(pending)
-        raise
-    except Exception as error:
-        raise HySdkMigrationError(code) from error
-
-
-async def _finish_cancelled_call[T](pending: asyncio.Task[T]) -> None:
-    while not pending.done():
-        try:
-            await asyncio.shield(pending)
-        except asyncio.CancelledError:
-            continue
-        except Exception:
-            return
-    if not pending.cancelled():
-        pending.exception()
-
-
-def _memory_page(raw: object, *, limit: int, offset: int) -> HySdkMemoryPage:
-    if not isinstance(raw, Mapping):
-        raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
-    rows: list[object] = []
-    total = 0
-    for bucket_name, row_name in (("vdb", "memories"), ("graph", "nodes")):
-        bucket = raw.get(bucket_name)
-        if bucket is None and bucket_name == "graph":
-            continue
-        if not isinstance(bucket, Mapping):
-            raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
-        bucket_rows = bucket.get(row_name)
-        bucket_total = bucket.get("total")
-        if not isinstance(bucket_rows, list) or type(bucket_total) is not int or bucket_total < 0:
-            raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
-        rows.extend(bucket_rows)
-        total += bucket_total
-
-    records: dict[str, HySdkMemoryRecord] = {}
-    for row in rows:
-        record = _memory_record(row)
-        records.setdefault(record.memory_id, record)
-    return HySdkMemoryPage(
-        memories=tuple(records.values()),
-        total=total,
-        limit=limit,
-        offset=offset,
-    )
-
-
-def _memory_record(raw: object) -> HySdkMemoryRecord:
-    if not isinstance(raw, Mapping):
-        raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid")
-    required = ("memory_id", "content", "layer", "status", "user_id", "agent_id")
-    if any(not isinstance(raw.get(name), str) or not raw[name] for name in required):
-        raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid")
-    custom = raw.get("custom")
-    source_raw_ids: list[object] = []
-    if isinstance(custom, Mapping) and isinstance(custom.get("source_raw_memory_ids"), list):
-        source_raw_ids.extend(custom["source_raw_memory_ids"])
-    if isinstance(raw.get("source_raw_memory_id"), str):
-        source_raw_ids.append(raw["source_raw_memory_id"])
-    try:
-        return HySdkMemoryRecord(
-            memory_id=raw["memory_id"],
-            content=raw["content"],
-            layer=raw["layer"],
-            status=raw["status"],
-            is_latest=bool(raw.get("is_latest", True)),
-            supersedes=_string_tuple(raw.get("supersedes")),
-            superseded_by=_string_tuple(raw.get("superseded_by")),
-            tags=_string_tuple(raw.get("tags")),
-            owner=raw.get("owner") if raw.get("owner") in {"user", "agent"} else None,
-            source_raw_ids=_string_tuple(source_raw_ids),
-            user_id=raw["user_id"],
-            agent_id=raw["agent_id"],
-            session_id=_optional_string(raw.get("session_id")),
-            source=_optional_string(raw.get("source")),
-            memory_at=_optional_int(raw.get("memory_at")),
-            gmt_created=_optional_int(raw.get("gmt_created")),
-        )
-    except (TypeError, ValueError) as error:
-        raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid") from error
-
-
-def _raw_page(raw: object, *, limit: int, offset: int) -> HySdkRawPage:
-    if not isinstance(raw, Mapping):
-        raise HySdkMigrationError("hy_sdk_raw_page_shape_invalid")
-    rows = raw.get("raw")
-    total = raw.get("total")
-    if not isinstance(rows, list) or type(total) is not int or total < 0:
-        raise HySdkMigrationError("hy_sdk_raw_page_shape_invalid")
-    return HySdkRawPage(
-        raw=tuple(_raw_record(row) for row in rows),
-        total=total,
-        limit=limit,
-        offset=offset,
-    )
-
-
-def _node_timestamp(node: Any) -> float:
-    value = getattr(node, "memory_at", None) or getattr(node, "gmt_created", None)
-    return value.timestamp() if isinstance(value, datetime) else 0.0
-
-
-def _raw_node(node: Any) -> dict[str, object]:
-    def timestamp(value: object) -> int | None:
-        return int(value.timestamp()) if isinstance(value, datetime) else None
-
-    return {
-        "raw_id": node.node_id,
-        "content": node.content,
-        "user_id": node.user_id,
-        "agent_id": node.agent_id,
-        "session_id": node.session_id,
-        "source": node.source,
-        "memory_at": timestamp(node.memory_at),
-        "gmt_created": timestamp(node.gmt_created),
-        "custom": node.custom or {},
-    }
-
-
-def _raw_record(raw: object) -> HySdkRawRecord:
-    if not isinstance(raw, Mapping):
-        raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid")
-    required = ("raw_id", "content", "user_id", "agent_id", "session_id", "source")
-    if any(not isinstance(raw.get(name), str) or not raw[name] for name in required):
-        raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid")
-    custom = raw.get("custom")
-    try:
-        return HySdkRawRecord(
-            raw_id=raw["raw_id"],
-            content=raw["content"],
-            user_id=raw["user_id"],
-            agent_id=raw["agent_id"],
-            session_id=raw["session_id"],
-            source=raw["source"],
-            memory_at=_optional_int(raw.get("memory_at")),
-            gmt_created=_optional_int(raw.get("gmt_created")),
-            custom=dict(custom) if isinstance(custom, Mapping) else {},
-        )
-    except (TypeError, ValueError) as error:
-        raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid") from error
-
-
-def _successful_result(raw: object, code: str) -> dict[str, object]:
-    if not isinstance(raw, Mapping) or raw.get("success") is not True:
-        raise HySdkMigrationError(code)
-    return dict(raw)
-
-
-def _string_tuple(value: object) -> tuple[str, ...]:
-    if not isinstance(value, list | tuple):
-        return ()
-    return tuple(item for item in value if isinstance(item, str) and item)
-
-
-def _optional_string(value: object) -> str | None:
-    return value if isinstance(value, str) else None
-
-
-def _optional_int(value: object) -> int | None:
-    return value if type(value) is int else None

memhub/apps/core/src/one_memory/hy_sdk_runtime.py

@@ -237,15 +237,9 @@ def build_sdk_config(
 
     root = data_dir.resolve() / "hy-memory" / "governed-v2"
     root.mkdir(parents=True, exist_ok=True)
-    collection_identity = "\0".join(
-        (
-            embedding_provider.value,
-            embedding_endpoint or "local",
-            embedding_model,
-            str(dimensions or "auto"),
-        )
+    collection = _collection_base(
+        embedding_provider, embedding_endpoint, embedding_model, dimensions
     )
-    collection = "memhub_" + hashlib.sha256(collection_identity.encode()).hexdigest()[:16]
 
     return MemoryConfig.from_dict(
         {
@@ -348,6 +342,49 @@ def _generation_access(
     return endpoint, api_key
 
 
+def _collection_base(
+    provider: ProviderKind,
+    endpoint: str | None,
+    model: str,
+    dimensions: int | None,
+) -> str:
+    identity = "\0".join((provider.value, endpoint or "local", model, str(dimensions or "auto")))
+    return "memhub_" + hashlib.sha256(identity.encode()).hexdigest()[:16]
+
+
+def vector_collection_bases(profile: ModelProfileSnapshot) -> tuple[str, tuple[str, ...]]:
+    """The SDK vector collection base for this embedding space, and the older ones.
+
+    The SDK derives every vector collection (chat, Coding keys, Working facts,
+    each variant) by suffixing this base. Builds before #145 named the same
+    embedding space `one_memory_<12 hex>`, with or without a declared width;
+    the offline migration renames those so their vectors stay readable.
+    """
+    provider = profile.embedding_provider_kind or profile.provider_kind
+    local = provider is ProviderKind.LOCAL_ONNX
+    model = profile.embedding_model or (BUILTIN_EMBEDDING_MODEL if local else None)
+    if not model:
+        raise HySdkRuntimeError("hy_sdk_embedding_model_required")
+    endpoint: str | None
+    dimensions: int | None
+    if local:
+        endpoint, dimensions = None, profile.embedding_dimensions or BUILTIN_EMBEDDING_DIMENSIONS
+        legacy_endpoint = "local-onnx"
+    else:
+        endpoint = _openai_endpoint(provider, profile.embedding_base_url or profile.base_url)
+        dimensions = profile.embedding_dimensions
+        legacy_endpoint = endpoint
+    legacy_model = profile.embedding_model or "bge-small-zh-v1.5"
+    legacy = tuple(
+        "one_memory_"
+        + hashlib.sha256(
+            ":".join((provider.value, legacy_endpoint, legacy_model, *declared)).encode()
+        ).hexdigest()[:12]
+        for declared in ((), ("auto",), (str(profile.embedding_dimensions or "auto"),))
+    )
+    return _collection_base(provider, endpoint, model, dimensions), tuple(dict.fromkeys(legacy))
+
+
 def _openai_endpoint(provider: ProviderKind, endpoint: str | None) -> str:
     value = (endpoint or "").rstrip("/")
     if not value:

memhub/apps/core/src/one_memory/imports/live_sync.py

@@ -557,12 +557,10 @@ class PollingLiveSyncSupervisor:
         )
 
     def _note_internal_error(self, source_id: str, error: Exception) -> None:
-        # Type alone used to be all this said. It is not enough to act on:
-        # `OperationalError` covers a locked database, a missing table and a
-        # corrupt page alike, and without the message and the frames two dozen
-        # of these in a log could only be guessed at. The stack is the
-        # redacted, `one_memory`-relative one, so it names the call site
-        # without printing anyone's paths or query text.
+        # The ordinary local log keeps the exception message for debugging.
+        # The structured diagnostic event deliberately keeps only the stable
+        # error code, source id, type, and relative stack: exception messages
+        # can contain paths, query text, or credentials and are exportable data.
         logging.getLogger(__name__).warning(
             "live_sync_internal_error %s: %s", type(error).__name__, error
         )
@@ -572,10 +570,7 @@ class PollingLiveSyncSupervisor:
             event="live_sync.internal_error",
             outcome="failed",
             error_code="live_sync_internal_error",
-            # `reason` is the listed key that can carry the message; see
-            # `diagnostic_log._DETAIL_KEYS`. Truncated because a detail value
-            # longer than 256 characters is rejected.
-            details={"source_id": source_id, "reason": str(error)[:200]},
+            details={"source_id": source_id},
             **diagnostic_exception(error),
         )
         if isinstance(error, sqlite3.Error):

memhub/apps/core/src/one_memory/imports/runner.py

@@ -867,10 +867,7 @@ class ImportRunner:
             outcome="failed",
             job_id=job_id,
             error_code="import_heartbeat_failed",
-            # `reason` is the listed key for this; see
-            # `diagnostic_log._DETAIL_KEYS`. Truncated: a detail value
-            # over 256 characters is rejected.
-            details={"reason": str(error)[:200]},
+            details={"phase": "heartbeat"},
             **diagnostic_exception(error),
         )
 

memhub/apps/core/src/one_memory/live_capture.py

@@ -1,6 +1,7 @@
 """Write visible hook text into Evidence. Tool output stays out."""
 
 import json
+import os
 import time
 from datetime import UTC, datetime
 from enum import StrEnum
@@ -74,6 +75,7 @@ _FILE_FALLBACK = {
 }
 _FALLBACK_EVENTS = frozenset({"Stop", "stop", "SubagentStop", "StopFailure"})
 _TEXT_BLOCKS = frozenset({"text"})
+SDK_MIGRATION_CAPTURE_FENCE = ".sdk-migration-capture-fence.json"
 
 
 def database_path() -> Path:
@@ -88,17 +90,42 @@ def capture_stdio(source_id: str, stdin: TextIO, output: TextIO) -> int:
     del output
     if source_id not in CAPTURE_SOURCES:
         return 2
+    database = database_path()
+    if sdk_migration_capture_fenced(database.parent):
+        return 0
     try:
         payload = json.loads(stdin.read() or "")
     except json.JSONDecodeError:
         return 0
     try:
-        capture_hook(source_id, payload, database_path())
+        capture_hook(source_id, payload, database)
     except Exception:
         return 0
     return 0
 
 
+def sdk_migration_capture_fenced(data_dir: Path) -> bool:
+    """Ignore stale fences; only a live Desktop migration pauses hook writes."""
+    marker = data_dir / SDK_MIGRATION_CAPTURE_FENCE
+    try:
+        document = json.loads(marker.read_text(encoding="utf-8"))
+        pid = int(document["pid"])
+        if pid < 1:
+            raise ValueError("invalid migration fence pid")
+        os.kill(pid, 0)
+        return True
+    except FileNotFoundError:
+        return False
+    except ProcessLookupError:
+        marker.unlink(missing_ok=True)
+        return False
+    except PermissionError:
+        return True
+    except (KeyError, TypeError, ValueError, json.JSONDecodeError, OSError):
+        marker.unlink(missing_ok=True)
+        return False
+
+
 def capture_hook(source_id: str, payload: object, database: Path) -> bool:
     if source_id not in CAPTURE_SOURCES or not isinstance(payload, dict):
         return False

memhub/apps/core/src/one_memory/local_embedding.py

@@ -10,16 +10,12 @@ weights alone.
 from __future__ import annotations
 
 import threading
-from dataclasses import dataclass
-from hashlib import sha256
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
 from one_memory.embedding_assets import (
     BUILTIN_EMBEDDING_DIMENSIONS,
     BUILTIN_EMBEDDING_MAX_TOKENS,
-    EMBEDDING_ASSETS,
-    AssetVerificationError,
     MissingEmbeddingAssetsError,
     assets_present,
     resolve_asset_dir,
@@ -34,28 +30,6 @@ _FETCH_HINT = (
 )
 
 
-@dataclass(frozen=True)
-class TokenOffset:
-    """Half-open Python Unicode code point offsets; synthetic specials use [0,0)."""
-
-    start_cp: int
-    end_cp: int
-    special: bool = False
-
-
-@dataclass(frozen=True)
-class TokenizedText:
-    """Untruncated, unpadded encoding, including tokenizer postprocessor specials."""
-
-    token_ids: tuple[int, ...]
-    offsets: tuple[TokenOffset, ...]
-
-    def __post_init__(self) -> None:
-        # Defensive copies also keep runtime callers from retaining mutable lists.
-        object.__setattr__(self, "token_ids", tuple(self.token_ids))
-        object.__setattr__(self, "offsets", tuple(self.offsets))
-
-
 class LocalEmbedder:
     """Encodes text with the built-in model, loading it on first use."""
 
@@ -68,43 +42,6 @@ class LocalEmbedder:
         self._session: Any | None = None
         self._tokenizer: Any | None = None
         self._input_names: frozenset[str] = frozenset()
-        self._offset_lock = threading.Lock()
-        self._offset_tokenizer: Any | None = None
-
-    def tokenize_with_offsets(self, text: str) -> TokenizedText:
-        """Tokenize original text without encode()'s truncation or batch padding.
-
-        The Python tokenizers binding returns code point offsets into the input,
-        even when its model normalizer changes spelling. This separate lazy
-        tokenizer uses the exact locked asset; it never loads an ONNX session or
-        mutates the tokenizer shared by encode(). Offsets are not UTF-16 indices.
-        """
-        if not isinstance(text, str):
-            raise TypeError("embedding_offsets_require_text")
-        with self._offset_lock:
-            if self._offset_tokenizer is None:
-                directory = resolve_asset_dir()
-                path = directory / "tokenizer.json"
-                if not path.is_file():
-                    raise MissingEmbeddingAssetsError(f"{_FETCH_HINT} Looked in {directory}.")
-                data = path.read_bytes()
-                expected = next(a.sha256 for a in EMBEDDING_ASSETS if a.filename == path.name)
-                if sha256(data).hexdigest() != expected:
-                    raise AssetVerificationError("tokenizer.json: locked digest mismatch")
-                from tokenizers import Tokenizer
-
-                tokenizer = Tokenizer.from_str(data.decode("utf-8"))
-                tokenizer.no_truncation()
-                tokenizer.no_padding()
-                self._offset_tokenizer = tokenizer
-            encoding = self._offset_tokenizer.encode(text, add_special_tokens=True)
-            offsets = tuple(
-                TokenOffset(start, end, bool(special))
-                for (start, end), special in zip(
-                    encoding.offsets, encoding.special_tokens_mask, strict=True
-                )
-            )
-            return TokenizedText(tuple(encoding.ids), offsets)
 
     def _load(self) -> tuple[Any, Any]:
         # Double-checked under a lock: memory processing encodes from worker

memhub/apps/core/src/one_memory/loop_watchdog.py

@@ -14,6 +14,7 @@ stuck in it.
 """
 
 import logging
+import re
 import sys
 import threading
 import time
@@ -23,6 +24,7 @@ from types import FrameType
 logger = logging.getLogger("one_memory.loop_watchdog")
 
 DEFAULT_STALL_SECONDS = 2.0
+_SAFE_FALLBACK_FILE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$")
 
 
 class LoopWatchdog:
@@ -120,22 +122,28 @@ def _safe_stack_for_main_thread() -> tuple[str, ...]:
 
 
 def _safe_stacks_for_all_threads() -> tuple[str, ...]:
-    """Return one content-free, bounded stack summary for every live thread."""
-    names = {thread.ident: thread.name for thread in threading.enumerate()}
+    """Return bounded stack summaries without recording thread names."""
     lines: list[str] = []
-    for ident, frame in sorted(sys._current_frames().items()):
+    for _ident, frame in sorted(sys._current_frames().items()):
         stack: list[str] = []
-        # Most specific frame first so the 256-character privacy bound keeps
-        # the likely culprit rather than only the thread bootstrap frames.
+        # Most specific frame first; append only complete frames so truncation
+        # can never turn a valid location into free text.
         for entry in reversed(traceback.extract_stack(frame)[-8:]):
             parts = entry.filename.replace("\\", "/").split("/")
             try:
                 root = parts.index("one_memory")
                 relative = "/".join(parts[root:])
             except ValueError:
-                relative = parts[-1]
+                fallback = parts[-1]
+                relative = fallback if _SAFE_FALLBACK_FILE.fullmatch(fallback) else "unknown.py"
             name = entry.name if entry.name.isidentifier() else "call"
-            stack.append(f"{relative}:{entry.lineno}:{name}")
-        thread_name = names.get(ident, "unknown").replace("\n", " ").replace("\r", " ")
-        lines.append(f"thread={thread_name[:48]}|{' > '.join(stack)}"[:256])
+            candidate = f"{relative}:{entry.lineno}:{name}"
+            separator = "" if not stack else " > "
+            if len("thread|") + len(" > ".join(stack)) + len(separator) + len(candidate) > 256:
+                break
+            stack.append(candidate)
+        if stack:
+            lines.append(f"thread|{' > '.join(stack)}")
+        if len(lines) == 16:
+            break
     return tuple(lines)

memhub/apps/core/src/one_memory/memory_server/migrations.py

@@ -2217,34 +2217,6 @@ CREATE TABLE IF NOT EXISTS retrieval_chunks (
 """
 
 
-def retrieval_documents_v2_ddl(dialect: str = "mysql") -> str:
-    """The v2 documents table under its temporary `_v2` name.
-
-    `index_store.migrate_v2` creates this, copies rows from the v1 table, drops
-    the v1 table and renames. The rename is the only reason a `_v2` name exists
-    at all, and it is derived from the single definition so the copy-across
-    cannot drift from what fresh databases get.
-    """
-    documents = next(
-        statement
-        for statement in _split_statements(retrieval_index_ddl(dialect))
-        if "CREATE TABLE IF NOT EXISTS retrieval_documents (" in statement
-    )
-    return documents.replace(
-        "CREATE TABLE IF NOT EXISTS retrieval_documents (",
-        "CREATE TABLE retrieval_documents_v2 (",
-    )
-
-
-def retrieval_build_maps_ddl(dialect: str = "mysql") -> str:
-    """The build-maps table added by v2."""
-    return next(
-        statement
-        for statement in _split_statements(retrieval_index_ddl(dialect))
-        if "retrieval_build_maps" in statement
-    )
-
-
 async def _033_retrieval_index(cursor: Any) -> None:
     for statement in _split_statements(retrieval_index_ddl(_dialect(cursor))):
         await cursor.execute(statement)

memhub/apps/core/src/one_memory/memory_server/sqlite_semantics.py

@@ -64,7 +64,6 @@ EXEMPT_PATHS = frozenset(
         "memory_server/sqlite_semantics.py",
         "memory_tasks.py",
         "sdk_data_plane_migration.py",
-        "sdk_migration_inventory.py",
         "target_schema.py",
         # D93 A: the visibility cache is SQLite-only by construction. It is a
         # derived table with triggers on the six tables the read check reads —

memhub/apps/core/src/one_memory/retired_authority.py

@@ -22,6 +22,13 @@ class RetiredEvidenceStore:
         del limit
         return 0
 
+    def release_sdk_staging(self, evidence_ids: tuple[str, ...]) -> int:
+        # The write drain releases staged Evidence after every SDK write. On
+        # SDK authority nothing is staged; without this, `__getattr__` raised
+        # past `getattr(..., None)` and every completed write logged an error.
+        del evidence_ids
+        return 0
+
     def get(self, _identifier: str) -> None:
         return None
 

memhub/apps/core/src/one_memory/retrieval/chunking.py

@@ -1,167 +0,0 @@
-"""Single-Evidence identity chunking; no reads, persistence, normalization or grants.
-
-Callers must first establish eligibility/sanitization/permission. This module
-does not strip wrappers or secrets and must not be used as a content safety gate.
-"""
-
-from dataclasses import dataclass
-from typing import Protocol
-
-from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS
-from one_memory.local_embedding import TokenizedText, TokenOffset
-
-
-class OffsetTokenizer(Protocol):
-    def tokenize_with_offsets(self, text: str) -> TokenizedText: ...
-
-
-class ChunkingError(ValueError):
-    """Invalid offsets, uncovered text or insufficient encoding budget; never truncate."""
-
-
-@dataclass(frozen=True)
-class TextChunk:
-    ordinal: int
-    start_cp: int
-    end_cp: int
-    token_count: int
-    text: str
-
-
-@dataclass(frozen=True)
-class _Span:
-    start: int
-    end: int
-    tokens: int
-
-
-def _spans(text: str, encoded: TokenizedText) -> tuple[_Span, ...]:
-    if not isinstance(encoded, TokenizedText) or len(encoded.token_ids) != len(encoded.offsets):
-        raise ChunkingError("chunk_token_offset_count_mismatch")
-    spans: list[_Span] = []
-    previous_start = previous_end = 0
-    for token_id, offset in zip(encoded.token_ids, encoded.offsets, strict=True):
-        if type(token_id) is not int or token_id < 0 or not isinstance(offset, TokenOffset):
-            raise ChunkingError("chunk_invalid_token_offset")
-        if (
-            type(offset.start_cp) is not int
-            or type(offset.end_cp) is not int
-            or type(offset.special) is not bool
-        ):
-            raise ChunkingError("chunk_invalid_token_offset")
-        start, end = offset.start_cp, offset.end_cp
-        if offset.special:
-            if start != 0 or end != 0:
-                raise ChunkingError("chunk_invalid_special_offset")
-            continue
-        if not 0 <= start < end <= len(text) or start < previous_start or end < previous_end:
-            raise ChunkingError("chunk_invalid_offset_order_or_bounds")
-        if start > previous_end and not text[previous_end:start].isspace():
-            raise ChunkingError("chunk_uncovered_nonwhitespace")
-        # Overlapping subpieces (including several pieces for one code point)
-        # form an indivisible interval; slicing inside it loses token identity.
-        if spans and start < spans[-1].end:
-            prior = spans[-1]
-            spans[-1] = _Span(prior.start, end, prior.tokens + 1)
-        else:
-            spans.append(_Span(start, end, 1))
-        previous_start, previous_end = start, end
-    if text[previous_end:] and not text[previous_end:].isspace():
-        raise ChunkingError("chunk_uncovered_nonwhitespace")
-    return tuple(spans)
-
-
-def _encoding(text: str, tokenizer: OffsetTokenizer) -> tuple[TokenizedText, tuple[_Span, ...]]:
-    try:
-        result = tokenizer.tokenize_with_offsets(text)
-    except Exception as error:
-        raise ChunkingError("chunk_tokenization_failed") from error
-    return result, _spans(text, result)
-
-
-def chunk_evidence(
-    text: str,
-    tokenizer: OffsetTokenizer,
-    *,
-    prefix: str = "",
-    target_tokens: int = 384,
-    overlap_tokens: int = 64,
-    max_tokens: int = BUILTIN_EMBEDDING_MAX_TOKENS,
-) -> tuple[TextChunk, ...]:
-    """Chunk one original with code point [start,end), preserving exact slices.
-
-    target/overlap count original non-special tokens. Overlap is up to the
-    requested token count at indivisible offset boundaries; forward progress is
-    mandatory. token_count is re-tokenized prefix + chunk.text INCLUDING specials,
-    never an additive estimate. Prefix is caller-provided, unchanged, and omitted
-    from chunk.text/offsets. Empty/whitespace-only inputs have no index chunks.
-    """
-    if not isinstance(text, str) or not isinstance(prefix, str):
-        raise ChunkingError("chunk_requires_text")
-    if (
-        any(type(value) is not int for value in (target_tokens, overlap_tokens, max_tokens))
-        or not 0 <= overlap_tokens < target_tokens
-        or not 1 <= max_tokens <= BUILTIN_EMBEDDING_MAX_TOKENS
-        or target_tokens > max_tokens
-    ):
-        raise ChunkingError("chunk_invalid_token_budget")
-    if not text or text.isspace():
-        return ()
-    _, spans = _encoding(text, tokenizer)
-    if not spans:
-        raise ChunkingError("chunk_uncovered_nonwhitespace")
-    chunks: list[TextChunk] = []
-    start_index = 0
-    previous_end_cp = 0
-    while start_index < len(spans):
-        end_index = start_index
-        body_tokens = 0
-        while end_index < len(spans):
-            candidate = body_tokens + spans[end_index].tokens
-            if end_index > start_index and candidate > target_tokens:
-                break
-            body_tokens = candidate
-            end_index += 1
-        # Prefer the last paragraph boundary in the latter half of the target.
-        # Never trade away an entire overlap window or split a token interval.
-        if end_index < len(spans):
-            count = 0
-            preferred = None
-            for boundary in range(start_index + 1, end_index + 1):
-                count += spans[boundary - 1].tokens
-                if count >= max(target_tokens // 2, overlap_tokens + 1):
-                    left, right = spans[boundary - 1].end, spans[boundary].start
-                    if "\n" in text[max(spans[boundary - 1].start, left - 1) : right]:
-                        preferred = boundary
-            if preferred is not None:
-                end_index = preferred
-        start_cp = 0 if start_index == 0 else spans[start_index].start
-        while end_index > start_index:
-            # Include gaps verbatim, including whitespace and combining marks;
-            # source coverage is never reconstructed by joining decoded tokens.
-            end_cp = spans[end_index].start if end_index < len(spans) else len(text)
-            encoded, _ = _encoding(prefix + text[start_cp:end_cp], tokenizer)
-            token_count = len(encoded.token_ids)
-            if token_count <= max_tokens:
-                break
-            end_index -= 1
-        else:
-            raise ChunkingError("chunk_indivisible_span_exceeds_token_budget")
-        if end_cp <= previous_end_cp or start_cp > previous_end_cp:
-            raise ChunkingError("chunk_coverage_cannot_progress")
-        chunks.append(TextChunk(len(chunks), start_cp, end_cp, token_count, text[start_cp:end_cp]))
-        previous_end_cp = end_cp
-        if end_index == len(spans):
-            break
-        next_start = end_index
-        overlap = 0
-        while next_start > start_index + 1:
-            candidate = overlap + spans[next_start - 1].tokens
-            if candidate > overlap_tokens:
-                break
-            next_start -= 1
-            overlap = candidate
-        start_index = next_start
-    if previous_end_cp != len(text):
-        raise ChunkingError("chunk_incomplete_coverage")
-    return tuple(chunks)

memhub/apps/core/src/one_memory/retrieval/content.py

@@ -1,92 +0,0 @@
-"""Shared qualification for identity-mapped raw indexing and candidate reads."""
-
-from collections.abc import Sequence
-
-from one_memory.conversation_processing import _is_hy_input_turn
-from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
-from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers
-from one_memory.input_sanitization import sanitize_model_input
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.mapped_content import PreparedMappedContent, prepare_mapped_content
-
-
-def identity_content_ready(record: EvidenceRecord) -> bool:
-    # Reuse pure existing hygiene checks, without invoking HY. Nonidentity
-    # cleaning requires a source map and remains explicitly uncovered for now.
-    return (
-        record.content_kind is EvidenceContentKind.RAW_CONVERSATION
-        and record.role in {"user", "assistant"}
-        and _is_hy_input_turn(record)
-        and not is_host_generated_user_turn(record.verbatim)
-        and strip_host_wrappers(record.verbatim) == record.verbatim.strip()
-        and sanitize_model_input(record.verbatim) == record.verbatim
-    )
-
-
-def index_content(
-    record: EvidenceRecord, *, normalizer_version: int, chunker_version: int
-) -> PreparedMappedContent | None:
-    """Generation-owned content semantics, never silently upgrade identity generations."""
-    if (normalizer_version, chunker_version) == (1, 1):
-        if not identity_content_ready(record):
-            raise ValueError("content_mapping_required")
-        return None
-    if (normalizer_version, chunker_version) == (2, 2):
-        return prepare_mapped_content(record)
-    raise ValueError("unsupported_content_generation")
-
-
-def validate_chunk_coverage(
-    record: EvidenceRecord, chunks: Sequence[TextChunk], prepared: PreparedMappedContent | None
-) -> None:
-    """Verify exact slices and complete citation-safe coverage without a tokenizer call."""
-    if len(chunks) > 8192 or (prepared is None and not chunks):
-        raise ValueError("invalid_chunk_manifest")
-    if prepared is None:
-        ranges = [(0, len(record.verbatim))]
-    else:
-        ranges = []
-        for span in prepared.metadata.segments:
-            if span.kind == "copy":
-                assert span.source_start_cp is not None and span.source_end_cp is not None
-                ranges.append((span.source_start_cp, span.source_end_cp))
-    range_index = 0
-    covered_end = ranges[0][0] if ranges else 0
-    previous_end = -1
-    for index, chunk in enumerate(chunks):
-        if not isinstance(chunk, TextChunk) or any(
-            type(value) is not int
-            for value in (chunk.ordinal, chunk.start_cp, chunk.end_cp, chunk.token_count)
-        ):
-            raise ValueError("invalid_chunk_manifest")
-        while range_index < len(ranges) and chunk.start_cp >= ranges[range_index][1]:
-            if record.verbatim[covered_end : ranges[range_index][1]].strip():
-                raise ValueError("incomplete_chunk_manifest")
-            range_index += 1
-            covered_end = ranges[range_index][0] if range_index < len(ranges) else 0
-        if range_index >= len(ranges):
-            raise ValueError("invalid_chunk_manifest")
-        left, right = ranges[range_index]
-        if (
-            chunk.ordinal != index
-            or not left <= chunk.start_cp < chunk.end_cp <= right
-            or chunk.end_cp <= previous_end
-            or not 1 <= chunk.token_count <= 512
-            or record.verbatim[chunk.start_cp : chunk.end_cp] != chunk.text
-            or (
-                chunk.start_cp > covered_end
-                and (prepared is None or record.verbatim[covered_end : chunk.start_cp].strip())
-            )
-        ):
-            raise ValueError("invalid_chunk_manifest")
-        covered_end = previous_end = chunk.end_cp
-    while range_index < len(ranges):
-        if prepared is None:
-            if covered_end != ranges[range_index][1]:
-                raise ValueError("incomplete_chunk_manifest")
-        elif record.verbatim[covered_end : ranges[range_index][1]].strip():
-            raise ValueError("incomplete_chunk_manifest")
-        range_index += 1
-        covered_end = ranges[range_index][0] if range_index < len(ranges) else 0
-    if prepared is not None and (not chunks) != (prepared.state == "no_indexable_content"):
-        raise ValueError("invalid_chunk_manifest")

memhub/apps/core/src/one_memory/retrieval/index_provisioning.py

@@ -1,96 +0,0 @@
-"""Publish one active generation for the built-in encoder, when asked to.
-
-The runtime deliberately only resumes an index somebody already initialized, so
-without this the feature could be switched on and still never index anything.
-Provisioning stays separate from `start()` for the same reason it exists: it is
-the one place allowed to create, and it creates only for a first enable.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-from pathlib import Path
-
-from one_memory.database_access import main_database
-from one_memory.embedding_assets import assets_present
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder
-from one_memory.retrieval.vector_store import RawVectorStore
-
-DATABASE_NAME = "one-memory.sqlite3"
-VECTOR_DIR_NAME = "retrieval-vectors"
-
-
-def provision_retrieval_index(data_dir: Path) -> str | None:
-    """The active generation id for the built-in encoder, creating it once.
-
-    Returns None whenever the right answer is to leave the data alone: no
-    built-in encoder, no database yet, an index from an older schema, or one
-    already published for a different encoder. None is not an error; the
-    lexical lane works without an index and says so through `degraded`.
-    """
-    database = data_dir / DATABASE_NAME
-    if not assets_present() or not database.is_file():
-        return None
-    identity = LocalRetrievalEncoder().identity
-    store = RetrievalIndexStore(database)
-    try:
-        with main_database(database).write() as db, db:
-            db.execute("BEGIN IMMEDIATE")
-            if (existing := _published(db)) is not None:
-                return existing if _matches(store, db, existing, identity) else None
-            version = _schema_version(db)
-            if version == 1:
-                # Someone's v1 index. Migrating it is maintenance, not startup.
-                return None
-            if version is None:
-                store.initialize_v2(db)
-            generation = store.create_generation(db, identity)
-        store.activate_generation(generation)
-        _ensure_vectors(data_dir / VECTOR_DIR_NAME, generation, identity.dimensions)
-    except Exception:
-        logging.getLogger(__name__).warning("retrieval_index_provision_failed", exc_info=True)
-        return None
-    return generation
-
-
-def _schema_version(db: sqlite3.Connection) -> int | None:
-    if not db.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_index_state'"
-    ).fetchone():
-        return None
-    row = db.execute(
-        "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
-    ).fetchone()
-    return None if row is None else int(row[0])
-
-
-def _published(db: sqlite3.Connection) -> str | None:
-    if not db.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_index_generations'"
-    ).fetchone():
-        return None
-    rows = db.execute(
-        "SELECT generation_id FROM retrieval_index_generations WHERE state='active' LIMIT 2"
-    ).fetchall()
-    return str(rows[0][0]) if len(rows) == 1 else None
-
-
-def _matches(
-    store: RetrievalIndexStore,
-    db: sqlite3.Connection,
-    generation: str,
-    identity: object,
-) -> bool:
-    try:
-        return store._identity(db, generation) == identity
-    except Exception:
-        return False
-
-
-def _ensure_vectors(directory: Path, generation: str, dimensions: int) -> None:
-    if directory.exists():
-        return
-    vectors = RawVectorStore.create(directory, generation_id=generation, dimensions=dimensions)
-    vectors.close()

memhub/apps/core/src/one_memory/retrieval/index_store.py

@@ -1,856 +0,0 @@
-"""Explicit derived-index transactions. No constructor migration or model invocation."""
-
-import hashlib
-import json
-import math
-import sqlite3
-import time
-from collections.abc import Callable, Iterator, Sequence
-from contextlib import contextmanager
-from dataclasses import asdict, dataclass, replace
-from datetime import UTC, datetime
-from pathlib import Path
-from uuid import uuid4
-
-from one_memory import ingest_journal as journal
-from one_memory.context_raw import _check, _connection
-from one_memory.database import take_the_write_lock_first
-from one_memory.database_access import main_database
-from one_memory.imports.models import EvidenceRecord
-from one_memory.imports.storage import EvidenceStore
-from one_memory.memory_server.migrations import (
-    _split_statements,
-    retrieval_build_maps_ddl,
-    retrieval_documents_v2_ddl,
-    retrieval_index_ddl,
-)
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.content import index_content, validate_chunk_coverage
-from one_memory.retrieval.mapped_content import ContentMap, PreparedMappedContent
-from one_memory.retrieval.models import TimeRange
-
-
-def digest(value: object) -> str:
-    return hashlib.sha256(
-        json.dumps(
-            value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False
-        ).encode()
-    ).hexdigest()
-
-
-@dataclass(frozen=True)
-class EncoderIdentity:
-    encoder_id: str
-    encoder_digest: str
-    tokenizer_digest: str
-    dimensions: int
-    chunker_version: int = 1
-    normalizer_version: int = 1
-    query_template_version: int = 1
-
-    def __post_init__(self) -> None:
-        if not self.encoder_id or len(self.encoder_id) > 128:
-            raise ValueError("invalid_encoder")
-        for value in (self.encoder_digest, self.tokenizer_digest):
-            if len(value) != 64 or any(c not in "0123456789abcdef" for c in value):
-                raise ValueError("invalid_encoder_digest")
-        for number in (
-            self.dimensions,
-            self.chunker_version,
-            self.normalizer_version,
-            self.query_template_version,
-        ):
-            if type(number) is not int or not 1 <= number <= 1000000:
-                raise ValueError("invalid_encoder_number")
-
-
-@dataclass(frozen=True)
-class IndexLease:
-    generation_id: str
-    evidence_id: str
-    build_id: str
-    event_seq: int
-    owner: str
-    token: str
-    expires_at: float
-    grant_fingerprint: str
-
-
-@dataclass(frozen=True)
-class PublishedChunk:
-    chunk_id: str
-    build_id: str
-    evidence_id: str
-    revision: int
-    start_cp: int
-    end_cp: int
-    ordinal: int
-    source_id: str
-    event_time: datetime | None
-
-
-def chunk_identity(lease: IndexLease, chunk: TextChunk, map_digest: str | None = None) -> str:
-    values = [lease.build_id, chunk.ordinal, chunk.start_cp, chunk.end_cp, digest(chunk.text)]
-    return digest(values if map_digest is None else [*values, map_digest])
-
-
-@contextmanager
-def transaction(db: sqlite3.Connection) -> Iterator[None]:
-    if not db.in_transaction:
-        raise RuntimeError("retrieval_requires_caller_transaction")
-    name = "retrieval_" + uuid4().hex
-    previous = db.row_factory
-    db.row_factory = sqlite3.Row
-    db.execute(f"SAVEPOINT {name}")
-    try:
-        yield
-    except BaseException:
-        db.execute(f"ROLLBACK TO {name}")
-        db.execute(f"RELEASE {name}")
-        raise
-    else:
-        db.execute(f"RELEASE {name}")
-    finally:
-        db.row_factory = previous
-
-
-class RetrievalIndexStore:
-    def __init__(self, database_path: Path):
-        self.database_path = database_path
-
-    @contextmanager
-    def _db(
-        self, *, write: bool = False, deadline: float | None = None
-    ) -> Iterator[sqlite3.Connection]:
-        if deadline is not None:
-            if write:
-                raise ValueError("deadline_connection_is_read_only")
-            with _connection(self.database_path, deadline) as db:
-                db.execute("PRAGMA query_only=ON")
-                yield db
-            return
-        database = main_database(self.database_path)
-        with (
-            database.write(create=False, foreign_keys=True, row_factory=sqlite3.Row)
-            if write
-            else database.read(foreign_keys=True, row_factory=sqlite3.Row)
-        ) as db:
-            try:
-                if write:
-                    take_the_write_lock_first(db)
-                else:
-                    db.execute("PRAGMA query_only=ON")
-                    db.execute("BEGIN")
-                yield db
-                if write:
-                    db.commit()
-            except BaseException:
-                db.rollback()
-                raise
-
-    def initialize(self, db: sqlite3.Connection) -> None:
-        """Caller owns maintenance/exclusive writer admission; rollback includes baseline."""
-        self._initialize(db, version=1)
-
-    def initialize_v2(self, db: sqlite3.Connection) -> None:
-        """Initialize a fresh store directly at the folded v2 schema.
-
-        Startup provisioning uses this path so a brand-new database does not
-        create a v1 schema and immediately migrate it. Explicit maintenance
-        callers keep `initialize()`'s v1 baseline contract instead.
-        """
-        self._initialize(db, version=2)
-
-    def _initialize(self, db: sqlite3.Connection, *, version: int) -> None:
-        with transaction(db):
-            exists = db.execute(
-                "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
-            ).fetchone()
-            if exists:
-                self._store_id(db)
-            elif db.execute(
-                "SELECT 1 FROM sqlite_master WHERE name IN ('retrieval_documents',"
-                "'retrieval_chunks','retrieval_index_generations','retrieval_consumers')"
-            ).fetchone():
-                raise ValueError("retrieval_partial_initialization")
-            journal.initialize(db)
-            # Migration 33 owns the schema. Explicit initialization keeps the
-            # maintenance-facing v1 baseline so `migrate_v2` remains observable;
-            # startup provisioning may instead ask for the folded v2 directly.
-            for sql in _split_statements(retrieval_index_ddl("sqlite", version=version)):
-                db.execute(sql)
-            row = db.execute(
-                "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
-            ).fetchone()
-            if row is None:
-                db.execute(
-                    "INSERT INTO retrieval_index_state VALUES (1,?,?)",
-                    (version, uuid4().hex),
-                )
-            elif row[0] not in (1, 2):
-                raise ValueError("retrieval_schema_mismatch")
-
-    def migrate_v2(self, db: sqlite3.Connection) -> None:
-        """Explicit maintenance transaction only; readers/worker never invoke this."""
-        with transaction(db):
-            self._store_id(db)
-            version = db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0]
-            if version == 2:
-                db.execute(
-                    "SELECT build_id,source_map_json,map_digest,coverage_state "
-                    "FROM retrieval_build_maps LIMIT 0"
-                )
-                return
-            if (
-                db.execute(
-                    "SELECT 1 FROM sqlite_master WHERE type='table' AND name='retrieval_build_maps'"
-                ).fetchone()
-                is not None
-            ):
-                # A v1 index must not already own a v2-only table. Treat this
-                # as a foreign/partial schema instead of letting `IF NOT
-                # EXISTS` silently accept a table with the wrong columns.
-                raise sqlite3.OperationalError("retrieval_build_maps already exists")
-            # Both tables come from migration 33, named/derived there so this
-            # module holds no DDL text of its own.
-            db.execute(retrieval_documents_v2_ddl("sqlite"))
-            db.execute("INSERT INTO retrieval_documents_v2 SELECT * FROM retrieval_documents")
-            db.execute("DROP TABLE retrieval_documents")
-            db.execute("ALTER TABLE retrieval_documents_v2 RENAME TO retrieval_documents")
-            db.execute(
-                "CREATE INDEX retrieval_jobs ON retrieval_documents(generation_id,state,event_seq)"
-            )
-            db.execute(retrieval_build_maps_ddl("sqlite"))
-            db.execute("UPDATE retrieval_index_state SET schema_version=2 WHERE singleton=1")
-
-    @staticmethod
-    def _store_id(db: sqlite3.Connection) -> str:
-        row = db.execute(
-            "SELECT schema_version,data_store_id FROM retrieval_index_state WHERE singleton=1"
-        ).fetchone()
-        if row is None or row[0] not in (1, 2) or not isinstance(row[1], str) or len(row[1]) != 32:
-            raise ValueError("retrieval_not_initialized")
-        return str(row[1])
-
-    def create_generation(self, db: sqlite3.Connection, identity: EncoderIdentity) -> str:
-        with transaction(db):
-            self._check_content_version(db, identity)
-            key = digest([self._store_id(db), asdict(identity)])
-            encoded = json.dumps(asdict(identity), sort_keys=True)
-            row = db.execute(
-                "SELECT identity_json FROM retrieval_index_generations WHERE generation_id=?",
-                (key,),
-            ).fetchone()
-            if row is not None and row[0] != encoded:
-                raise ValueError("generation_identity_changed")
-            if row is None:
-                db.execute(
-                    "INSERT INTO retrieval_index_generations VALUES (?,?,'building',?)",
-                    (key, encoded, time.time()),
-                )
-                db.execute("INSERT INTO retrieval_consumers VALUES (?,0,0)", (key,))
-            return key
-
-    def consume(self, db: sqlite3.Connection, generation_id: str, *, limit: int = 256) -> int:
-        if type(limit) is not int or not 1 <= limit <= 256:
-            raise ValueError("invalid_consume_limit")
-        with transaction(db):
-            store_id = self._store_id(db)
-            generation = db.execute(
-                "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
-                (generation_id,),
-            ).fetchone()
-            if generation is None or generation[0] == "retired":
-                raise ValueError("generation_not_writable")
-            cursor = db.execute(
-                "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)
-            ).fetchone()
-            if cursor is None:
-                raise ValueError("consumer_missing")
-            rows = db.execute(
-                "SELECT seq FROM evidence_ingest_events WHERE seq>? ORDER BY seq LIMIT ?",
-                (cursor[0], limit),
-            ).fetchall()
-            for row in rows:
-                event = journal.event(db, row[0])
-                if event is None:
-                    raise ValueError("journal_event_missing")
-                if event.user_id != "local":
-                    continue
-                build = digest(
-                    [
-                        store_id,
-                        generation_id,
-                        event.evidence_id,
-                        event.incarnation,
-                        event.evidence_revision,
-                        event.fingerprint,
-                    ]
-                )
-                state = "deleted" if event.kind == "delete" else "pending"
-                db.execute(
-                    """INSERT INTO retrieval_documents
-                    (generation_id,evidence_id,source_id,event_seq,build_id,state)
-                    VALUES (?,?,?,?,?,?) ON CONFLICT(generation_id,evidence_id) DO UPDATE SET
-                    source_id=excluded.source_id,event_seq=excluded.event_seq,build_id=excluded.build_id,
-                    state=excluded.state,active_build_id=NULL,lease_owner=NULL,lease_token=NULL,
-                    lease_until=NULL,grant_fingerprint=NULL
-                    WHERE excluded.event_seq>retrieval_documents.event_seq""",
-                    (generation_id, event.evidence_id, event.source_id, event.seq, build, state),
-                )
-            if rows:
-                db.execute(
-                    "UPDATE retrieval_consumers SET after_seq=? WHERE generation_id=?",
-                    (rows[-1][0], generation_id),
-                )
-            return len(rows)
-
-    def consume_pending(self, generation_id: str, *, limit: int = 256) -> int:
-        with self._db(write=True) as db:
-            return self.consume(db, generation_id, limit=limit)
-
-    def encoder_identity(self, generation_id: str) -> EncoderIdentity:
-        with self._db() as db:
-            return self._identity(db, generation_id)
-
-    @staticmethod
-    def _check_content_version(db: sqlite3.Connection, identity: EncoderIdentity) -> None:
-        versions = (identity.normalizer_version, identity.chunker_version)
-        if versions not in ((1, 1), (2, 2)):
-            raise ValueError("unsupported_content_generation")
-        if versions == (2, 2):
-            row = db.execute(
-                "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
-            ).fetchone()
-            if row is None or row[0] != 2:
-                raise ValueError("retrieval_schema2_required")
-
-    def _identity(self, db: sqlite3.Connection, generation_id: str) -> EncoderIdentity:
-        row = db.execute(
-            "SELECT identity_json FROM retrieval_index_generations WHERE generation_id=?",
-            (generation_id,),
-        ).fetchone()
-        if row is None:
-            raise ValueError("generation_missing")
-        identity = EncoderIdentity(**json.loads(row[0]))
-        if digest([self._store_id(db), asdict(identity)]) != generation_id:
-            raise ValueError("generation_identity_changed")
-        self._check_content_version(db, identity)
-        return identity
-
-    def _prepare(
-        self, db: sqlite3.Connection, generation_id: str, record: EvidenceRecord
-    ) -> PreparedMappedContent | None:
-        identity = self._identity(db, generation_id)
-        return index_content(
-            record,
-            normalizer_version=identity.normalizer_version,
-            chunker_version=identity.chunker_version,
-        )
-
-    @staticmethod
-    def _grant(db: sqlite3.Connection, source_id: str) -> str | None:
-        row = db.execute(
-            """SELECT root_path,candidate_kind,granted_at,root_identity,revision
-            FROM source_grants
-            WHERE user_id='local' AND source_id=? AND revoked_at IS NULL""",
-            (source_id,),
-        ).fetchone()
-        return digest(tuple(row)) if row and row[3] and row[4] > 0 else None
-
-    def claim(
-        self, generation_id: str, owner: str, *, now: float | None = None, lease_seconds: int = 60
-    ) -> IndexLease | None:
-        now = time.time() if now is None else now
-        if not owner or len(owner) > 128 or not math.isfinite(now) or not 1 <= lease_seconds <= 300:
-            raise ValueError("invalid_lease")
-        with self._db(write=True) as db:
-            generation = db.execute(
-                "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
-                (generation_id,),
-            ).fetchone()
-            if generation is None or generation[0] == "retired":
-                raise ValueError("generation_not_writable")
-            rows = db.execute(
-                """SELECT * FROM retrieval_documents WHERE generation_id=? AND
-                (state='pending' OR (state='leased' AND lease_until<=?))
-                ORDER BY event_seq LIMIT 256""",
-                (generation_id, now),
-            ).fetchall()
-            for row in rows:
-                event = journal.event(db, row["event_seq"])
-                grant = self._grant(db, row["source_id"])
-                if event is None or grant is None or not journal.matches_current(db, event):
-                    db.execute(
-                        "UPDATE retrieval_documents SET state='blocked',active_build_id=NULL "
-                        "WHERE generation_id=? AND evidence_id=?",
-                        (generation_id, row["evidence_id"]),
-                    )
-                    continue
-                token = uuid4().hex
-                until = now + lease_seconds
-                db.execute(
-                    """UPDATE retrieval_documents SET state='leased',lease_owner=?,lease_token=?,
-                    lease_until=?,grant_fingerprint=?,attempts=attempts+1
-                    WHERE generation_id=? AND evidence_id=?""",
-                    (owner, token, until, grant, generation_id, row["evidence_id"]),
-                )
-                return IndexLease(
-                    generation_id,
-                    row["evidence_id"],
-                    row["build_id"],
-                    row["event_seq"],
-                    owner,
-                    token,
-                    until,
-                    grant,
-                )
-        return None
-
-    def _validate_lease(
-        self, db: sqlite3.Connection, lease: IndexLease, now: float
-    ) -> journal.IngestEvent:
-        if not math.isfinite(now):
-            raise ValueError("invalid_lease_time")
-        generation = db.execute(
-            "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
-            (lease.generation_id,),
-        ).fetchone()
-        if generation is None or generation[0] == "retired":
-            raise ValueError("lease_generation_retired")
-        row = db.execute(
-            "SELECT * FROM retrieval_documents WHERE generation_id=? AND evidence_id=?",
-            (lease.generation_id, lease.evidence_id),
-        ).fetchone()
-        if (
-            row is None
-            or row["state"] != "leased"
-            or (
-                row["build_id"],
-                row["event_seq"],
-                row["lease_owner"],
-                row["lease_token"],
-                row["lease_until"],
-                row["grant_fingerprint"],
-            )
-            != (
-                lease.build_id,
-                lease.event_seq,
-                lease.owner,
-                lease.token,
-                lease.expires_at,
-                lease.grant_fingerprint,
-            )
-            or lease.expires_at <= now
-        ):
-            raise ValueError("lease_changed_or_expired")
-        event = journal.event(db, lease.event_seq)
-        if (
-            event is None
-            or event.user_id != "local"
-            or event.evidence_id != lease.evidence_id
-            or not journal.matches_current(db, event)
-        ):
-            raise ValueError("lease_evidence_changed")
-        if self._grant(db, event.source_id) != lease.grant_fingerprint:
-            raise ValueError("lease_source_authority_changed")
-        return event
-
-    def read_lease(self, lease: IndexLease, *, now: float | None = None) -> EvidenceRecord:
-        with self._db() as db:
-            self._validate_lease(db, lease, time.time() if now is None else now)
-            record = EvidenceStore._get_with_connection(db, lease.evidence_id)
-            if record is None or len(record.verbatim) > 2_000_000:
-                raise ValueError("lease_input_unavailable_or_too_large")
-            return record
-
-    def renew(
-        self, lease: IndexLease, *, now: float | None = None, lease_seconds: int = 60
-    ) -> IndexLease:
-        now = time.time() if now is None else now
-        if type(lease_seconds) is not int or not 1 <= lease_seconds <= 300:
-            raise ValueError("invalid_lease")
-        with self._db(write=True) as db:
-            self._validate_lease(db, lease, now)
-            expires = max(lease.expires_at, now + lease_seconds)
-            db.execute(
-                "UPDATE retrieval_documents SET lease_until=? "
-                "WHERE generation_id=? AND evidence_id=?",
-                (expires, lease.generation_id, lease.evidence_id),
-            )
-            return replace(lease, expires_at=expires)
-
-    def block(self, lease: IndexLease) -> None:
-        """Release only this claim; preserve uncertain derived vectors for later cleanup."""
-        with self._db(write=True) as db:
-            db.execute(
-                """UPDATE retrieval_documents SET state='blocked',active_build_id=NULL,
-                lease_owner=NULL,lease_token=NULL,lease_until=NULL
-                WHERE generation_id=? AND evidence_id=? AND state='leased'
-                AND build_id=? AND lease_token=? AND lease_owner=?""",
-                (lease.generation_id, lease.evidence_id, lease.build_id, lease.token, lease.owner),
-            )
-
-    def retry_blocked(self, generation_id: str) -> int:
-        """Bounded, durable round-robin retry/regrant pass; never creates grants."""
-        changed = 0
-        with self._db(write=True) as db:
-            generation = db.execute(
-                "SELECT g.state,c.retry_after_seq FROM retrieval_index_generations g "
-                "JOIN retrieval_consumers c USING(generation_id) WHERE generation_id=?",
-                (generation_id,),
-            ).fetchone()
-            if generation is None or generation[0] == "retired":
-                raise ValueError("generation_not_writable")
-            rows = db.execute(
-                "SELECT * FROM retrieval_documents WHERE generation_id=? "
-                "AND state IN ('blocked','ready','excluded') AND event_seq>? "
-                "ORDER BY event_seq LIMIT 256",
-                (generation_id, generation[1]),
-            ).fetchall()
-            for row in rows:
-                event = journal.event(db, row["event_seq"])
-                grant = self._grant(db, row["source_id"])
-                if row["state"] in ("ready", "excluded") and grant == row["grant_fingerprint"]:
-                    continue
-                if (
-                    event
-                    and event.source_id == row["source_id"]
-                    and grant
-                    and journal.matches_current(db, event)
-                ):
-                    db.execute(
-                        "UPDATE retrieval_documents SET state='pending',active_build_id=NULL,"
-                        "lease_owner=NULL,lease_token=NULL,lease_until=NULL,grant_fingerprint=NULL "
-                        "WHERE generation_id=? AND evidence_id=?",
-                        (generation_id, row["evidence_id"]),
-                    )
-                    changed += 1
-            db.execute(
-                "UPDATE retrieval_consumers SET retry_after_seq=? WHERE generation_id=?",
-                (rows[-1]["event_seq"] if len(rows) == 256 else 0, generation_id),
-            )
-        return changed
-
-    def publish(
-        self,
-        lease: IndexLease,
-        chunks: Sequence[TextChunk],
-        *,
-        vectors_verified: bool,
-        now: float | None = None,
-        source_map: ContentMap | None = None,
-        checkpoint: Callable[[], None] | None = None,
-    ) -> tuple[str, ...]:
-        """Caller verifies actual immutable vectors before entering this short transaction."""
-        if vectors_verified is not True:
-            raise ValueError("vectors_not_verified")
-        now = time.time() if now is None else now
-        if not math.isfinite(now):
-            raise ValueError("invalid_lease_time")
-        if len(chunks) > 8192:
-            raise ValueError("invalid_chunk_manifest")
-        with self._db(write=True) as db:
-            if checkpoint is not None:
-                checkpoint()
-            event = self._validate_lease(db, lease, now)
-            record = EvidenceStore._get_with_connection(db, lease.evidence_id)
-            if record is None or len(record.verbatim) > 2_000_000:
-                raise ValueError("lease_evidence_missing")
-            prepared = self._prepare(db, lease.generation_id, record)
-            if source_map != (prepared.metadata if prepared is not None else None):
-                raise ValueError("source_map_mismatch")
-            validate_chunk_coverage(record, chunks, prepared)
-            map_digest = source_map.digest if source_map is not None else None
-            keys = []
-            manifest = []
-            for index, chunk in enumerate(chunks):
-                text_hash = digest(chunk.text)
-                key = chunk_identity(lease, chunk, map_digest)
-                keys.append(key)
-                manifest.append(
-                    (
-                        key,
-                        lease.generation_id,
-                        lease.evidence_id,
-                        lease.build_id,
-                        index,
-                        chunk.start_cp,
-                        chunk.end_cp,
-                        chunk.token_count,
-                        text_hash,
-                    )
-                )
-            existing = db.execute(
-                "SELECT * FROM retrieval_chunks WHERE build_id=? ORDER BY ordinal",
-                (lease.build_id,),
-            ).fetchall()
-            if existing:
-                if [tuple(row) for row in existing] != manifest:
-                    raise ValueError("immutable_chunk_manifest_conflict")
-            else:
-                db.executemany("INSERT INTO retrieval_chunks VALUES (?,?,?,?,?,?,?,?,?)", manifest)
-            state = "ready" if chunks else "excluded"
-            if source_map is not None:
-                values = (
-                    lease.build_id,
-                    source_map.canonical_json(),
-                    map_digest,
-                    "indexable" if chunks else "excluded",
-                )
-                prior = db.execute(
-                    "SELECT * FROM retrieval_build_maps WHERE build_id=?", (lease.build_id,)
-                ).fetchone()
-                if prior is not None and tuple(prior) != values:
-                    raise ValueError("immutable_source_map_conflict")
-                if prior is None:
-                    db.execute("INSERT INTO retrieval_build_maps VALUES (?,?,?,?)", values)
-            if not journal.matches_current(db, event):
-                raise ValueError("lease_evidence_changed")
-            db.execute(
-                "UPDATE retrieval_documents SET state=?,active_build_id=build_id,"
-                "lease_owner=NULL,lease_token=NULL,lease_until=NULL "
-                "WHERE generation_id=? AND evidence_id=?",
-                (state, lease.generation_id, lease.evidence_id),
-            )
-            if checkpoint is not None:
-                checkpoint()
-            return tuple(keys)
-
-    def activate_generation(self, generation_id: str) -> None:
-        with self._db(write=True) as db:
-            row = db.execute(
-                "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
-                (generation_id,),
-            ).fetchone()
-            cursor = db.execute(
-                "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)
-            ).fetchone()
-            if (
-                row is None
-                or row[0] == "retired"
-                or cursor is None
-                or cursor[0] != journal.watermark(db)
-            ):
-                raise ValueError("generation_not_caught_up")
-            for document in db.execute(
-                "SELECT * FROM retrieval_documents WHERE generation_id=? "
-                "AND state NOT IN ('ready','excluded','deleted')",
-                (generation_id,),
-            ):
-                if self._grant(db, document["source_id"]) is not None:
-                    raise ValueError("generation_incomplete")
-            for document in db.execute(
-                "SELECT * FROM retrieval_documents WHERE generation_id=? "
-                "AND state IN ('ready','excluded')",
-                (generation_id,),
-            ):
-                if self._grant(db, document["source_id"]) is not None:
-                    event = journal.event(db, document["event_seq"])
-                    if (
-                        not event
-                        or not journal.matches_current(db, event)
-                        or self._grant(db, document["source_id"]) != document["grant_fingerprint"]
-                    ):
-                        raise ValueError("generation_stale")
-                    record = EvidenceStore._get_with_connection(db, document["evidence_id"])
-                    if record is None:
-                        raise ValueError("generation_stale")
-                    self.validate_document(db, document, record)
-            db.execute(
-                "UPDATE retrieval_index_generations SET state='retired' "
-                "WHERE state='active' AND generation_id!=?",
-                (generation_id,),
-            )
-            db.execute(
-                "UPDATE retrieval_index_generations SET state='active' WHERE generation_id=?",
-                (generation_id,),
-            )
-
-    def active_chunks(
-        self,
-        generation_id: str,
-        *,
-        source_ids: frozenset[str] | None = None,
-        time_range: TimeRange | None = None,
-        deadline: float | None = None,
-        require_active: bool = False,
-    ) -> tuple[PublishedChunk, ...]:
-        """Return current chunks after SQL-level source/time eligibility filtering.
-
-        Grant fingerprints, journal freshness, and manifests are still checked
-        below in the same snapshot. The SQL predicate keeps unauthorized rows
-        from filling an in-memory candidate set before those checks run.
-        """
-        if source_ids is not None and not source_ids:
-            return ()
-        with self._db(deadline=deadline) as db:
-            generation = db.execute(
-                "SELECT state FROM retrieval_index_generations WHERE generation_id=?",
-                (generation_id,),
-            ).fetchone()
-            if generation is None or generation[0] != "active":
-                if require_active:
-                    raise ValueError("active_generation_changed")
-                return ()
-            clauses = ["d.generation_id=?", "d.state IN ('ready','excluded')"]
-            parameters: list[object] = [generation_id]
-            join = ""
-            if source_ids is not None:
-                clauses.append("d.source_id IN (SELECT value FROM json_each(?))")
-                parameters.append(json.dumps(sorted(source_ids)))
-            if time_range is not None:
-                join = "JOIN evidence e ON e.evidence_id=d.evidence_id "
-                clauses.extend(
-                    (
-                        "e.created_at IS NOT NULL",
-                        "julianday(e.created_at) >= julianday(?)",
-                        "julianday(e.created_at) < julianday(?)",
-                    )
-                )
-                parameters.extend((time_range.start.isoformat(), time_range.end.isoformat()))
-            output = []
-            for row in db.execute(
-                "SELECT d.* FROM retrieval_documents d "
-                + join
-                + f"WHERE {' AND '.join(clauses)} ORDER BY d.event_seq,d.evidence_id",
-                parameters,
-            ):
-                if deadline is not None:
-                    _check(deadline)
-                grant = self._grant(db, row["source_id"])
-                if grant is None or grant != row["grant_fingerprint"]:
-                    continue
-                event = journal.event(db, row["event_seq"])
-                if (
-                    event is None
-                    or event.source_id != row["source_id"]
-                    or not journal.matches_current(db, event)
-                ):
-                    continue
-                record = EvidenceStore._get_with_connection(db, row["evidence_id"])
-                if (
-                    record is None
-                    or record.user_id != "local"
-                    or record.source_id != row["source_id"]
-                ):
-                    continue
-                event_time = record.created_at
-                if event_time is not None:
-                    event_time = event_time.astimezone(UTC) if event_time.tzinfo else None
-                if time_range is not None and (
-                    event_time is None or not time_range.start <= event_time < time_range.end
-                ):
-                    continue
-                chunks = self.validate_document(db, row, record, deadline=deadline)
-                for chunk in chunks:
-                    if deadline is not None:
-                        _check(deadline)
-                    output.append(
-                        PublishedChunk(
-                            chunk["chunk_id"],
-                            row["build_id"],
-                            row["evidence_id"],
-                            event.evidence_revision,
-                            chunk["start_cp"],
-                            chunk["end_cp"],
-                            chunk["ordinal"],
-                            record.source_id,
-                            event_time,
-                        )
-                    )
-            return tuple(output)
-
-    def validate_document(
-        self,
-        db: sqlite3.Connection,
-        document: sqlite3.Row,
-        record: EvidenceRecord,
-        *,
-        deadline: float | None = None,
-    ) -> tuple[sqlite3.Row, ...]:
-        """Same-snapshot complete manifest/map check shared by candidate and final read."""
-        if deadline is not None:
-            _check(deadline)
-        if document["active_build_id"] != document["build_id"]:
-            raise ValueError("published_build_binding_invalid")
-        prepared = self._prepare(db, document["generation_id"], record)
-        map_digest = None
-        if prepared is not None:
-            source_map = db.execute(
-                "SELECT source_map_json,map_digest,coverage_state FROM retrieval_build_maps "
-                "WHERE build_id=?",
-                (document["build_id"],),
-            ).fetchone()
-            expected_state = "excluded" if prepared.state == "no_indexable_content" else "ready"
-            if (
-                source_map is None
-                or tuple(source_map)
-                != (
-                    prepared.metadata.canonical_json(),
-                    prepared.metadata.digest,
-                    "excluded" if expected_state == "excluded" else "indexable",
-                )
-                or document["state"] != expected_state
-            ):
-                raise ValueError("published_source_map_invalid")
-            map_digest = prepared.metadata.digest
-        elif document["state"] != "ready":
-            raise ValueError("published_manifest_invalid")
-        rows = tuple(
-            db.execute(
-                "SELECT * FROM retrieval_chunks WHERE generation_id=? AND evidence_id=? "
-                "AND build_id=? ORDER BY ordinal LIMIT 8193",
-                (document["generation_id"], record.evidence_id, document["build_id"]),
-            ).fetchall()
-        )
-        chunks = []
-        for row in rows:
-            if deadline is not None:
-                _check(deadline)
-            start, end = row["start_cp"], row["end_cp"]
-            if (
-                type(start) is not int
-                or type(end) is not int
-                or not 0 <= start < end <= len(record.verbatim)
-            ):
-                raise ValueError("published_manifest_invalid")
-            text = record.verbatim[start:end]
-            text_hash = digest(text)
-            identity = [document["build_id"], row["ordinal"], start, end, text_hash]
-            if map_digest is not None:
-                identity.append(map_digest)
-            if row["text_hash"] != text_hash or row["chunk_id"] != digest(identity):
-                raise ValueError("published_manifest_invalid")
-            chunks.append(TextChunk(row["ordinal"], start, end, row["token_count"], text))
-        try:
-            validate_chunk_coverage(record, chunks, prepared)
-        except ValueError as error:
-            raise ValueError("published_manifest_incomplete_or_invalid") from error
-        if deadline is not None:
-            _check(deadline)
-        return rows
-
-    def status(self, generation_id: str) -> dict[str, int]:
-        with self._db() as db:
-            row = db.execute(
-                "SELECT after_seq FROM retrieval_consumers WHERE generation_id=?", (generation_id,)
-            ).fetchone()
-            if row is None:
-                raise ValueError("generation_missing")
-            result = {
-                "after_seq": int(row[0]),
-                "pending": 0,
-                "leased": 0,
-                "ready": 0,
-                "blocked": 0,
-                "deleted": 0,
-                "excluded": 0,
-            }
-            for state, count in db.execute(
-                "SELECT state,count(*) FROM retrieval_documents "
-                "WHERE generation_id=? GROUP BY state",
-                (generation_id,),
-            ):
-                result[state] = count
-            return result

memhub/apps/core/src/one_memory/retrieval/index_supervisor.py

@@ -1,200 +0,0 @@
-"""Explicit ownership of one admitted index worker; no migration or resource close."""
-
-import asyncio
-import math
-import threading
-import time
-from collections.abc import Callable
-from dataclasses import dataclass
-from typing import Literal, Protocol
-
-from one_memory.retrieval.index_worker import IndexWorkResult
-from one_memory.service_levels import ServiceLevel, service_level_context
-
-
-class IndexWorker(Protocol):
-    def run_once(self, *, cancel: threading.Event | None = None) -> IndexWorkResult: ...
-
-
-@dataclass(frozen=True)
-class IndexSupervisorSnapshot:
-    state: Literal["new", "running", "stopping", "stopped"]
-    last_result: str | None
-    error_code: str | None
-    slices: int
-    failures: int
-    thread_alive: bool
-
-
-class RetrievalIndexSupervisor:
-    """Single-use supervisor. Caller owns admission and closes vectors only after stop.
-
-    Cancellation is cooperative: stop/wait cannot finish until a blocking native
-    run_once returns. The lifecycle owner must drain before shutting its event loop.
-    """
-
-    def __init__(
-        self,
-        worker: IndexWorker,
-        *,
-        idle_wait_seconds: float = 5,
-        failure_wait_seconds: float = 5,
-        yield_wait_seconds: float = 0.05,
-        max_consecutive_slices: int = 8,
-        background_should_yield: Callable[[], bool] | None = None,
-    ):
-        for interval in (idle_wait_seconds, failure_wait_seconds, yield_wait_seconds):
-            if (
-                isinstance(interval, bool)
-                or not isinstance(interval, (int, float))
-                or not math.isfinite(interval)
-                or not 0.001 <= interval <= 300
-            ):
-                raise ValueError("index_supervisor_invalid_config")
-        if type(max_consecutive_slices) is not int or not 1 <= max_consecutive_slices <= 256:
-            raise ValueError("index_supervisor_invalid_config")
-        self._worker = worker
-        self._idle_wait = idle_wait_seconds
-        self._failure_wait = failure_wait_seconds
-        self._yield_wait = yield_wait_seconds
-        self._max_slices = max_consecutive_slices
-        self._background_should_yield = background_should_yield or (lambda: False)
-        self._condition = threading.Condition()
-        self._cancel = threading.Event()
-        self._wake = False
-        self._thread: threading.Thread | None = None
-        self._state: Literal["new", "running", "stopping", "stopped"] = "new"
-        self._last_result: str | None = None
-        self._error_code: str | None = None
-        self._slices = self._failures = 0
-
-    def start(self) -> None:
-        with self._condition:
-            if self._state != "new":
-                raise RuntimeError("index_supervisor_already_started")
-            thread = threading.Thread(target=self._run, name="retrieval-index", daemon=False)
-            self._thread = thread
-            self._state = "running"
-            try:
-                thread.start()
-            except Exception:
-                self._thread = None
-                self._state = "stopped"
-                self._error_code = "index_supervisor_start_failed"
-                raise RuntimeError("index_supervisor_start_failed") from None
-
-    def trigger(self) -> bool:
-        with self._condition:
-            if self._state != "running":
-                return False
-            self._wake = True
-            self._condition.notify_all()
-            return True
-
-    def snapshot(self) -> IndexSupervisorSnapshot:
-        with self._condition:
-            return IndexSupervisorSnapshot(
-                self._state,
-                self._last_result,
-                self._error_code,
-                self._slices,
-                self._failures,
-                self._thread is not None and self._thread.is_alive(),
-            )
-
-    def _request_stop(self) -> None:
-        with self._condition:
-            self._cancel.set()
-            self._wake = False
-            if self._state == "new":
-                self._state = "stopped"
-            elif self._state == "running":
-                self._state = "stopping"
-            self._condition.notify_all()
-
-    async def stop(self) -> None:
-        self._request_stop()
-        await self.wait()
-
-    async def wait(self) -> None:
-        """Cancellation requests shutdown, drains the real thread, then propagates."""
-        cancelled = False
-        while True:
-            with self._condition:
-                thread = self._thread
-            if thread is None or not thread.is_alive():
-                if thread is not None:
-                    thread.join()
-                break
-            try:
-                # No executor waiter or to_thread future to orphan on cancellation.
-                await asyncio.sleep(0.01)
-            except asyncio.CancelledError:
-                cancelled = True
-                self._request_stop()
-        if cancelled:
-            raise asyncio.CancelledError
-
-    def _pause(self, seconds: float, *, wakeable: bool) -> None:
-        deadline = time.monotonic() + seconds
-        with self._condition:
-            while not self._cancel.is_set():
-                if wakeable and self._wake:
-                    break
-                remaining = deadline - time.monotonic()
-                if remaining <= 0:
-                    break
-                self._condition.wait(remaining)
-            self._wake = False
-
-    def _run(self) -> None:
-        consecutive = 0
-        try:
-            while not self._cancel.is_set():
-                if self._background_should_yield():
-                    self._pause(self._yield_wait, wakeable=False)
-                    continue
-                result: str | None = None
-                error: str | None = None
-                try:
-                    with service_level_context(ServiceLevel.BACKGROUND):
-                        value = self._worker.run_once(cancel=self._cancel)
-                    if not isinstance(value, IndexWorkResult) or value.state not in {
-                        "idle",
-                        "ready",
-                        "excluded",
-                        "blocked",
-                        "cancelled",
-                        "deferred",
-                    }:
-                        error = "index_worker_invalid_result"
-                    else:
-                        result = value.state
-                        if result == "blocked":
-                            error = "index_worker_blocked"
-                except BaseException:
-                    # Worker/provider exceptions must never escape via thread stderr.
-                    error = "index_worker_failed"
-                with self._condition:
-                    self._last_result, self._error_code = result, error
-                    self._slices += 1
-                    if error is not None:
-                        self._failures += 1
-                if self._cancel.is_set():
-                    break
-                if error is not None:
-                    consecutive = 0
-                    self._pause(self._failure_wait, wakeable=False)
-                elif result in {"ready", "excluded"}:
-                    consecutive += 1
-                    if consecutive >= self._max_slices:
-                        consecutive = 0
-                        self._pause(self._yield_wait, wakeable=False)
-                else:
-                    consecutive = 0
-                    self._pause(self._idle_wait, wakeable=True)
-        finally:
-            with self._condition:
-                self._state = "stopped"
-                self._wake = False
-                self._condition.notify_all()

memhub/apps/core/src/one_memory/retrieval/index_worker.py

@@ -1,262 +0,0 @@
-"""Bounded raw-index work. No migration, scheduler, HY or remote model invocation.
-
-Runtime must admit a single worker, own its thread/process, and drain that owner
-before closing the vector client. Cancellation here is cooperative between
-native calls, not a claim that ONNX/Chroma can be preempted in a Python thread.
-"""
-
-import math
-import threading
-import time
-from collections.abc import Callable, Sequence
-from dataclasses import dataclass
-from typing import Literal, Protocol
-
-from one_memory.embedding_assets import (
-    BUILTIN_EMBEDDING_MODEL,
-    EMBEDDING_ASSETS,
-    verify_assets,
-)
-from one_memory.local_embedding import LocalEmbedder, TokenizedText
-from one_memory.retrieval.chunking import ChunkingError, chunk_evidence
-from one_memory.retrieval.content import index_content
-from one_memory.retrieval.index_store import (
-    EncoderIdentity,
-    IndexLease,
-    RetrievalIndexStore,
-    chunk_identity,
-)
-from one_memory.retrieval.mapped_content import MappedContentError, chunk_mapped_content
-from one_memory.retrieval.vector_store import RawVectorRecord, RawVectorStore, VectorStoreError
-
-
-class RetrievalEncoder(Protocol):
-    @property
-    def identity(self) -> EncoderIdentity: ...
-
-    def tokenize_with_offsets(self, text: str) -> TokenizedText: ...
-
-    def encode_documents(self, texts: list[str]) -> list[list[float]]: ...
-
-
-class LocalRetrievalEncoder:
-    """Pinned local assets; mapped content v2 by default, explicit v1 for old generations."""
-
-    def __init__(self, embedder: LocalEmbedder | None = None, *, content_version: int = 2):
-        if type(content_version) is not int or content_version not in (1, 2):
-            raise ValueError("unsupported_content_generation")
-        self._embedder = embedder if embedder is not None else LocalEmbedder()
-        digests = {asset.filename: asset.sha256 for asset in EMBEDDING_ASSETS}
-        self.identity = EncoderIdentity(
-            BUILTIN_EMBEDDING_MODEL,
-            digests["model_int8.onnx"],
-            digests["tokenizer.json"],
-            self._embedder.dimensions,
-            chunker_version=content_version,
-            normalizer_version=content_version,
-        )
-        self._verified = False
-        self._lock = threading.Lock()
-
-    def _verify(self) -> None:
-        with self._lock:
-            if not self._verified:
-                verify_assets()
-                self._verified = True
-
-    def tokenize_with_offsets(self, text: str) -> TokenizedText:
-        self._verify()
-        return self._embedder.tokenize_with_offsets(text)
-
-    def encode_documents(self, texts: list[str]) -> list[list[float]]:
-        self._verify()
-        if len(texts) > 8 or any(
-            not 1 <= len(self.tokenize_with_offsets(text).token_ids) <= 512 for text in texts
-        ):
-            raise ValueError("encoder_batch_or_token_budget")
-        with self._lock:
-            return self._embedder.encode(texts)
-
-
-@dataclass(frozen=True)
-class IndexWorkResult:
-    state: Literal["idle", "ready", "excluded", "blocked", "cancelled", "deferred"]
-    reason: str | None = None
-    chunk_count: int = 0
-
-
-class _Interrupted(Exception):
-    def __init__(self, state: Literal["cancelled", "deferred"]):
-        self.state = state
-
-
-class RetrievalIndexWorker:
-    def __init__(
-        self,
-        store: RetrievalIndexStore,
-        vectors: RawVectorStore,
-        encoder: RetrievalEncoder,
-        *,
-        generation_id: str,
-        owner: str,
-        clock: Callable[[], float] = time.monotonic,
-        wall_clock: Callable[[], float] = time.time,
-        max_job_seconds: float = 120,
-    ):
-        if (
-            not owner
-            or len(owner) > 128
-            or isinstance(max_job_seconds, bool)
-            or not math.isfinite(max_job_seconds)
-            or not 0 < max_job_seconds <= 300
-        ):
-            raise ValueError("invalid_worker_budget")
-        if store.encoder_identity(generation_id) != encoder.identity:
-            raise ValueError("worker_encoder_identity_mismatch")
-        if (
-            vectors.generation_id != generation_id
-            or vectors.dimensions != encoder.identity.dimensions
-        ):
-            raise ValueError("worker_vector_identity_mismatch")
-        self.store, self.vectors, self.encoder = store, vectors, encoder
-        self.generation_id, self.owner = generation_id, owner
-        self.clock, self.wall_clock = clock, wall_clock
-        self.max_job_seconds = max_job_seconds
-        self._run_lock = threading.Lock()
-
-    def run_once(self, *, cancel: threading.Event | None = None) -> IndexWorkResult:
-        """Consume <=256 events and handle <=1 document; never activate a generation.
-
-        Interrupted jobs retain their finite lease for restart recovery. Failed
-        jobs are blocked for explicit retry; arbitrary provider exceptions and
-        original text never enter the public result or a log.
-        """
-        if not self._run_lock.acquire(blocking=False):
-            raise RuntimeError("index_worker_already_running")
-        lease: IndexLease | None = None
-        deadline = self.clock() + self.max_job_seconds
-
-        def check() -> None:
-            if cancel is not None and cancel.is_set():
-                raise _Interrupted("cancelled")
-            now = self.clock()
-            if not math.isfinite(now) or now >= deadline:
-                raise _Interrupted("deferred")
-
-        def boundary() -> None:
-            nonlocal lease
-            check()
-            assert lease is not None
-            # This revalidates the original's current fingerprint and authority,
-            # not just a heartbeat on an otherwise stale claim.
-            lease = self.store.renew(lease, now=self.wall_clock())
-            check()
-
-        def publish_checkpoint() -> None:
-            check()
-            assert lease is not None
-            now = self.wall_clock()
-            if not math.isfinite(now) or now >= lease.expires_at:
-                raise ValueError("lease_changed_or_expired")
-
-        try:
-            check()
-            self.store.consume_pending(self.generation_id)
-            check()
-            lease = self.store.claim(self.generation_id, self.owner, now=self.wall_clock())
-            if lease is None:
-                return IndexWorkResult("idle")
-            record = self.store.read_lease(lease, now=self.wall_clock())
-            check()
-            identity = self.encoder.identity
-            try:
-                prepared = index_content(
-                    record,
-                    normalizer_version=identity.normalizer_version,
-                    chunker_version=identity.chunker_version,
-                )
-            except ValueError:
-                self.store.block(lease)
-                return IndexWorkResult("blocked", "content_mapping_required")
-
-            worker_encoder = self.encoder
-
-            class CheckedTokenizer:
-                def tokenize_with_offsets(self, text: str) -> TokenizedText:
-                    check()
-                    result = worker_encoder.tokenize_with_offsets(text)
-                    check()
-                    return result
-
-            chunks = (
-                chunk_mapped_content(prepared, CheckedTokenizer())
-                if prepared is not None
-                else chunk_evidence(record.verbatim, CheckedTokenizer())
-            )
-            if (not chunks and prepared is None) or len(chunks) > 8192:
-                raise ChunkingError("invalid_chunk_count")
-            source_map = prepared.metadata if prepared is not None else None
-            map_digest = source_map.digest if source_map is not None else None
-            boundary()
-            expected: list[RawVectorRecord] = []
-            for start in range(0, len(chunks), 8):
-                batch = chunks[start : start + 8]
-                boundary()
-                encoded = self.encoder.encode_documents([chunk.text for chunk in batch])
-                boundary()
-                if len(encoded) != len(batch):
-                    raise ValueError("encoder_count_mismatch")
-                records = [
-                    RawVectorRecord(
-                        chunk_identity(lease, chunk, map_digest),
-                        lease.build_id,
-                        lease.generation_id,
-                        tuple(vector),
-                    )
-                    for chunk, vector in zip(batch, encoded, strict=True)
-                ]
-                self.vectors.upsert(records)
-                boundary()
-                expected.extend(records)
-            self._verify(expected, boundary)
-            assert lease is not None
-            check()
-            self.store.publish(
-                lease,
-                chunks,
-                vectors_verified=True,
-                now=self.wall_clock(),
-                source_map=source_map,
-                checkpoint=publish_checkpoint,
-            )
-            return IndexWorkResult("ready" if chunks else "excluded", chunk_count=len(chunks))
-        except _Interrupted as error:
-            return IndexWorkResult(
-                error.state, "cancelled" if error.state == "cancelled" else "deadline"
-            )
-        except Exception as error:
-            cause: BaseException | None = error
-            while cause is not None:
-                if isinstance(cause, _Interrupted):
-                    return IndexWorkResult(
-                        cause.state, "cancelled" if cause.state == "cancelled" else "deadline"
-                    )
-                cause = cause.__cause__
-            if lease is None:
-                raise
-            self.store.block(lease)
-            reason = (
-                "chunking_unavailable"
-                if isinstance(error, (ChunkingError, MappedContentError))
-                else "vector_unavailable"
-                if isinstance(error, VectorStoreError)
-                else "index_build_unavailable"
-            )
-            return IndexWorkResult("blocked", reason)
-        finally:
-            self._run_lock.release()
-
-    def _verify(self, records: Sequence[RawVectorRecord], boundary: Callable[[], None]) -> None:
-        for start in range(0, len(records), 8):
-            self.vectors.verify(records[start : start + 8])
-            boundary()

memhub/apps/core/src/one_memory/retrieval/mapped_content.py

@@ -1,349 +0,0 @@
-"""Pure raw-content admission and exact source maps; no grants, writes or HY calls."""
-
-import hashlib
-import json
-from dataclasses import asdict, dataclass, field
-from typing import Literal
-
-from one_memory.conversation_processing import _is_hy_input_turn
-from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS
-from one_memory.imports.models import (
-    MAX_VERBATIM_CHARACTERS,
-    EvidenceContentKind,
-    EvidenceRecord,
-)
-from one_memory.imports.sources.parsing import (
-    is_host_generated_user_turn,
-    unwrap_host_envelope_with_positions,
-)
-from one_memory.input_sanitization import sanitize_model_input_with_spans
-from one_memory.local_embedding import TokenizedText
-from one_memory.retrieval.chunking import ChunkingError, OffsetTokenizer, TextChunk, chunk_evidence
-
-MAP_VERSION = "raw-content-map-v1"
-PARSER_VERSION = "host-envelope-positions-v1"
-SANITIZER_VERSION = "model-input-spans-v1"
-MAX_MAPPED_CHUNKS = 8192
-
-
-class MappedContentError(ValueError):
-    """Fixed reason only; never include original or sanitized content."""
-
-
-@dataclass(frozen=True)
-class MappedSegment:
-    kind: Literal["copy", "synthetic"]
-    start_cp: int
-    end_cp: int
-    source_start_cp: int | None
-    source_end_cp: int | None
-
-
-@dataclass(frozen=True)
-class ExcludedRange:
-    start_cp: int
-    end_cp: int
-    reason: Literal["envelope_or_trim", "sanitizer"]
-
-
-@dataclass(frozen=True)
-class ContentMap:
-    map_version: str
-    parser_version: str
-    sanitizer_version: str
-    source_length_cp: int
-    body_start_cp: int | None
-    body_end_cp: int | None
-    sanitized_length_cp: int
-    sanitized_sha256: str
-    segments: tuple[MappedSegment, ...]
-    excluded: tuple[ExcludedRange, ...]
-
-    def canonical_json(self) -> str:
-        return json.dumps(asdict(self), sort_keys=True, separators=(",", ":"), allow_nan=False)
-
-    @property
-    def digest(self) -> str:
-        return hashlib.sha256(self.canonical_json().encode()).hexdigest()
-
-
-@dataclass(frozen=True)
-class PreparedMappedContent:
-    actual_text: str = field(repr=False)
-    metadata: ContentMap
-    state: Literal["ready", "no_indexable_content"]
-
-
-def _validate(content: PreparedMappedContent, original: str | None = None) -> None:
-    meta, text = content.metadata, content.actual_text
-    invalid = "mapped_invalid_source_map"
-    if (
-        (meta.map_version, meta.parser_version, meta.sanitizer_version)
-        != (MAP_VERSION, PARSER_VERSION, SANITIZER_VERSION)
-        or type(meta.source_length_cp) is not int
-        or not 0 <= meta.source_length_cp <= MAX_VERBATIM_CHARACTERS
-        or type(meta.sanitized_length_cp) is not int
-        or meta.sanitized_length_cp != len(text)
-        or meta.sanitized_sha256 != hashlib.sha256(text.encode()).hexdigest()
-        or type(meta.segments) is not tuple
-        or type(meta.excluded) is not tuple
-    ):
-        raise MappedContentError(invalid)
-    body_start, body_end = meta.body_start_cp, meta.body_end_cp
-    if body_start is None or body_end is None:
-        if body_start is not None or body_end is not None or text or meta.segments:
-            raise MappedContentError(invalid)
-    elif (
-        type(body_start) is not int
-        or type(body_end) is not int
-        or not 0 <= body_start < body_end <= meta.source_length_cp
-    ):
-        raise MappedContentError(invalid)
-    cursor = source_cursor = 0
-    has_content = False
-    partition: list[tuple[int, int]] = []
-    for span in meta.segments:
-        if (
-            not isinstance(span, MappedSegment)
-            or type(span.start_cp) is not int
-            or type(span.end_cp) is not int
-            or not cursor == span.start_cp < span.end_cp <= len(text)
-        ):
-            raise MappedContentError(invalid)
-        cursor = span.end_cp
-        if span.kind == "synthetic":
-            if span.source_start_cp is not None or span.source_end_cp is not None:
-                raise MappedContentError(invalid)
-            continue
-        left, right = span.source_start_cp, span.source_end_cp
-        if (
-            span.kind != "copy"
-            or type(left) is not int
-            or type(right) is not int
-            or body_start is None
-            or body_end is None
-            or not body_start <= left < right <= body_end
-            or left < source_cursor
-            or right - left != span.end_cp - span.start_cp
-        ):
-            raise MappedContentError(invalid)
-        if original is not None and text[span.start_cp : span.end_cp] != original[left:right]:
-            raise MappedContentError(invalid)
-        has_content |= bool(text[span.start_cp : span.end_cp].strip())
-        source_cursor = right
-        partition.append((left, right))
-    if cursor != len(text):
-        raise MappedContentError(invalid)
-    excluded_end = 0
-    for excluded in meta.excluded:
-        if (
-            not isinstance(excluded, ExcludedRange)
-            or type(excluded.start_cp) is not int
-            or type(excluded.end_cp) is not int
-            or not excluded_end <= excluded.start_cp < excluded.end_cp <= meta.source_length_cp
-            or excluded.reason not in {"envelope_or_trim", "sanitizer"}
-        ):
-            raise MappedContentError(invalid)
-        excluded_end = excluded.end_cp
-        partition.append((excluded.start_cp, excluded.end_cp))
-    source_cursor = 0
-    for left, right in sorted(partition):
-        if left != source_cursor:
-            raise MappedContentError(invalid)
-        source_cursor = right
-    if source_cursor != meta.source_length_cp or content.state != (
-        "ready" if has_content else "no_indexable_content"
-    ):
-        raise MappedContentError(invalid)
-
-
-def prepare_mapped_content(record: EvidenceRecord) -> PreparedMappedContent:
-    """Map a qualified body to Evidence.verbatim code points, without modifying it.
-
-    The returned text is transient; only metadata is suitable for an outer manifest.
-    That manifest must independently bind Evidence identity/revision/hash and grants.
-    """
-    if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or record.role not in {
-        "user",
-        "assistant",
-    }:
-        raise MappedContentError("mapped_content_ineligible")
-    original = record.verbatim
-    if len(original) > MAX_VERBATIM_CHARACTERS:
-        raise MappedContentError("mapped_source_limit_exceeded")
-    envelope = unwrap_host_envelope_with_positions(original)
-    if envelope.status == "ambiguous":
-        raise MappedContentError("mapped_ambiguous_envelope")
-    body, start, end = envelope.text, envelope.start_cp, envelope.end_cp
-    segments: list[MappedSegment] = []
-    excluded: list[ExcludedRange] = []
-    text = ""
-    if body is None:
-        if start is not None or end is not None:
-            raise MappedContentError("mapped_invalid_envelope")
-        if original:
-            excluded.append(ExcludedRange(0, len(original), "envelope_or_trim"))
-    else:
-        if (
-            type(start) is not int
-            or type(end) is not int
-            or not 0 <= start < end <= len(original)
-            or original[start:end] != body
-        ):
-            raise MappedContentError("mapped_invalid_envelope")
-        if is_host_generated_user_turn(original) or is_host_generated_user_turn(body):
-            raise MappedContentError("mapped_generated_content")
-        if not _is_hy_input_turn(record.model_copy(update={"verbatim": body})):
-            raise MappedContentError("mapped_body_hygiene_rejected")
-        sanitized = sanitize_model_input_with_spans(body)
-        text = sanitized.text
-        cursor = 0
-        for span in sanitized.spans:
-            if span.start_cp > cursor:
-                segments.append(MappedSegment("synthetic", cursor, span.start_cp, None, None))
-            copied = MappedSegment(
-                "copy",
-                span.start_cp,
-                span.end_cp,
-                start + span.source_start_cp,
-                start + span.source_end_cp,
-            )
-            # Only source AND output adjacency can make one citation-safe interval.
-            if (
-                segments
-                and segments[-1].kind == "copy"
-                and (
-                    segments[-1].end_cp == copied.start_cp
-                    and segments[-1].source_end_cp == copied.source_start_cp
-                )
-            ):
-                previous = segments.pop()
-                copied = MappedSegment(
-                    "copy",
-                    previous.start_cp,
-                    copied.end_cp,
-                    previous.source_start_cp,
-                    copied.source_end_cp,
-                )
-            segments.append(copied)
-            cursor = span.end_cp
-        if cursor < len(text):
-            segments.append(MappedSegment("synthetic", cursor, len(text), None, None))
-        if start:
-            excluded.append(ExcludedRange(0, start, "envelope_or_trim"))
-        excluded.extend(
-            ExcludedRange(start + left, start + right, "sanitizer")
-            for left, right in sanitized.excluded
-        )
-        if end < len(original):
-            excluded.append(ExcludedRange(end, len(original), "envelope_or_trim"))
-    metadata = ContentMap(
-        MAP_VERSION,
-        PARSER_VERSION,
-        SANITIZER_VERSION,
-        len(original),
-        start,
-        end,
-        len(text),
-        hashlib.sha256(text.encode()).hexdigest(),
-        tuple(segments),
-        tuple(excluded),
-    )
-    prepared = PreparedMappedContent(
-        text,
-        metadata,
-        "ready"
-        if any(
-            span.kind == "copy" and text[span.start_cp : span.end_cp].strip() for span in segments
-        )
-        else "no_indexable_content",
-    )
-    _validate(prepared, original)
-    return prepared
-
-
-class _BoundedTokenizer:
-    """Count successful chunk re-encodings without copying the chunking algorithm."""
-
-    def __init__(self, tokenizer: OffsetTokenizer, remaining: int, max_tokens: int):
-        self.tokenizer = tokenizer
-        self.remaining = remaining
-        self.max_tokens = max_tokens
-        self.initial = True
-
-    def tokenize_with_offsets(self, text: str) -> TokenizedText:
-        if not self.initial and self.remaining == 0:
-            raise MappedContentError("mapped_chunk_limit_exceeded")
-        result = self.tokenizer.tokenize_with_offsets(text)
-        # chunk_evidence first encodes the whole copy span, then prefix+candidate;
-        # an in-budget candidate emits exactly one chunk. Shrink retries do not.
-        if (
-            not self.initial
-            and isinstance(result, TokenizedText)
-            and len(result.token_ids) <= self.max_tokens
-        ):
-            self.remaining -= 1
-        self.initial = False
-        return result
-
-
-def chunk_mapped_content(
-    content: PreparedMappedContent,
-    tokenizer: OffsetTokenizer,
-    *,
-    prefix: str = "",
-    target_tokens: int = 384,
-    overlap_tokens: int = 64,
-    max_tokens: int = BUILTIN_EMBEDDING_MAX_TOKENS,
-    max_chunks: int = MAX_MAPPED_CHUNKS,
-) -> tuple[TextChunk, ...]:
-    """Chunk each continuous copy independently; never encode synthetic placeholders."""
-    if type(max_chunks) is not int or not 1 <= max_chunks <= MAX_MAPPED_CHUNKS:
-        raise MappedContentError("mapped_invalid_chunk_limit")
-    _validate(content)
-    # Reuse budget validation even for an explicitly nonindexable record.
-    chunk_evidence(
-        "",
-        tokenizer,
-        prefix=prefix,
-        target_tokens=target_tokens,
-        overlap_tokens=overlap_tokens,
-        max_tokens=max_tokens,
-    )
-    chunks: list[TextChunk] = []
-    for span in content.metadata.segments:
-        if span.kind != "copy":
-            continue
-        text = content.actual_text[span.start_cp : span.end_cp]
-        if not text.strip():
-            continue
-        if len(chunks) == max_chunks:
-            raise MappedContentError("mapped_chunk_limit_exceeded")
-        try:
-            pieces = chunk_evidence(
-                text,
-                _BoundedTokenizer(tokenizer, max_chunks - len(chunks), max_tokens),
-                prefix=prefix,
-                target_tokens=target_tokens,
-                overlap_tokens=overlap_tokens,
-                max_tokens=max_tokens,
-            )
-        except ChunkingError as error:
-            if isinstance(error.__cause__, MappedContentError):
-                raise error.__cause__ from None
-            raise
-        if len(chunks) + len(pieces) > max_chunks:
-            raise MappedContentError("mapped_chunk_limit_exceeded")
-        assert span.source_start_cp is not None  # Validated copy span.
-        ordinal = len(chunks)
-        chunks.extend(
-            TextChunk(
-                ordinal + i,
-                span.source_start_cp + piece.start_cp,
-                span.source_start_cp + piece.end_cp,
-                piece.token_count,
-                piece.text,
-            )
-            for i, piece in enumerate(pieces)
-        )
-    return tuple(chunks)

memhub/apps/core/src/one_memory/retrieval/raw_candidates.py

@@ -1,147 +0,0 @@
-"""Internal raw semantic candidate lookup. No route, raw body release or full coverage claim."""
-
-import math
-import sqlite3
-from collections.abc import Sequence
-from dataclasses import dataclass
-from typing import Literal
-
-from one_memory.context_raw import _check
-from one_memory.retrieval.index_store import PublishedChunk, RetrievalIndexStore
-from one_memory.retrieval.models import RetrievalPath, TimeRange
-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError
-
-
-@dataclass(frozen=True)
-class RawSemanticCandidate:
-    chunk: PublishedChunk
-    distance: float
-
-
-@dataclass(frozen=True)
-class RawCandidateBatch:
-    hits: tuple[RawSemanticCandidate, ...] = ()
-    complete: bool = False
-    reason: str | None = None
-    coverage: Literal["indexed_only"] = "indexed_only"
-    time_range: TimeRange | None = None
-
-
-def read_raw_semantic_candidates(
-    store: RetrievalIndexStore,
-    vectors: RawVectorStore,
-    query_vector: Sequence[float],
-    *,
-    generation_id: str,
-    source_ids: frozenset[str],
-    deadline: float,
-    limit: int = 64,
-    time_range: TimeRange | None = None,
-) -> RawCandidateBatch:
-    """Caller resolves host authority; explicit source IDs can only narrow it.
-
-    complete means the eligible published chunks were checked and searched, not
-    that every authorized original is indexed. The future coordinator must still
-    revalidate host identity and all raw/memory data together before body release.
-    """
-    if not isinstance(source_ids, frozenset) or any(
-        not isinstance(source, str) or not 1 <= len(source) <= 128 for source in source_ids
-    ):
-        raise ValueError("explicit_resolved_sources_required")
-    if type(limit) is not int or not 1 <= limit <= 64:
-        raise ValueError("invalid_candidate_limit")
-    if isinstance(deadline, bool) or not math.isfinite(deadline):
-        raise ValueError("invalid_candidate_deadline")
-    if not source_ids:
-        return RawCandidateBatch(reason="not_authorized")
-    if vectors.generation_id != generation_id:
-        raise ValueError("candidate_generation_mismatch")
-    try:
-        _check(deadline)
-        eligible = store.active_chunks(
-            generation_id, source_ids=source_ids, time_range=time_range, deadline=deadline
-        )
-        if not eligible:
-            return RawCandidateBatch(
-                complete=True, reason="no_eligible_chunks", time_range=time_range
-            )
-        before = {chunk.chunk_id: chunk for chunk in eligible}
-        expected_vectors = [(chunk.chunk_id, chunk.build_id) for chunk in eligible]
-        # When one query can return the entire allowlist, that response itself
-        # is the integrity check. Avoid a duplicate Chroma get-before-query on
-        # the hot small/medium-authority path. Larger allowlists still need the
-        # explicit complete verification because search returns only top-K.
-        verify_before_search = len(eligible) > limit
-        if verify_before_search and not vectors.verify_candidates(
-            expected_vectors, deadline=deadline
-        ):
-            return RawCandidateBatch(reason="deadline")
-        result = vectors.search(query_vector, tuple(before), limit, deadline)
-        if not verify_before_search and {hit.chunk_id for hit in result.hits} != set(before):
-            raise VectorStoreError("vector_record_missing")
-        _check(deadline)
-        current = {
-            chunk.chunk_id: chunk
-            for chunk in store.active_chunks(
-                generation_id, source_ids=source_ids, time_range=time_range, deadline=deadline
-            )
-        }
-        hits = []
-        for hit in result.hits:
-            if (
-                hit.chunk_id not in before
-                or hit.generation_id != generation_id
-                or hit.build_id != before[hit.chunk_id].build_id
-            ):
-                raise VectorStoreError("candidate_identity_mismatch")
-            if current.get(hit.chunk_id) == before[hit.chunk_id]:
-                hits.append(RawSemanticCandidate(before[hit.chunk_id], hit.distance))
-        _check(deadline)
-        unchanged = before == current
-        return RawCandidateBatch(
-            tuple(hits),
-            result.complete and unchanged,
-            "evidence_changed" if not unchanged else result.reason,
-            time_range=time_range,
-        )
-    except TimeoutError:
-        return RawCandidateBatch(reason="deadline")
-    except VectorStoreError:
-        return RawCandidateBatch(reason="vector_unavailable")
-    except (sqlite3.Error, OSError, ValueError):
-        return RawCandidateBatch(reason="index_unavailable")
-
-
-def compose_raw_retrieval_path(
-    batch: RawCandidateBatch,
-    *,
-    unpublished_authorized: bool,
-    lexical_hit_count: int,
-) -> RetrievalPath:
-    """Map channel results to a path. Indexed-only search is never complete empty."""
-    if type(lexical_hit_count) is not int or lexical_hit_count < 0:
-        raise ValueError("invalid_lexical_hit_count")
-    hits = len(batch.hits) + lexical_hit_count
-    incomplete = unpublished_authorized or not batch.complete
-    channels: tuple[Literal["lexical", "semantic"], ...] = ("lexical", "semantic")
-    if hits:
-        return RetrievalPath(
-            status="ok",
-            returned_count=min(hits, 32),
-            searched_channels=channels,
-            coverage="partial" if incomplete else "complete",
-        )
-    if incomplete:
-        return RetrievalPath(
-            status="degraded",
-            reason="indexed_only",
-            returned_count=0,
-            searched_channels=channels,
-            coverage="partial",
-        )
-    return RetrievalPath(
-        status="empty",
-        returned_count=0,
-        searched_channels=channels,
-        coverage="complete",
-    )

memhub/apps/core/src/one_memory/retrieval/raw_evidence.py

@@ -1,189 +0,0 @@
-"""Materialize internal semantic candidates from current authorized Evidence.
-
-Not an arbitrary-ID endpoint or signed expansion capability. The combined v2
-coordinator must revalidate this lane with memory and host identity at release.
-"""
-
-import json
-import sqlite3
-from dataclasses import dataclass
-from datetime import UTC
-
-from one_memory import ingest_journal as journal
-from one_memory.context_raw import _check, _connection
-from one_memory.imports.models import EvidenceRecord
-from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.models import EvidenceSpan, RawEvidence
-from one_memory.retrieval.raw_candidates import RawCandidateBatch
-
-
-@dataclass(frozen=True)
-class RawEvidenceBatch:
-    items: tuple[RawEvidence, ...] = ()
-    complete: bool = False
-    reason: str | None = None
-    used_characters: int = 0
-
-
-def _merge(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:
-    output: list[tuple[int, int]] = []
-    for start, end in sorted(intervals):
-        if output and start <= output[-1][1]:
-            output[-1] = (output[-1][0], max(end, output[-1][1]))
-        else:
-            output.append((start, end))
-    return output
-
-
-def materialize_raw_candidates(
-    store: RetrievalIndexStore,
-    candidates: RawCandidateBatch,
-    *,
-    generation_id: str,
-    source_ids: frozenset[str],
-    deadline: float,
-    character_budget: int = 6000,
-) -> RawEvidenceBatch:
-    if not isinstance(source_ids, frozenset) or any(
-        not isinstance(source, str) or not 1 <= len(source) <= 128 for source in source_ids
-    ):
-        raise ValueError("explicit_resolved_sources_required")
-    if type(character_budget) is not int or not 1 <= character_budget <= 20000:
-        raise ValueError("invalid_evidence_budget")
-    if len(candidates.hits) > 64:
-        raise ValueError("candidate_budget_exceeded")
-    if not source_ids:
-        return RawEvidenceBatch(reason="not_authorized")
-    try:
-        _check(deadline)
-        expected = {hit.chunk.chunk_id: hit.chunk for hit in candidates.hits}
-        if len(expected) != len(candidates.hits):
-            raise ValueError("duplicate_candidates")
-        if not expected:
-            # Empty/excluded builds still carry an integrity claim. Validate
-            # current authorized completed manifests, never only nonempty hits.
-            current_chunks = store.active_chunks(
-                generation_id,
-                source_ids=source_ids,
-                time_range=candidates.time_range,
-                deadline=deadline,
-                require_active=True,
-            )
-            _check(deadline)
-            if current_chunks and candidates.reason == "no_eligible_chunks":
-                return RawEvidenceBatch(reason="evidence_changed")
-            return RawEvidenceBatch(complete=candidates.complete, reason=candidates.reason)
-        changed = False
-        grouped: dict[str, tuple[EvidenceRecord, list[tuple[int, int]]]] = {}
-        with _connection(store.database_path, deadline) as db:
-            db.execute("PRAGMA query_only=ON")
-            rows = {
-                row["chunk_id"]: row
-                for row in db.execute(
-                    """SELECT c.*, d.event_seq,d.grant_fingerprint,d.source_id
-                    FROM retrieval_chunks c JOIN retrieval_documents d
-                      ON d.generation_id=c.generation_id AND d.evidence_id=c.evidence_id
-                      AND d.active_build_id=c.build_id AND d.build_id=c.build_id
-                    JOIN retrieval_index_generations g ON g.generation_id=c.generation_id
-                    WHERE c.generation_id=? AND g.state='active' AND d.state='ready'
-                      AND c.chunk_id IN (SELECT value FROM json_each(?))""",
-                    (generation_id, json.dumps(tuple(expected))),
-                )
-            }
-            for key, chunk in expected.items():
-                _check(deadline)
-                row = rows.get(key)
-                if row is None or row["source_id"] not in source_ids:
-                    changed = True
-                    continue
-                event = journal.event(db, row["event_seq"])
-                grant = store._grant(db, row["source_id"])
-                if (
-                    event is None
-                    or event.user_id != "local"
-                    or event.source_id != row["source_id"]
-                    or event.evidence_id != chunk.evidence_id
-                    or event.evidence_revision != chunk.revision
-                    or grant is None
-                    or grant != row["grant_fingerprint"]
-                    or not journal.matches_current(db, event)
-                ):
-                    changed = True
-                    continue
-                record = EvidenceStore._get_with_connection(db, chunk.evidence_id)
-                if (
-                    record is None
-                    or record.user_id != "local"
-                    or record.source_id != row["source_id"]
-                ):
-                    changed = True
-                    continue
-                event_time = record.created_at
-                if event_time is not None:
-                    event_time = event_time.astimezone(UTC) if event_time.tzinfo else None
-                if (
-                    (row["build_id"], row["ordinal"], row["start_cp"], row["end_cp"])
-                    != (chunk.build_id, chunk.ordinal, chunk.start_cp, chunk.end_cp)
-                    or (record.source_id, event_time) != (chunk.source_id, chunk.event_time)
-                    or not 0 <= chunk.start_cp < chunk.end_cp <= len(record.verbatim)
-                ):
-                    changed = True
-                    continue
-                if record.evidence_id not in grouped:
-                    document = db.execute(
-                        "SELECT * FROM retrieval_documents WHERE generation_id=? AND evidence_id=?",
-                        (generation_id, record.evidence_id),
-                    ).fetchone()
-                    if document is None:
-                        raise ValueError("published_document_missing")
-                    store.validate_document(db, document, record, deadline=deadline)
-                    grouped[record.evidence_id] = (record, [])
-                grouped[record.evidence_id][1].append((chunk.start_cp, chunk.end_cp))
-            items = []
-            used = 0
-            truncated = False
-            for record, intervals in grouped.values():
-                _check(deadline)
-                spans: list[EvidenceSpan] = []
-                for start, end in _merge(intervals):
-                    if len(spans) >= 4 or end - start > min(20000, character_budget - used):
-                        truncated = True
-                        continue
-                    spans.append(
-                        EvidenceSpan(start_cp=start, end_cp=end, text=record.verbatim[start:end])
-                    )
-                    used += end - start
-                if not spans:
-                    continue
-                event_time = record.created_at
-                if event_time is not None:
-                    event_time = event_time.astimezone(UTC) if event_time.tzinfo else None
-                items.append(
-                    RawEvidence(
-                        evidence_id=record.evidence_id,
-                        revision=record.revision,
-                        source_id=record.source_id,
-                        conversation_id=record.conversation_id,
-                        role=record.role,
-                        written_by="person" if record.role == "user" else "agent",
-                        event_time=event_time,
-                        spans=tuple(spans),
-                        matched_by=("semantic",),
-                    )
-                )
-            _check(deadline)
-        return RawEvidenceBatch(
-            tuple(items),
-            candidates.complete and not changed and not truncated,
-            "evidence_changed"
-            if changed
-            else "character_budget"
-            if truncated
-            else candidates.reason,
-            used,
-        )
-    except TimeoutError:
-        return RawEvidenceBatch(reason="deadline")
-    except (sqlite3.Error, OSError, ValueError):
-        return RawEvidenceBatch(reason="index_unavailable")

memhub/apps/core/src/one_memory/retrieval/raw_lane.py

@@ -1,181 +0,0 @@
-"""Fuse lexical raw context with an optional derived index. No new HTTP schema."""
-
-import sqlite3
-from collections.abc import Callable
-from pathlib import Path
-from typing import Protocol
-
-from pydantic import ValidationError
-
-from one_memory.context_models import (
-    ContextLaneResult,
-    ContextSearchRequest,
-    ContextSemanticStatus,
-    RawContextItem,
-)
-from one_memory.context_raw import (
-    _METADATA,
-    _check,
-    _connection,
-    _execute,
-    _message,
-    _readable,
-    read_raw_context,
-)
-from one_memory.retrieval.models import TimeRange
-from one_memory.retrieval.raw_candidates import RawCandidateBatch, compose_raw_retrieval_path
-
-
-class RawIndexLookup(Protocol):
-    def lookup_raw(
-        self,
-        query: str,
-        *,
-        source_ids: frozenset[str] | None,
-        deadline: float,
-        time_range: TimeRange | None = None,
-    ) -> tuple[RawCandidateBatch, bool]: ...
-
-
-def fuse_raw_lane(
-    lexical: ContextLaneResult,
-    batch: RawCandidateBatch,
-    *,
-    unpublished_authorized: bool,
-    extra_items: tuple[RawContextItem, ...] = (),
-) -> ContextLaneResult:
-    """v1 has no coverage field; degraded + lexical_only stands in for partial index."""
-    semantic_status: ContextSemanticStatus = (
-        "timed_out"
-        if batch.reason == "deadline"
-        else "failed"
-        if batch.reason in {"index_unavailable", "vector_unavailable"}
-        else "completed"
-    )
-    index_complete = batch.complete and not unpublished_authorized
-    timed_out = semantic_status == "timed_out"
-    if lexical.status not in {"ok", "empty", "degraded"}:
-        return lexical.model_copy(
-            update={
-                "searched_channels": ("lexical", "semantic"),
-                "semantic_status": semantic_status,
-                "index_complete": index_complete,
-                "timed_out": timed_out,
-            }
-        )
-    seen = {item.anchor_evidence_id for item in lexical.items if isinstance(item, RawContextItem)}
-    extras = tuple(item for item in extra_items if isinstance(item, RawContextItem))
-    # Semantic similarity is a candidate signal, not a verified recall result.
-    # Release only semantic items that the lexical lane independently confirmed
-    # for the same evidence identity. Unconfirmed semantic hits stay in the
-    # coverage receipt as partial/degraded and must not reach host injection.
-    unconfirmed_semantic = any(item.anchor_evidence_id not in seen for item in extras)
-    items = lexical.items
-    path = compose_raw_retrieval_path(
-        batch,
-        unpublished_authorized=unpublished_authorized,
-        lexical_hit_count=len(lexical.items),
-    )
-    status = path.status
-    if unconfirmed_semantic:
-        status = "degraded"
-    if lexical.status == "degraded" and status == "ok":
-        status = "degraded"
-    return ContextLaneResult(
-        status=status,
-        reason=(
-            "lexical_only" if unconfirmed_semantic and lexical.reason is None else lexical.reason
-        ),
-        items=items[:21],
-        truncated_by=lexical.truncated_by,
-        searched_channels=("lexical", "semantic"),
-        semantic_status=semantic_status,
-        index_complete=index_complete,
-        timed_out=timed_out,
-    )
-
-
-def load_raw_items_from_candidates(
-    database_path: Path,
-    batch: RawCandidateBatch,
-    *,
-    source_ids: frozenset[str] | None,
-    deadline: float,
-    character_budget: int = 6000,
-) -> tuple[RawContextItem, ...]:
-    """Release authorized Evidence bodies for semantic IDs. Not a vector dump."""
-    ids = tuple(dict.fromkeys(hit.chunk.evidence_id for hit in batch.hits))
-    if not ids or source_ids == frozenset():
-        return ()
-    readable, authority = _readable(source_ids)
-    body_limit = min(20000, max(1, character_budget))
-    items: list[RawContextItem] = []
-    try:
-        with _connection(database_path, deadline) as db:
-            for evidence_id in ids[:21]:
-                _check(deadline)
-                row = _execute(
-                    db,
-                    deadline,
-                    f"""
-                    SELECT {_METADATA}, substr(e.verbatim,1,?) AS text,
-                           length(e.verbatim)>? AS body_truncated
-                    FROM evidence e
-                    WHERE {readable} AND e.evidence_id=?
-                    """,
-                    [body_limit, body_limit, *authority, evidence_id],
-                ).fetchone()
-                if row is None:
-                    continue
-                try:
-                    message = _message(row)
-                    items.append(
-                        RawContextItem(
-                            source_id=row["source_id"],
-                            conversation_id=row["conversation_id"],
-                            anchor_evidence_id=message.evidence_id,
-                            messages=(message,),
-                            truncated=message.truncated,
-                        )
-                    )
-                except (ValidationError, ValueError, TypeError):
-                    continue
-    except TimeoutError:
-        raise
-    except (sqlite3.Error, OSError):
-        return ()
-    return tuple(items)
-
-
-def read_raw_indexed_context(
-    database_path: Path,
-    request: ContextSearchRequest,
-    *,
-    source_ids: frozenset[str] | None,
-    deadline: float,
-    index: RawIndexLookup | None = None,
-    load_semantic_items: Callable[..., tuple[RawContextItem, ...]] | None = None,
-) -> ContextLaneResult:
-    lexical = read_raw_context(database_path, request, source_ids=source_ids, deadline=deadline)
-    if index is None:
-        return lexical
-    try:
-        batch, unpublished = index.lookup_raw(
-            request.query, source_ids=source_ids, deadline=deadline
-        )
-    except Exception:
-        batch, unpublished = RawCandidateBatch(reason="index_unavailable"), True
-    extra: tuple[RawContextItem, ...] = ()
-    if batch.hits:
-        extra = (
-            load_semantic_items(database_path, batch, source_ids=source_ids, deadline=deadline)
-            if load_semantic_items is not None
-            else load_raw_items_from_candidates(
-                database_path,
-                batch,
-                source_ids=source_ids,
-                deadline=deadline,
-                character_budget=request.character_budget,
-            )
-        )
-    return fuse_raw_lane(lexical, batch, unpublished_authorized=unpublished, extra_items=extra)

memhub/apps/core/src/one_memory/retrieval/runtime.py

@@ -1,379 +0,0 @@
-"""Resume an explicitly configured local index; never create or migrate one."""
-
-import asyncio
-import json
-import os
-import shutil
-import sqlite3
-import stat
-import sys
-import threading
-from collections.abc import Callable
-from dataclasses import asdict, dataclass
-from pathlib import Path
-from typing import Literal
-
-from one_memory.database_access import main_database
-from one_memory.retrieval.index_store import EncoderIdentity, RetrievalIndexStore
-from one_memory.retrieval.index_supervisor import IndexSupervisorSnapshot, RetrievalIndexSupervisor
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder, RetrievalIndexWorker
-from one_memory.retrieval.models import TimeRange
-from one_memory.retrieval.raw_candidates import RawCandidateBatch, read_raw_semantic_candidates
-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError
-from one_memory.service_levels import ServiceLevel, ServiceLevelScheduler, ServiceResource
-
-type RuntimeState = Literal[
-    "new", "starting", "running", "not_initialized", "failed", "stopping", "stopped"
-]
-
-
-@dataclass(frozen=True)
-class ExistingRetrievalSnapshot:
-    state: RuntimeState
-    error_code: str | None
-    generation_id: str | None
-    supervisor: IndexSupervisorSnapshot | None
-
-
-class _AdmissionError(Exception):
-    def __init__(self, code: str, *, missing: bool = False):
-        self.code, self.missing = code, missing
-
-
-async def _native[T](operation: Callable[[], T]) -> T:
-    """Owned, joined native call. Only retained internal lifecycle tasks invoke this."""
-    result: list[T] = []
-    errors: list[BaseException] = []
-
-    def run() -> None:
-        try:
-            result.append(operation())
-        except BaseException as error:
-            errors.append(error)
-
-    thread = threading.Thread(target=run, name="retrieval-resource", daemon=False)
-    thread.start()
-    while thread.is_alive():
-        await asyncio.sleep(0.01)
-    thread.join()
-    if errors:
-        raise errors[0]
-    return result[0]
-
-
-class ExistingRetrievalRuntime:
-    """Caller guarantees managed Core single ownership and explicit configuration.
-
-    Construct without I/O; start and stop on the owning event loop. A retained
-    lifecycle task shields native work from caller cancellation until cleanup.
-    """
-
-    def __init__(
-        self,
-        data_dir: Path,
-        *,
-        minimum_free_bytes: int = 256 * 1024 * 1024,
-        service_levels: ServiceLevelScheduler | None = None,
-    ):
-        if type(minimum_free_bytes) is not int or minimum_free_bytes < 0:
-            raise ValueError("retrieval_runtime_invalid_disk_limit")
-        self._directory = Path(data_dir)
-        self._minimum_free = minimum_free_bytes
-        self._service_levels = service_levels or ServiceLevelScheduler()
-        self._state: ExistingRetrievalSnapshot = ExistingRetrievalSnapshot("new", None, None, None)
-        self._vectors: RawVectorStore | None = None
-        self._store: RetrievalIndexStore | None = None
-        self._encoder: LocalRetrievalEncoder | None = None
-        self._supervisor: RetrievalIndexSupervisor | None = None
-        self._last_supervisor_snapshot: IndexSupervisorSnapshot | None = None
-        self._start_task: asyncio.Task[bool] | None = None
-        self._stop_task: asyncio.Task[None] | None = None
-        self._stop_requested = False
-
-    def snapshot(self) -> ExistingRetrievalSnapshot:
-        return ExistingRetrievalSnapshot(
-            self._state.state,
-            self._state.error_code,
-            self._state.generation_id,
-            self._supervisor.snapshot()
-            if self._supervisor is not None
-            else self._last_supervisor_snapshot,
-        )
-
-    def _set(
-        self, state: RuntimeState, code: str | None = None, generation: str | None = None
-    ) -> None:
-        self._state = ExistingRetrievalSnapshot(
-            state,
-            code,
-            generation if generation is not None else self._state.generation_id,
-            None,
-        )
-
-    def trigger(self) -> bool:
-        return bool(
-            self._state.state == "running" and self._supervisor and self._supervisor.trigger()
-        )
-
-    def lookup_raw(
-        self,
-        query: str,
-        *,
-        source_ids: frozenset[str] | None,
-        deadline: float,
-        time_range: TimeRange | None = None,
-    ) -> tuple[RawCandidateBatch, bool]:
-        """Read-only candidate lookup. Does not release original bodies."""
-        store, encoder, vectors, generation = (
-            self._store,
-            self._encoder,
-            self._vectors,
-            self._state.generation_id,
-        )
-        if (
-            self._state.state != "running"
-            or store is None
-            or encoder is None
-            or vectors is None
-            or generation is None
-        ):
-            return RawCandidateBatch(reason="index_unavailable"), True
-        counts = store.status(generation)
-        unpublished = counts["pending"] + counts["leased"] + counts["blocked"] > 0
-        resolved = source_ids
-        if resolved is None:
-            resolved = frozenset(
-                chunk.source_id for chunk in store.active_chunks(generation, deadline=deadline)
-            )
-        if not resolved:
-            return (
-                RawCandidateBatch(complete=not unpublished, reason="no_eligible_chunks"),
-                unpublished,
-            )
-        try:
-            vector = encoder.encode_documents([query.strip()])[0]
-        except Exception:
-            return RawCandidateBatch(reason="index_unavailable"), True
-        batch = read_raw_semantic_candidates(
-            store,
-            vectors,
-            vector,
-            generation_id=generation,
-            source_ids=resolved,
-            deadline=deadline,
-            time_range=time_range,
-        )
-        return batch, unpublished
-
-    async def start(self) -> bool:
-        if self._state.state != "new" or self._stop_requested:
-            raise RuntimeError("retrieval_runtime_already_started")
-        self._set("starting")
-        self._start_task = asyncio.create_task(self._start())
-        try:
-            return await asyncio.shield(self._start_task)
-        except asyncio.CancelledError:
-            # Retain and drain opening resources, then release them, before cancellation escapes.
-            await self.stop()
-            raise
-
-    async def stop(self) -> None:
-        self._stop_requested = True
-        if self._stop_task is None:
-            self._stop_task = asyncio.create_task(self._stop())
-        cancelled = False
-        while not self._stop_task.done():
-            try:
-                await asyncio.shield(self._stop_task)
-            except asyncio.CancelledError:
-                cancelled = True
-        self._stop_task.result()
-        if cancelled:
-            raise asyncio.CancelledError
-
-    async def _start(self) -> bool:
-        try:
-            store, encoder, vectors, generation = await _native(self._open)
-            self._store = store
-            self._encoder = encoder
-            self._vectors = vectors
-            self._set("starting", generation=generation)
-            if self._stop_requested:
-                return False
-            worker = RetrievalIndexWorker(
-                store, vectors, encoder, generation_id=generation, owner="managed-core-retrieval"
-            )
-            self._supervisor = RetrievalIndexSupervisor(
-                worker,
-                background_should_yield=lambda: self._service_levels.should_yield(
-                    ServiceResource.INDEX,
-                    ServiceLevel.BACKGROUND,
-                ),
-            )
-            self._supervisor.start()
-            self._set("running")
-            return True
-        except _AdmissionError as error:
-            self._set("not_initialized" if error.missing else "failed", error.code)
-        except Exception:
-            self._set("failed", "retrieval_runtime_start_failed")
-        await self._release()
-        return False
-
-    async def _release(self) -> None:
-        stop_failed = False
-        if self._supervisor is not None:
-            try:
-                await self._supervisor.stop()
-            except Exception:
-                stop_failed = True
-            supervisor_snapshot = self._supervisor.snapshot()
-            if supervisor_snapshot.thread_alive:
-                self._set("failed", "retrieval_runtime_stop_failed")
-                raise RuntimeError("retrieval_runtime_stop_failed") from None
-            # The supervisor owns the worker, which owns the native vector
-            # client. Retain only the immutable receipt once its thread drains.
-            self._last_supervisor_snapshot = supervisor_snapshot
-            self._supervisor = None
-        if self._vectors is not None:
-            try:
-                await _native(self._vectors.close)
-            except Exception:
-                self._set("failed", "retrieval_runtime_close_failed")
-                raise RuntimeError("retrieval_runtime_close_failed") from None
-            self._vectors = None
-        self._store = None
-        self._encoder = None
-        if stop_failed:
-            self._set("failed", "retrieval_runtime_stop_failed")
-            raise RuntimeError("retrieval_runtime_stop_failed") from None
-
-    async def _stop(self) -> None:
-        if self._start_task is not None:
-            try:
-                await asyncio.shield(self._start_task)
-            except Exception:
-                # Startup can itself fail during cleanup. Do not lose resource ownership.
-                self._set("failed", "retrieval_runtime_start_failed")
-        self._set("stopping", self._state.error_code)
-        await self._release()
-        if self._state.error_code == "retrieval_runtime_close_failed":
-            return
-        self._set("stopped", self._state.error_code)
-
-    @staticmethod
-    def _path(path: Path, *, directory: bool) -> tuple[int, int]:
-        if any(part.is_symlink() for part in (path, *path.parents)):
-            raise _AdmissionError("retrieval_runtime_unsafe_path")
-        try:
-            info = path.stat()
-        except FileNotFoundError:
-            raise _AdmissionError("retrieval_runtime_not_initialized", missing=True) from None
-        if not (stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)):
-            raise _AdmissionError("retrieval_runtime_unsafe_path")
-        if not directory and info.st_nlink != 1:
-            raise _AdmissionError("retrieval_runtime_unsafe_path")
-        if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) & 0o022:
-            raise _AdmissionError("retrieval_runtime_unsafe_path")
-        return info.st_dev, info.st_ino
-
-    def _sidecars(self, database: Path) -> None:
-        for suffix in ("-wal", "-shm", "-journal"):
-            path = database.with_name(database.name + suffix)
-            if path.is_symlink() or path.exists():
-                self._path(path, directory=False)
-
-    def _inspect(self, database: Path) -> tuple[RetrievalIndexStore, EncoderIdentity, str]:
-        with main_database(database.absolute()).read(busy_timeout_ms=1_000) as db:
-            db.row_factory = sqlite3.Row
-            db.execute("PRAGMA query_only=ON")
-            db.execute("BEGIN")
-            if not db.execute(
-                "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
-            ).fetchone():
-                raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)
-            state = db.execute(
-                "SELECT schema_version FROM retrieval_index_state WHERE singleton=1"
-            ).fetchone()
-            if state is not None and state[0] == 1:
-                raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)
-            if state is None or state[0] != 2:
-                raise _AdmissionError("retrieval_runtime_invalid_index")
-            rows = db.execute(
-                "SELECT generation_id,identity_json FROM retrieval_index_generations "
-                "WHERE state='active' LIMIT 2"
-            ).fetchall()
-            if not rows:
-                raise _AdmissionError("retrieval_runtime_not_initialized", missing=True)
-            if len(rows) != 1:
-                raise _AdmissionError("retrieval_runtime_invalid_index")
-            store = RetrievalIndexStore(database)
-            generation = rows[0]["generation_id"]
-            identity = store._identity(db, generation)
-            raw = json.loads(rows[0]["identity_json"])
-            if raw != asdict(identity) or set(raw) != set(asdict(identity)):
-                raise _AdmissionError("retrieval_runtime_invalid_index")
-            # Check required schema shape without repairing it or reading original bodies.
-            db.execute(
-                "SELECT build_id,source_map_json,map_digest,coverage_state "
-                "FROM retrieval_build_maps LIMIT 0"
-            )
-            db.execute("SELECT after_seq,retry_after_seq FROM retrieval_consumers LIMIT 0")
-            db.execute("SELECT lease_token,active_build_id FROM retrieval_documents LIMIT 0")
-            db.execute("SELECT chunk_id,text_hash FROM retrieval_chunks LIMIT 0")
-            return store, identity, generation
-
-    def _open(self) -> tuple[RetrievalIndexStore, LocalRetrievalEncoder, RawVectorStore, str]:
-        if sys.platform == "win32" or not hasattr(os, "getuid"):
-            raise _AdmissionError("retrieval_runtime_platform_unsupported")
-        directory = self._directory.absolute()
-        database, vector_dir = directory / "one-memory.sqlite3", directory / "retrieval-vectors"
-        root_pin = self._path(directory, directory=True)
-        database_pin = self._path(database, directory=False)
-        self._sidecars(database)
-        try:
-            store, identity, generation = self._inspect(database)
-        except (ValueError, TypeError, sqlite3.Error):
-            raise _AdmissionError("retrieval_runtime_invalid_index") from None
-        vector_pin = self._path(vector_dir, directory=True)
-        vector_database = vector_dir / "chroma.sqlite3"
-        vector_db_pin = self._path(vector_database, directory=False)
-        self._sidecars(vector_database)
-
-        def recheck() -> None:
-            if (
-                self._path(directory, directory=True) != root_pin
-                or self._path(database, directory=False) != database_pin
-                or self._path(vector_dir, directory=True) != vector_pin
-                or self._path(vector_database, directory=False) != vector_db_pin
-            ):
-                raise _AdmissionError("retrieval_runtime_index_changed")
-            self._sidecars(database)
-            self._sidecars(vector_database)
-
-        if shutil.disk_usage(directory).free < self._minimum_free:
-            raise _AdmissionError("retrieval_runtime_low_disk")
-        encoder = LocalRetrievalEncoder(content_version=identity.normalizer_version)
-        if encoder.identity != identity:
-            raise _AdmissionError("retrieval_runtime_encoder_mismatch")
-        recheck()
-        try:
-            vectors = RawVectorStore.open_writer_existing(
-                vector_dir, generation_id=generation, dimensions=identity.dimensions
-            )
-            # Retain ownership immediately so all later failures use the same cleanup path.
-            self._vectors = vectors
-            # A generation change during the native open is not authorization to resume another.
-            recheck()
-            if self._inspect(database)[1:] != (identity, generation):
-                raise _AdmissionError("retrieval_runtime_index_changed")
-            recheck()
-            return store, encoder, vectors, generation
-        except BaseException as error:
-            if isinstance(error, VectorStoreError):
-                if str(error) in {"vector_store_not_found", "vector_collection_not_found"}:
-                    raise _AdmissionError(
-                        "retrieval_runtime_not_initialized", missing=True
-                    ) from None
-                raise _AdmissionError("retrieval_runtime_vector_unavailable") from None
-            raise

memhub/apps/core/src/one_memory/retrieval/vector_store.py

@@ -1,534 +0,0 @@
-"""Independent raw-vector adapter. Eligibility is supplied by Core's SQLite snapshot.
-
-No HY collection, document text, embedding function, migration/repair or authority
-store lives here. Chroma PersistentClient is NOT physically read-only: even the
-existing-collection handle can maintain backend metadata/index files. A caller
-requiring filesystem read-only operation must not use this backend.
-
-Scoring is an exact pass over the eligible vectors, not a global ANN top-K that
-is filtered afterwards. Chroma's filtered query does not score every candidate
-in its `where $in` allowlist: at 10k stored chunks an explicit 64-ID allowlist
-returned 26 of 64 while reporting the query as complete, and 0 of 64 after the
-writer handle was reopened. A post-filtered global search would be neither
-permission-correct nor recall-correct, so the eligible embeddings are read
-back and ranked here.
-"""
-
-from __future__ import annotations
-
-import math
-import re
-import sqlite3
-import struct
-import threading
-from collections.abc import Callable, Sequence
-from contextlib import closing
-from dataclasses import dataclass
-from hashlib import sha256
-from numbers import Real
-from pathlib import Path
-from time import monotonic
-from typing import Any, Literal
-
-ALLOWLIST_BATCH_SIZE = 2048
-
-
-class VectorStoreError(RuntimeError):
-    """Bounded failure code; callers must not treat corruption as complete empty search."""
-
-
-@dataclass(frozen=True)
-class RawVectorRecord:
-    chunk_id: str
-    build_id: str
-    generation_id: str
-    vector: tuple[float, ...]
-
-    def __post_init__(self) -> None:
-        object.__setattr__(self, "vector", tuple(self.vector))
-
-
-@dataclass(frozen=True)
-class VectorHit:
-    chunk_id: str
-    build_id: str
-    generation_id: str
-    distance: float
-
-
-@dataclass(frozen=True)
-class VectorSearchResult:
-    hits: tuple[VectorHit, ...]
-    complete: bool
-    eligible_count: int
-    searched_count: int
-    reason: Literal["deadline"] | None
-
-
-class RawVectorStore:
-    """Unopened constructor has no I/O. Use create/open_existing explicitly.
-
-    Directory must be a dedicated, trusted Runtime retrieval-vectors directory,
-    never the HY data directory. Only create/open_writer_existing enable upsert.
-    Core must serialize writers across handles/processes; this instance's lock
-    is not a persistent writer lease. close() releases only this Chroma client,
-    retaining other live clients' references.
-    """
-
-    def __init__(
-        self,
-        directory: Path,
-        *,
-        generation_id: str,
-        dimensions: int,
-        clock: Callable[[], float] = monotonic,
-    ) -> None:
-        _identifier(generation_id)
-        if type(dimensions) is not int or not 1 <= dimensions <= 65536:
-            raise VectorStoreError("vector_invalid_dimensions")
-        self.directory = Path(directory)
-        self.generation_id = generation_id
-        self.dimensions = dimensions
-        self.collection_name = "raw_" + sha256(generation_id.encode()).hexdigest()
-        self._clock = clock
-        self._client: Any = None
-        self._collection: Any = None
-        self._writable = False
-        self._closed = False
-        self._lock = threading.RLock()
-
-    @classmethod
-    def create(cls, directory: Path, *, generation_id: str, dimensions: int) -> RawVectorStore:
-        """Explicit writer initialization. Refuses to reuse an existing collection."""
-        value = cls(directory, generation_id=generation_id, dimensions=dimensions)
-        value._open(create=True)
-        return value
-
-    @classmethod
-    def open_existing(
-        cls,
-        directory: Path,
-        *,
-        generation_id: str,
-        dimensions: int,
-    ) -> RawVectorStore:
-        """No logical index writes; persistent Chroma may still write maintenance files."""
-        value = cls(directory, generation_id=generation_id, dimensions=dimensions)
-        value._open(create=False)
-        return value
-
-    @classmethod
-    def open_writer_existing(
-        cls,
-        directory: Path,
-        *,
-        generation_id: str,
-        dimensions: int,
-    ) -> RawVectorStore:
-        """Explicit worker recovery; verifies existing collection, never creates/repairs."""
-        value = cls(directory, generation_id=generation_id, dimensions=dimensions)
-        value._open(create=False, writable=True)
-        return value
-
-    def _metadata(self) -> dict[str, str | int]:
-        return {
-            "adapter": "one-memory-raw-v1",
-            "generation_id": self.generation_id,
-            "dimensions": self.dimensions,
-            "distance": "cosine",
-            "normalization": "l2-float32-v1",
-        }
-
-    def _preflight(self, create: bool) -> None:
-        if any(part.is_symlink() for part in (self.directory, *self.directory.parents)):
-            raise VectorStoreError("vector_unsafe_directory")
-        if not self.directory.exists():
-            if not create:
-                raise VectorStoreError("vector_store_not_found")
-            self.directory.mkdir(parents=True)
-        if not self.directory.is_dir():
-            raise VectorStoreError("vector_unsafe_directory")
-        database = self.directory / "chroma.sqlite3"
-        if database.is_symlink():
-            raise VectorStoreError("vector_unsafe_database")
-        if not database.is_file():
-            if not create:
-                raise VectorStoreError("vector_store_not_found")
-            if any(self.directory.iterdir()):
-                raise VectorStoreError("vector_unknown_directory")
-            return
-        if database.stat().st_nlink != 1:
-            raise VectorStoreError("vector_unsafe_database")
-        # Refuse missing/corrupt collections BEFORE PersistentClient can perform
-        # startup maintenance. This checks the installed Chroma 1.5.9 schema.
-        try:
-            with closing(sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True)) as db:
-                found = db.execute(
-                    "SELECT 1 FROM collections WHERE name=?", (self.collection_name,)
-                ).fetchone()
-                if not create and found is None:
-                    raise VectorStoreError("vector_collection_not_found")
-                if create and found is not None:
-                    raise VectorStoreError("vector_collection_exists")
-        except sqlite3.Error:
-            raise VectorStoreError("vector_store_unavailable") from None
-
-    def _open(self, *, create: bool, writable: bool = False) -> None:
-        self._preflight(create)
-        try:
-            import chromadb
-            from chromadb.config import Settings
-
-            self._client = chromadb.PersistentClient(
-                path=str(self.directory), settings=Settings(anonymized_telemetry=False)
-            )
-            if create:
-                self._collection = self._client.create_collection(
-                    self.collection_name,
-                    embedding_function=None,
-                    metadata=self._metadata(),
-                    # Unit vectors + inner product give cosine distance without
-                    # cosine HNSW's lossy normalization/de-normalization on get.
-                    configuration={"hnsw": {"space": "ip"}},
-                )
-            else:
-                self._collection = self._client.get_collection(
-                    self.collection_name, embedding_function=None
-                )
-            if (
-                self._collection.metadata != self._metadata()
-                or type(self._collection.metadata.get("dimensions")) is not int
-            ):
-                raise VectorStoreError("vector_collection_identity_mismatch")
-            config = self._collection.configuration
-            if (
-                config.get("hnsw", {}).get("space") != "ip"
-                or config.get("embedding_function") is not None
-            ):
-                raise VectorStoreError("vector_collection_configuration_mismatch")
-            self._writable = create or writable
-        except Exception as error:
-            self.close()
-            if isinstance(error, VectorStoreError):
-                raise
-            raise VectorStoreError("vector_store_unavailable") from None
-
-    def close(self) -> None:
-        with self._lock:
-            if self._closed:
-                return
-            self._closed = True
-            if self._client is not None:
-                self._client.close()
-            self._client = self._collection = None
-
-    def _require_open(self) -> Any:
-        if self._closed:
-            raise VectorStoreError("vector_store_closed")
-        if self._collection is None:
-            raise VectorStoreError("vector_store_not_open")
-        return self._collection
-
-    def _records(self, records: Sequence[RawVectorRecord]) -> tuple[RawVectorRecord, ...]:
-        result = []
-        seen: set[str] = set()
-        for record in records:
-            if not isinstance(record, RawVectorRecord):
-                raise VectorStoreError("vector_invalid_record")
-            for identifier in (record.chunk_id, record.build_id, record.generation_id):
-                _identifier(identifier)
-            if record.generation_id != self.generation_id or record.chunk_id in seen:
-                raise VectorStoreError("vector_invalid_record_identity")
-            seen.add(record.chunk_id)
-            result.append(
-                RawVectorRecord(
-                    record.chunk_id,
-                    record.build_id,
-                    record.generation_id,
-                    _vector(record.vector, self.dimensions),
-                )
-            )
-        return tuple(result)
-
-    def _read_records(self, ids: Sequence[str]) -> dict[str, tuple[RawVectorRecord, str]]:
-        if not ids:
-            return {}
-        collection = self._require_open()
-        result: dict[str, tuple[RawVectorRecord, str]] = {}
-        for start in range(0, len(ids), ALLOWLIST_BATCH_SIZE):
-            batch = ids[start : start + ALLOWLIST_BATCH_SIZE]
-            try:
-                data = collection.get(ids=list(batch), include=["embeddings", "metadatas"])
-                returned, vectors, metadatas = data["ids"], data["embeddings"], data["metadatas"]
-                if (
-                    vectors is None
-                    or metadatas is None
-                    or not len(returned) == len(vectors) == len(metadatas)
-                ):
-                    raise VectorStoreError("vector_invalid_record_response")
-                for key, vector, metadata in zip(returned, vectors, metadatas, strict=True):
-                    if key not in batch or key in result:
-                        raise VectorStoreError("vector_ineligible_record")
-                    result[key] = self._decode(key, vector, metadata)
-            except VectorStoreError:
-                raise
-            except Exception:
-                raise VectorStoreError("vector_read_unavailable") from None
-        return result
-
-    def _decode(
-        self, key: str, vector: Sequence[float], metadata: Any
-    ) -> tuple[RawVectorRecord, str]:
-        if not isinstance(metadata, dict) or set(metadata) != {
-            "chunk_id",
-            "build_id",
-            "generation_id",
-            "vector_sha256",
-            "input_sha256",
-        }:
-            raise VectorStoreError("vector_invalid_metadata")
-        if metadata["chunk_id"] != key or metadata["generation_id"] != self.generation_id:
-            raise VectorStoreError("vector_metadata_identity_mismatch")
-        _identifier(key)
-        _identifier(metadata["build_id"])
-        if not isinstance(metadata["input_sha256"], str) or not re.fullmatch(
-            r"[0-9a-f]{64}", metadata["input_sha256"]
-        ):
-            raise VectorStoreError("vector_invalid_input_digest")
-        normalized = _vector(vector, self.dimensions)
-        if metadata["vector_sha256"] != _digest(normalized):
-            raise VectorStoreError("vector_content_mismatch")
-        return RawVectorRecord(key, metadata["build_id"], self.generation_id, normalized), metadata[
-            "input_sha256"
-        ]
-
-    def upsert(self, records: Sequence[RawVectorRecord]) -> None:
-        """Idempotent immutable insertion; add-only at Chroma, never overwrite IDs.
-
-        Equality is canonical float32 input identity plus explicit normalized
-        storage identity. Conflict is checked before
-        any add. Backend failure may leave a partial new batch: caller must verify
-        the entire build before publication. Existing IDs are never modified.
-        """
-        with self._lock:
-            collection = self._require_open()
-            if not self._writable:
-                raise VectorStoreError("vector_store_read_only")
-            incoming = self._records(records)
-            existing = self._read_records([record.chunk_id for record in incoming])
-            for record in incoming:
-                if record.chunk_id in existing and not _matches(existing[record.chunk_id], record):
-                    raise VectorStoreError("vector_immutable_conflict")
-            new = [record for record in incoming if record.chunk_id not in existing]
-            for start in range(0, len(new), ALLOWLIST_BATCH_SIZE):
-                batch = new[start : start + ALLOWLIST_BATCH_SIZE]
-                try:
-                    collection.add(
-                        ids=[record.chunk_id for record in batch],
-                        embeddings=[list(_unit_vector(record.vector)) for record in batch],
-                        metadatas=[
-                            {
-                                "chunk_id": record.chunk_id,
-                                "build_id": record.build_id,
-                                "generation_id": record.generation_id,
-                                "vector_sha256": _digest(_unit_vector(record.vector)),
-                                "input_sha256": _digest(record.vector),
-                            }
-                            for record in batch
-                        ],
-                    )
-                except Exception:
-                    raise VectorStoreError("vector_write_unavailable") from None
-            self.verify(incoming)
-
-    def verify(self, records: Sequence[RawVectorRecord]) -> None:
-        """Verify every expected ID, metadata, dimensions and exact float32 vector."""
-        with self._lock:
-            self._require_open()
-            expected = self._records(records)
-            actual = self._read_records([record.chunk_id for record in expected])
-            for record in expected:
-                if record.chunk_id not in actual:
-                    raise VectorStoreError("vector_record_missing")
-                if not _matches(actual[record.chunk_id], record):
-                    raise VectorStoreError("vector_immutable_conflict")
-
-    def verify_candidates(
-        self,
-        candidates: Sequence[tuple[str, str]],
-        *,
-        deadline: float,
-    ) -> bool:
-        """Read-only presence/integrity check for (chunk_id, expected_build_id).
-
-        Costs O(candidates * dimensions), not a bounded top-K lookup. False means
-        deadline exhaustion, never proof of absence. Each read covers at most
-        2048 IDs; synchronous backend calls cannot be interrupted. This is not a
-        snapshot across Chroma and SQLite: Core still owns final eligibility
-        checks and writer serialization. Chroma may perform maintenance writes.
-        """
-        with self._lock:
-            self._require_open()
-            if (
-                isinstance(deadline, bool)
-                or not isinstance(deadline, Real)
-                or not math.isfinite(deadline)
-            ):
-                raise VectorStoreError("vector_invalid_deadline")
-            if not isinstance(candidates, Sequence) or isinstance(candidates, (str, bytes)):
-                raise VectorStoreError("vector_invalid_candidates")
-            expected: list[tuple[str, str]] = []
-            seen: set[str] = set()
-            for pair in candidates:
-                if (
-                    not isinstance(pair, Sequence)
-                    or isinstance(pair, (str, bytes))
-                    or len(pair) != 2
-                ):
-                    raise VectorStoreError("vector_invalid_candidates")
-                key, build = pair
-                _identifier(key)
-                _identifier(build)
-                if key in seen:
-                    raise VectorStoreError("vector_duplicate_candidates")
-                seen.add(key)
-                expected.append((key, build))
-            for start in range(0, len(expected), ALLOWLIST_BATCH_SIZE):
-                if self._clock() >= deadline:
-                    return False
-                batch = expected[start : start + ALLOWLIST_BATCH_SIZE]
-                # Bound this call to one batch so it cannot read past the next
-                # deadline checkpoint via _read_records' internal loop.
-                actual = self._read_records([key for key, _build in batch])
-                for key, build in batch:
-                    if key not in actual:
-                        raise VectorStoreError("vector_record_missing")
-                    if actual[key][0].build_id != build:
-                        raise VectorStoreError("vector_metadata_identity_mismatch")
-                if self._clock() >= deadline:
-                    return False
-            return True
-
-    def search(
-        self,
-        query_vector: Sequence[float],
-        eligible_ids: Sequence[str],
-        limit: int,
-        deadline: float,
-    ) -> VectorSearchResult:
-        """Rank exactly the eligible candidates; synchronous backend cannot be preempted.
-
-        deadline is absolute monotonic seconds. A late batch is discarded; prior
-        on-time batches may return partial. complete means every eligible
-        vector was scored, NOT complete index coverage: only chunks the caller
-        already resolved as eligible and published are searched.
-
-        The eligible embeddings are read back in bounded batches and ranked
-        here, because Chroma's own filtered query does not score every ID in
-        its allowlist. A global top-K filtered afterwards would rank
-        unauthorized vectors first and let them crowd the eligible ones out.
-        """
-        with self._lock:
-            self._require_open()
-            vector = _unit_vector(_vector(query_vector, self.dimensions))
-            if type(limit) is not int or not 1 <= limit <= ALLOWLIST_BATCH_SIZE:
-                raise VectorStoreError("vector_invalid_limit")
-            if (
-                isinstance(deadline, bool)
-                or not isinstance(deadline, Real)
-                or not math.isfinite(deadline)
-            ):
-                raise VectorStoreError("vector_invalid_deadline")
-            if isinstance(eligible_ids, (str, bytes)):
-                raise VectorStoreError("vector_invalid_allowlist")
-            ids = tuple(eligible_ids)
-            for key in ids:
-                _identifier(key)
-            if len(set(ids)) != len(ids):
-                raise VectorStoreError("vector_duplicate_allowlist")
-            hits: list[VectorHit] = []
-            searched = 0
-            for start in range(0, len(ids), ALLOWLIST_BATCH_SIZE):
-                if self._clock() >= deadline:
-                    break
-                batch = ids[start : start + ALLOWLIST_BATCH_SIZE]
-                try:
-                    read = self._read_records(batch)
-                except VectorStoreError as error:
-                    # A missing or corrupt eligible vector is not "no hits".
-                    raise VectorStoreError("vector_query_unavailable") from error
-                if self._clock() >= deadline:
-                    break
-                try:
-                    missing = [key for key in batch if key not in read]
-                    if missing:
-                        # The caller resolved these as eligible and published,
-                        # so an absent vector is a broken build, not a miss.
-                        raise VectorStoreError("vector_query_unavailable")
-                    validated = [
-                        VectorHit(
-                            key,
-                            read[key][0].build_id,
-                            read[key][0].generation_id,
-                            # Stored vectors are unit length, so the inner
-                            # product is the cosine similarity.
-                            1.0 - _dot(read[key][0].vector, vector),
-                        )
-                        for key in batch
-                    ]
-                except VectorStoreError:
-                    raise
-                except Exception:
-                    raise VectorStoreError("vector_invalid_query_response") from None
-                if self._clock() >= deadline:
-                    break
-                searched += len(batch)
-                hits.extend(validated)
-                hits.sort(key=lambda hit: (hit.distance, hit.chunk_id))
-                del hits[limit:]
-            complete = searched == len(ids)
-            return VectorSearchResult(
-                tuple(hits), complete, len(ids), searched, None if complete else "deadline"
-            )
-
-
-def _dot(left: Sequence[float], right: Sequence[float]) -> float:
-    return math.fsum(a * b for a, b in zip(left, right, strict=True))
-
-
-def _identifier(value: Any) -> None:
-    if not isinstance(value, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.:-]{0,255}", value):
-        raise VectorStoreError("vector_invalid_identifier")
-
-
-def _vector(value: Sequence[float], dimensions: int) -> tuple[float, ...]:
-    try:
-        if len(value) != dimensions or any(
-            isinstance(v, bool) or not isinstance(v, Real) or not math.isfinite(v) for v in value
-        ):
-            raise VectorStoreError("vector_invalid_values_or_dimensions")
-        packed = struct.pack(f"<{dimensions}f", *value)
-        normalized = struct.unpack(f"<{dimensions}f", packed)
-        if not all(math.isfinite(v) for v in normalized) or not any(normalized):
-            raise VectorStoreError("vector_invalid_values_or_dimensions")
-        return normalized
-    except VectorStoreError:
-        raise
-    except (ValueError, TypeError, OverflowError, struct.error):
-        raise VectorStoreError("vector_invalid_values_or_dimensions") from None
-
-
-def _digest(vector: tuple[float, ...]) -> str:
-    return sha256(struct.pack(f"<{len(vector)}f", *vector)).hexdigest()
-
-
-def _unit_vector(vector: tuple[float, ...]) -> tuple[float, ...]:
-    length = math.sqrt(math.fsum(component * component for component in vector))
-    return _vector(tuple(component / length for component in vector), len(vector))
-
-
-def _matches(actual: tuple[RawVectorRecord, str], expected: RawVectorRecord) -> bool:
-    stored, input_digest = actual
-    return input_digest == _digest(expected.vector) and stored == RawVectorRecord(
-        expected.chunk_id, expected.build_id, expected.generation_id, _unit_vector(expected.vector)
-    )

memhub/apps/core/src/one_memory/sdk_data_plane_migration.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import json
 import sqlite3
 from dataclasses import dataclass
 from datetime import UTC, datetime
@@ -40,10 +39,6 @@ class MigrationProgress:
     migration_id: str
     state: str
     phase: str
-    counts: dict[str, int]
-
-
-TERMINAL_ITEM_STATES = frozenset({"verified", "adopted", "deleted", "skipped"})
 
 
 class SdkDataPlaneMigrationStore:
@@ -55,18 +50,29 @@ class SdkDataPlaneMigrationStore:
         return datetime.now(UTC).isoformat()
 
     def authority(self) -> MemoryAuthority:
+        return self.authority_state()[0]
+
+    def authority_state(self) -> tuple[MemoryAuthority, str | None]:
+        """The surviving singleton: authority and the migration that set it.
+
+        This is the only source of authority. The detailed cutover receipt is
+        dropped after cleanup and never exists on a fresh home.
+        """
         try:
             with main_database(self.database_path).read() as connection:
                 row = connection.execute(
-                    "SELECT authority FROM memory_authority_state WHERE singleton=1"
+                    "SELECT authority,migration_id FROM memory_authority_state WHERE singleton=1"
                 ).fetchone()
         except sqlite3.OperationalError as error:
             if "no such table" not in str(error).lower():
                 raise
         else:
             if row is not None:
-                return MemoryAuthority(str(row[0]))
-        return MemoryAuthority.SDK if self.cutover() is not None else MemoryAuthority.LEGACY
+                return MemoryAuthority(str(row[0])), (str(row[1]) if row[1] else None)
+        marker = self.cutover()
+        if marker is not None:
+            return MemoryAuthority.SDK, marker.migration_id
+        return MemoryAuthority.LEGACY, None
 
     def cutover(self) -> CutoverMarker | None:
         try:
@@ -136,37 +142,14 @@ class SdkDataPlaneMigrationStore:
                     """,
                     values,
                 )
-            else:
-                identity = (
-                    source_runtime_build,
-                    target_sdk_version,
-                    source_schema_version,
-                    namespace_fingerprint,
-                    backend_fingerprint,
+            elif backup_bundle_id and existing["backup_bundle_id"] is None:
+                connection.execute(
+                    "UPDATE sdk_data_plane_migrations "
+                    "SET backup_bundle_id=?,updated_at=? WHERE migration_id=?",
+                    (backup_bundle_id, now, migration_id),
                 )
-                existing_identity = tuple(
-                    existing[key]
-                    for key in (
-                        "source_runtime_build",
-                        "target_sdk_version",
-                        "source_schema_version",
-                        "namespace_fingerprint",
-                        "backend_fingerprint",
-                    )
-                )
-                if existing_identity != identity:
-                    raise DataPlaneMigrationError("sdk_data_plane_migration_identity_conflict")
-                if backup_bundle_id and existing["backup_bundle_id"] not in {
-                    None,
-                    backup_bundle_id,
-                }:
-                    raise DataPlaneMigrationError("sdk_data_plane_migration_bundle_conflict")
-                if backup_bundle_id and existing["backup_bundle_id"] is None:
-                    connection.execute(
-                        "UPDATE sdk_data_plane_migrations "
-                        "SET backup_bundle_id=?,updated_at=? WHERE migration_id=?",
-                        (backup_bundle_id, now, migration_id),
-                    )
+            elif backup_bundle_id and existing["backup_bundle_id"] != backup_bundle_id:
+                raise DataPlaneMigrationError("sdk_data_plane_migration_bundle_conflict")
             connection.execute("COMMIT")
         return self.progress(migration_id)
 
@@ -182,211 +165,18 @@ class SdkDataPlaneMigrationStore:
                 raise DataPlaneMigrationError("sdk_data_plane_migration_not_mutable")
             connection.execute("COMMIT")
 
-    def upsert_item(
-        self,
-        *,
-        migration_id: str,
-        item_key: str,
-        source_kind: str,
-        source_id: str,
-        source_hash: str,
-        operation_key: str,
-        source_revision: str | None = None,
-        state: str = "pending",
-    ) -> None:
-        now = self._now()
-        with main_database(self.database_path).write(
-            autocommit=True, row_factory=sqlite3.Row
-        ) as connection:
-            connection.execute("BEGIN IMMEDIATE")
-            row = connection.execute(
-                "SELECT * FROM sdk_data_plane_migration_items WHERE migration_id=? AND item_key=?",
-                (migration_id, item_key),
-            ).fetchone()
-            if row is None:
-                try:
-                    connection.execute(
-                        """
-                        INSERT INTO sdk_data_plane_migration_items (
-                            migration_id,item_key,source_kind,source_id,
-                            source_revision,source_hash,operation_key,state,updated_at
-                        ) VALUES (?,?,?,?,?,?,?,?,?)
-                        """,
-                        (
-                            migration_id,
-                            item_key,
-                            source_kind,
-                            source_id,
-                            source_revision,
-                            source_hash,
-                            operation_key,
-                            state,
-                            now,
-                        ),
-                    )
-                except sqlite3.IntegrityError as error:
-                    raise DataPlaneMigrationError(
-                        "sdk_data_plane_migration_operation_conflict"
-                    ) from error
-            else:
-                if str(row["state"]) in TERMINAL_ITEM_STATES and (
-                    source_id,
-                    source_revision,
-                    source_hash,
-                ) == (
-                    row["source_id"],
-                    row["source_revision"],
-                    row["source_hash"],
-                ):
-                    # A crash after an SDK side effect can make the next
-                    # inventory classify the same source as verified/adopted
-                    # instead of import/delete. Keep the settled operation as
-                    # the durable audit identity.
-                    connection.execute("COMMIT")
-                    return
-                identity = (
-                    source_kind,
-                    source_id,
-                    source_revision,
-                    source_hash,
-                    operation_key,
-                )
-                if identity != tuple(
-                    row[key]
-                    for key in (
-                        "source_kind",
-                        "source_id",
-                        "source_revision",
-                        "source_hash",
-                        "operation_key",
-                    )
-                ):
-                    raise DataPlaneMigrationError("sdk_data_plane_migration_item_conflict")
-            connection.execute("COMMIT")
-
-    def settle_item(
-        self,
-        *,
-        migration_id: str,
-        item_key: str,
-        state: str,
-        target_memory_id: str | None = None,
-        target_hash: str | None = None,
-        error_code: str | None = None,
-    ) -> None:
-        with main_database(self.database_path).write(autocommit=True) as connection:
-            connection.execute("BEGIN IMMEDIATE")
-            cursor = connection.execute(
-                """
-                UPDATE sdk_data_plane_migration_items
-                SET state=?,target_memory_id=?,target_hash=?,error_code=?,
-                    attempts=attempts+1,updated_at=?
-                WHERE migration_id=? AND item_key=?
-                """,
-                (
-                    state,
-                    target_memory_id,
-                    target_hash,
-                    error_code,
-                    self._now(),
-                    migration_id,
-                    item_key,
-                ),
-            )
-            if cursor.rowcount != 1:
-                raise DataPlaneMigrationError("sdk_data_plane_migration_item_not_found")
-            connection.execute("COMMIT")
-
-    def record_alias(
-        self,
-        *,
-        migration_id: str,
-        legacy_memory_id: str,
-        canonical_memory_id: str,
-    ) -> None:
-        with main_database(self.database_path).write(
-            autocommit=True, row_factory=sqlite3.Row
-        ) as connection:
-            connection.execute("BEGIN IMMEDIATE")
-            existing = connection.execute(
-                "SELECT canonical_memory_id,migration_id FROM memory_id_aliases "
-                "WHERE legacy_memory_id=?",
-                (legacy_memory_id,),
-            ).fetchone()
-            if existing is None:
-                connection.execute(
-                    """
-                    INSERT INTO memory_id_aliases (
-                        legacy_memory_id,canonical_memory_id,migration_id,created_at
-                    ) VALUES (?,?,?,?)
-                    """,
-                    (
-                        legacy_memory_id,
-                        canonical_memory_id,
-                        migration_id,
-                        self._now(),
-                    ),
-                )
-            elif existing["canonical_memory_id"] != canonical_memory_id:
-                raise DataPlaneMigrationError("sdk_data_plane_alias_conflict")
-            connection.execute("COMMIT")
-
-    def record_raw_alias(
-        self,
-        *,
-        migration_id: str,
-        legacy_evidence_id: str,
-        canonical_raw_id: str,
-        item_index: int = 0,
-    ) -> None:
-        with main_database(self.database_path).write(
-            autocommit=True,
-            row_factory=sqlite3.Row,
-        ) as connection:
-            connection.execute("BEGIN IMMEDIATE")
-            existing = connection.execute(
-                "SELECT canonical_raw_id FROM raw_id_aliases "
-                "WHERE legacy_evidence_id=? AND item_index=?",
-                (legacy_evidence_id, item_index),
-            ).fetchone()
-            if existing is None:
-                connection.execute(
-                    "INSERT INTO raw_id_aliases "
-                    "(legacy_evidence_id,item_index,canonical_raw_id,migration_id,created_at) "
-                    "VALUES (?,?,?,?,?)",
-                    (
-                        legacy_evidence_id,
-                        item_index,
-                        canonical_raw_id,
-                        migration_id,
-                        self._now(),
-                    ),
-                )
-            elif existing["canonical_raw_id"] != canonical_raw_id:
-                raise DataPlaneMigrationError("sdk_data_plane_raw_alias_conflict")
-            connection.execute("COMMIT")
-
     def progress(self, migration_id: str) -> MigrationProgress:
         with main_database(self.database_path).read(row_factory=sqlite3.Row) as connection:
             migration = connection.execute(
                 "SELECT state,phase FROM sdk_data_plane_migrations WHERE migration_id=?",
                 (migration_id,),
             ).fetchone()
-            if migration is None:
-                raise DataPlaneMigrationError("sdk_data_plane_migration_not_found")
-            counts = {
-                str(row["state"]): int(row["count"])
-                for row in connection.execute(
-                    "SELECT state,COUNT(*) AS count "
-                    "FROM sdk_data_plane_migration_items WHERE migration_id=? GROUP BY state",
-                    (migration_id,),
-                )
-            }
+        if migration is None:
+            raise DataPlaneMigrationError("sdk_data_plane_migration_not_found")
         return MigrationProgress(
             migration_id=migration_id,
             state=str(migration["state"]),
             phase=str(migration["phase"]),
-            counts=counts,
         )
 
     def commit_cutover(
@@ -409,23 +199,8 @@ class SdkDataPlaneMigrationStore:
             ).fetchone()
             if migration is None:
                 raise DataPlaneMigrationError("sdk_data_plane_migration_not_found")
-            if migration["phase"] != "reconciled" or migration["state"] != "ready":
+            if migration["phase"] != "prepared" or migration["state"] != "ready":
                 raise DataPlaneMigrationError("sdk_data_plane_migration_not_ready")
-            unresolved = connection.execute(
-                "SELECT state,COUNT(*) AS count FROM sdk_data_plane_migration_items "
-                "WHERE migration_id=? GROUP BY state",
-                (migration_id,),
-            ).fetchall()
-            blocked = {
-                str(row["state"]): int(row["count"])
-                for row in unresolved
-                if str(row["state"]) not in TERMINAL_ITEM_STATES
-            }
-            if blocked:
-                raise DataPlaneMigrationError(
-                    "sdk_data_plane_migration_items_unresolved:"
-                    + json.dumps(blocked, sort_keys=True, separators=(",", ":"))
-                )
             if migration["backup_bundle_id"] != backup_bundle_id:
                 raise DataPlaneMigrationError("sdk_data_plane_migration_bundle_conflict")
             existing = connection.execute(

memhub/apps/core/src/one_memory/sdk_memory_runtime.py

@@ -3,6 +3,9 @@
 from __future__ import annotations
 
 import asyncio
+import logging
+import re
+from contextlib import suppress
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
@@ -25,6 +28,21 @@ from one_memory.memory_server.durable_queue import COMPLETED, DurableWriteQueue
 from one_memory.sdk_write_processor import SdkWriteCoordinator
 from one_memory.semantic_deadline import SemanticMemoryHit
 
+# Only a code-shaped message is a code; anything else may carry provider text.
+_STABLE_CODE = re.compile(r"[a-z][a-z0-9_]{0,95}")
+
+logger = logging.getLogger(__name__)
+RAW_RETENTION_DAYS = 3
+RAW_RETENTION_CHECK_SECONDS = 6 * 60 * 60
+_RAW_STATUSES = (
+    "active",
+    "superseded",
+    "shadow",
+    "negated",
+    "conflicted",
+    "archived",
+)
+
 
 class SdkMemoryRuntime:
     """Product services that remain after legacy task/governance writes retire."""
@@ -43,6 +61,68 @@ class SdkMemoryRuntime:
         self.write_queue = write_queue
         self.extraction_settings = extraction_settings
         self.data_dir = data_dir
+        self._raw_retention_task: asyncio.Task[None] | None = None
+
+    def start_raw_retention(self) -> None:
+        if self._raw_retention_task is None or self._raw_retention_task.done():
+            self._raw_retention_task = asyncio.create_task(
+                self._raw_retention_loop(),
+                name="one-memory-sdk-raw-retention",
+            )
+
+    async def prune_expired_raw(self, *, now: datetime | None = None) -> int:
+        cutoff = (now or datetime.now(UTC)) - timedelta(days=RAW_RETENTION_DAYS)
+        cutoff_timestamp = int(cutoff.timestamp())
+        client = await self.read_handle.client()
+        deleted = 0
+        while True:
+            page = await asyncio.to_thread(
+                client.list_memories,
+                user_id="local",
+                layers=["l1_raw"],
+                statuses=list(_RAW_STATUSES),
+                order="asc",
+                limit=500,
+                offset=0,
+            )
+            bucket = page.get("vdb") if isinstance(page, dict) else None
+            rows = bucket.get("memories") if isinstance(bucket, dict) else None
+            if not isinstance(rows, list) or not rows:
+                return deleted
+            expired = [
+                row
+                for row in rows
+                if isinstance(row, dict)
+                and isinstance(row.get("memory_id"), str)
+                and int(row.get("memory_at") or row.get("gmt_created") or 0)
+                < cutoff_timestamp
+            ]
+            if not expired:
+                return deleted
+            removed_page = 0
+            for row in expired:
+                result = await asyncio.to_thread(client.delete, row["memory_id"])
+                if isinstance(result, dict) and result.get("success") is True:
+                    count = int(result.get("deleted_count") or 0)
+                    deleted += count
+                    removed_page += count
+            if removed_page == 0:
+                return deleted
+
+    async def _raw_retention_loop(self) -> None:
+        while True:
+            await asyncio.sleep(RAW_RETENTION_CHECK_SECONDS)
+            try:
+                removed = await self.prune_expired_raw()
+                if removed:
+                    logger.info("sdk_raw_retention_deleted count=%d", removed)
+            except asyncio.CancelledError:
+                raise
+            except Exception as error:
+                logger.warning(
+                    "sdk_raw_retention_failed error_type=%s",
+                    type(error).__name__,
+                )
 
     async def semantic_recall(
         self,
@@ -160,8 +240,22 @@ class SdkMemoryRuntime:
             decision=decision,
         )
 
+    async def _mutation_client(self) -> Any:
+        """The write client, with no-model and keychain refusals as SDK codes.
+
+        Callers record these against the edit or deletion instead of failing
+        the request that already committed it.
+        """
+        try:
+            return await self.coordinator.client()
+        except RuntimeError as error:
+            code = str(error)
+            if not _STABLE_CODE.fullmatch(code):
+                code = "hy_memory_client_unavailable"
+            raise HySdkError(code) from error
+
     async def update_hy_memory(self, memory_id: str, content: str) -> dict[str, object]:
-        client = await self.coordinator.client()
+        client = await self._mutation_client()
         try:
             result = await asyncio.to_thread(client.update, memory_id, content)
         except Exception as error:
@@ -171,7 +265,7 @@ class SdkMemoryRuntime:
         return result
 
     async def finish_deletion(self, memory_id: str) -> None:
-        client = await self.coordinator.client()
+        client = await self._mutation_client()
         try:
             result = await asyncio.to_thread(client.delete, memory_id)
         except Exception as error:
@@ -279,7 +373,12 @@ class SdkMemoryRuntime:
         return None
 
     async def shutdown(self) -> None:
-        return None
+        if self._raw_retention_task is None:
+            return
+        self._raw_retention_task.cancel()
+        with suppress(asyncio.CancelledError):
+            await self._raw_retention_task
+        self._raw_retention_task = None
 
 
 def _agent_ids(raw: object) -> tuple[str, ...]:

memhub/apps/core/src/one_memory/sdk_migration_inventory.py

@@ -1,684 +0,0 @@
-"""Legacy Governance to SDK inventory classification for cutover planning."""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import sqlite3
-from collections.abc import Awaitable, Mapping
-from dataclasses import dataclass
-from datetime import datetime
-from pathlib import Path
-from typing import Protocol
-
-from one_memory.database_access import main_database
-from one_memory.hy_sdk_types import HySdkMemoryPage, HySdkRawPage
-from one_memory.sdk_data_plane_migration import SdkDataPlaneMigrationStore
-
-
-def content_hash(content: str) -> str:
-    return hashlib.sha256(content.encode("utf-8")).hexdigest()
-
-
-@dataclass(frozen=True)
-class LegacyMemoryInventory:
-    legacy_memory_id: str
-    canonical_memory_id: str
-    user_id: str
-    agent_id: str
-    session_id: str
-    content: str
-    content_hash: str
-    status: str
-    layer: str | None
-    owner: str | None
-    tags: tuple[str, ...]
-    supersedes: tuple[str, ...]
-    superseded_by: tuple[str, ...]
-
-
-@dataclass(frozen=True)
-class SdkMemoryInventory:
-    memory_id: str
-    user_id: str
-    agent_id: str
-    content_hash: str
-    status: str
-
-
-@dataclass(frozen=True)
-class MemoryMigrationDecision:
-    item_key: str
-    action: str
-    legacy_memory_id: str | None
-    canonical_memory_id: str
-    reason: str
-
-
-@dataclass(frozen=True)
-class LegacyRawInventory:
-    evidence_id: str
-    user_id: str
-    agent_id: str
-    session_id: str
-    source: str
-    content: str
-    content_hash: str
-    memory_at: datetime | None
-    metadata: dict[str, object]
-
-
-@dataclass(frozen=True)
-class SdkRawInventory:
-    raw_id: str
-    user_id: str
-    agent_id: str
-    session_id: str
-    source: str
-    content_hash: str
-
-
-@dataclass(frozen=True)
-class RawMigrationDecision:
-    item_key: str
-    action: str
-    evidence_id: str | None
-    raw_id: str
-    reason: str
-
-
-_KIND_LAYER = {
-    "chat": "l2_fact",
-    "schema": "l6_schema",
-    "intention": "l7_intention",
-}
-
-SDK_LIFECYCLE_STATUSES = (
-    "active",
-    "superseded",
-    "shadow",
-    "negated",
-    "conflicted",
-    "archived",
-)
-
-
-class SdkMigrationClient(Protocol):
-    def list_memories(
-        self,
-        *,
-        user_id: str,
-        agent_id: str | None = None,
-        statuses: tuple[str, ...] = ("active",),
-        limit: int = 100,
-        offset: int = 0,
-        order: str = "desc",
-    ) -> Awaitable[HySdkMemoryPage]: ...
-
-    def write_legacy_memory(
-        self,
-        *,
-        content: str,
-        memory_id: str,
-        user_id: str,
-        agent_id: str,
-        session_id: str,
-        layer: str,
-        status: str,
-        owner: str | None,
-        tags: tuple[str, ...],
-        supersedes: tuple[str, ...],
-        superseded_by: tuple[str, ...],
-        is_latest: bool,
-        request_id: str,
-    ) -> Awaitable[dict[str, object]]: ...
-
-    def read_raw_inventory(
-        self,
-        *,
-        user_id: str,
-        agent_id: str | None = None,
-        session_id: str | None = None,
-        source: str | None = None,
-        limit: int = 100,
-        offset: int = 0,
-        order: str = "desc",
-    ) -> Awaitable[HySdkRawPage]: ...
-
-    def write_legacy_raw(
-        self,
-        *,
-        content: str,
-        raw_id: str,
-        user_id: str,
-        agent_id: str,
-        session_id: str,
-        source: str,
-        memory_at: datetime | None,
-        metadata: Mapping[str, object],
-        request_id: str,
-    ) -> Awaitable[dict[str, object]]: ...
-
-    def delete(self, memory_id: str) -> Awaitable[dict[str, object]]: ...
-
-
-def read_legacy_memory_inventory(database_path: Path) -> tuple[LegacyMemoryInventory, ...]:
-    """Read legacy memory intent without constructing a Governance writer."""
-    with main_database(database_path).read(row_factory=sqlite3.Row) as connection:
-        mappings: dict[str, tuple[str, str | None, str | None]] = {}
-        for row in connection.execute("SELECT body FROM memory_task_outputs"):
-            try:
-                body = json.loads(row["body"])
-            except (TypeError, ValueError):
-                continue
-            legacy_id = body.get("memory_id")
-            sdk_id = body.get("hy_memory_id")
-            if isinstance(legacy_id, str) and isinstance(sdk_id, str):
-                mappings[legacy_id] = (
-                    sdk_id,
-                    body.get("layer") if isinstance(body.get("layer"), str) else None,
-                    body.get("owner") if body.get("owner") in {"user", "agent"} else None,
-                )
-
-        supersedes: dict[str, list[str]] = {}
-        superseded_by: dict[str, list[str]] = {}
-        for row in connection.execute(
-            "SELECT memory_id,superseded_by FROM governed_memory_supersessions "
-            "ORDER BY memory_id,superseded_by"
-        ):
-            older, newer = str(row["memory_id"]), str(row["superseded_by"])
-            superseded_by.setdefault(older, []).append(newer)
-            supersedes.setdefault(newer, []).append(older)
-
-        records: list[LegacyMemoryInventory] = []
-        for row in connection.execute(
-            "SELECT memory_id,user_id,kind,status,content,tags_json "
-            "FROM governed_memories ORDER BY memory_id"
-        ):
-            legacy_id = str(row["memory_id"])
-            mapped_id, mapped_layer, mapped_owner = mappings.get(legacy_id, (legacy_id, None, None))
-            try:
-                tags_value = json.loads(row["tags_json"] or "[]")
-            except (TypeError, ValueError):
-                tags_value = []
-            tags = tuple(value for value in tags_value if isinstance(value, str))
-            kind = str(row["kind"])
-            status = str(row["status"])
-            records.append(
-                LegacyMemoryInventory(
-                    legacy_memory_id=legacy_id,
-                    canonical_memory_id=mapped_id,
-                    user_id=str(row["user_id"]),
-                    agent_id="one-memory",
-                    session_id="default_session",
-                    content=str(row["content"]),
-                    content_hash=content_hash(str(row["content"])),
-                    status=(
-                        "active"
-                        if status == "confirmed"
-                        else "superseded"
-                        if status == "superseded"
-                        else "deleted"
-                    ),
-                    layer=mapped_layer or _KIND_LAYER.get(kind),
-                    owner=mapped_owner,
-                    tags=tags,
-                    supersedes=tuple(supersedes.get(legacy_id, ())),
-                    superseded_by=tuple(superseded_by.get(legacy_id, ())),
-                )
-            )
-    return tuple(records)
-
-
-def _parse_datetime(value: object) -> datetime | None:
-    if not isinstance(value, str) or not value:
-        return None
-    try:
-        return datetime.fromisoformat(value)
-    except ValueError:
-        return None
-
-
-def read_legacy_raw_inventory(database_path: Path) -> tuple[LegacyRawInventory, ...]:
-    with main_database(database_path).read(row_factory=sqlite3.Row) as connection:
-        rows = connection.execute(
-            """
-            SELECT evidence_id,user_id,source_id,workspace,project,
-                   conversation_id,message_id,source_ordinal,content_kind,
-                   role,written_by,created_at,imported_at,verbatim,content_hash,revision
-            FROM evidence ORDER BY COALESCE(ingest_seq,rowid),evidence_id
-            """
-        ).fetchall()
-    return tuple(
-        LegacyRawInventory(
-            evidence_id=str(row["evidence_id"]),
-            user_id=str(row["user_id"]),
-            agent_id="one-memory",
-            session_id=str(row["conversation_id"]),
-            source=str(row["source_id"]),
-            content=str(row["verbatim"]),
-            content_hash=str(row["content_hash"]),
-            memory_at=_parse_datetime(row["created_at"] or row["imported_at"]),
-            metadata={
-                key: row[key]
-                for key in (
-                    "workspace",
-                    "project",
-                    "message_id",
-                    "source_ordinal",
-                    "content_kind",
-                    "role",
-                    "written_by",
-                    "revision",
-                )
-                if row[key] is not None
-            },
-        )
-        for row in rows
-    )
-
-
-def classify_memory_inventory(
-    legacy: tuple[LegacyMemoryInventory, ...],
-    sdk: tuple[SdkMemoryInventory, ...],
-) -> tuple[MemoryMigrationDecision, ...]:
-    """Classify every legacy and SDK memory; no item may silently disappear."""
-    sdk_by_id = {item.memory_id: item for item in sdk}
-    claimed_sdk_ids: set[str] = set()
-    decisions: list[MemoryMigrationDecision] = []
-    for item in legacy:
-        current = sdk_by_id.get(item.canonical_memory_id)
-        if current is not None:
-            claimed_sdk_ids.add(current.memory_id)
-        if item.status == "deleted":
-            decisions.append(
-                MemoryMigrationDecision(
-                    item_key=f"memory:{item.legacy_memory_id}",
-                    action="delete" if current is not None else "verified_deleted",
-                    legacy_memory_id=item.legacy_memory_id,
-                    canonical_memory_id=item.canonical_memory_id,
-                    reason="legacy_user_delete",
-                )
-            )
-            continue
-        if item.layer is None:
-            decisions.append(
-                MemoryMigrationDecision(
-                    item_key=f"memory:{item.legacy_memory_id}",
-                    action="quarantine",
-                    legacy_memory_id=item.legacy_memory_id,
-                    canonical_memory_id=item.canonical_memory_id,
-                    reason="legacy_layer_unknown",
-                )
-            )
-            continue
-        if current is None:
-            decisions.append(
-                MemoryMigrationDecision(
-                    item_key=f"memory:{item.legacy_memory_id}",
-                    action="import_exact",
-                    legacy_memory_id=item.legacy_memory_id,
-                    canonical_memory_id=item.canonical_memory_id,
-                    reason="governance_only",
-                )
-            )
-            continue
-        if (
-            current.content_hash == item.content_hash
-            and current.status == item.status
-            and current.user_id == item.user_id
-            and current.agent_id == item.agent_id
-        ):
-            action = (
-                "matched" if item.legacy_memory_id == item.canonical_memory_id else "record_alias"
-            )
-            reason = "same_identity" if action == "matched" else "mapped_identity"
-        else:
-            action, reason = "quarantine", "content_owner_or_lifecycle_diverged"
-        decisions.append(
-            MemoryMigrationDecision(
-                item_key=f"memory:{item.legacy_memory_id}",
-                action=action,
-                legacy_memory_id=item.legacy_memory_id,
-                canonical_memory_id=item.canonical_memory_id,
-                reason=reason,
-            )
-        )
-
-    for sdk_item in sdk:
-        if sdk_item.memory_id not in claimed_sdk_ids:
-            decisions.append(
-                MemoryMigrationDecision(
-                    item_key=f"sdk:{sdk_item.memory_id}",
-                    action="adopt",
-                    legacy_memory_id=None,
-                    canonical_memory_id=sdk_item.memory_id,
-                    reason="sdk_only",
-                )
-            )
-    return tuple(decisions)
-
-
-async def read_sdk_memory_inventory(
-    client: SdkMigrationClient,
-    *,
-    user_id: str = "local",
-    page_size: int = 500,
-) -> tuple[SdkMemoryInventory, ...]:
-    records: dict[str, SdkMemoryInventory] = {}
-    offset = 0
-    while True:
-        page = await client.list_memories(
-            user_id=user_id,
-            statuses=SDK_LIFECYCLE_STATUSES,
-            limit=page_size,
-            offset=offset,
-            order="asc",
-        )
-        for memory in page.memories:
-            records[memory.memory_id] = SdkMemoryInventory(
-                memory_id=memory.memory_id,
-                user_id=memory.user_id,
-                agent_id=memory.agent_id,
-                content_hash=content_hash(memory.content),
-                status=memory.status,
-            )
-        consumed = len(page.memories)
-        offset += consumed
-        if consumed == 0 or offset >= page.total:
-            break
-    return tuple(records[key] for key in sorted(records))
-
-
-def record_memory_migration_plan(
-    store: SdkDataPlaneMigrationStore,
-    *,
-    migration_id: str,
-    legacy: tuple[LegacyMemoryInventory, ...],
-    sdk: tuple[SdkMemoryInventory, ...],
-) -> tuple[MemoryMigrationDecision, ...]:
-    decisions = classify_memory_inventory(legacy, sdk)
-    legacy_by_id = {item.legacy_memory_id: item for item in legacy}
-    sdk_by_id = {item.memory_id: item for item in sdk}
-    for decision in decisions:
-        source_hash = (
-            legacy_by_id[decision.legacy_memory_id].content_hash
-            if decision.legacy_memory_id is not None
-            else sdk_by_id[decision.canonical_memory_id].content_hash
-        )
-        operation_key = hashlib.sha256(
-            f"{migration_id}:{decision.item_key}:{decision.action}:{source_hash}".encode()
-        ).hexdigest()
-        store.upsert_item(
-            migration_id=migration_id,
-            item_key=decision.item_key,
-            source_kind=f"memory:{decision.action}",
-            source_id=(
-                decision.legacy_memory_id
-                if decision.legacy_memory_id is not None
-                else decision.canonical_memory_id
-            ),
-            source_hash=source_hash,
-            operation_key=operation_key,
-        )
-    return decisions
-
-
-async def execute_memory_migration_plan(
-    client: SdkMigrationClient,
-    store: SdkDataPlaneMigrationStore,
-    *,
-    migration_id: str,
-    legacy: tuple[LegacyMemoryInventory, ...],
-    decisions: tuple[MemoryMigrationDecision, ...],
-) -> None:
-    legacy_by_id = {item.legacy_memory_id: item for item in legacy}
-    for decision in decisions:
-        item = (
-            legacy_by_id.get(decision.legacy_memory_id)
-            if decision.legacy_memory_id is not None
-            else None
-        )
-        if decision.action == "quarantine":
-            store.settle_item(
-                migration_id=migration_id,
-                item_key=decision.item_key,
-                state="quarantine",
-                error_code=decision.reason,
-            )
-            continue
-        if decision.action == "adopt":
-            store.settle_item(
-                migration_id=migration_id,
-                item_key=decision.item_key,
-                state="adopted",
-                target_memory_id=decision.canonical_memory_id,
-            )
-            continue
-        if decision.action in {"delete", "verified_deleted"}:
-            if decision.action == "delete":
-                await client.delete(decision.canonical_memory_id)
-            store.settle_item(
-                migration_id=migration_id,
-                item_key=decision.item_key,
-                state="deleted",
-                target_memory_id=decision.canonical_memory_id,
-            )
-            continue
-        if item is None:
-            raise RuntimeError(f"missing legacy inventory for {decision.item_key}")
-        if decision.action == "import_exact":
-            assert item.layer is not None
-            request_id = hashlib.sha256(
-                f"{migration_id}:{decision.item_key}:import_exact".encode()
-            ).hexdigest()
-            await client.write_legacy_memory(
-                content=item.content,
-                memory_id=item.canonical_memory_id,
-                user_id=item.user_id,
-                agent_id=item.agent_id,
-                session_id=item.session_id,
-                source="legacy-governance",
-                layer=item.layer,
-                status=item.status,
-                owner=item.owner,
-                tags=item.tags,
-                supersedes=item.supersedes,
-                superseded_by=item.superseded_by,
-                is_latest=item.status == "active",
-                memory_at=None,
-                metadata={},
-                request_id=request_id,
-            )
-        if decision.action == "record_alias":
-            store.record_alias(
-                migration_id=migration_id,
-                legacy_memory_id=item.legacy_memory_id,
-                canonical_memory_id=item.canonical_memory_id,
-            )
-        store.settle_item(
-            migration_id=migration_id,
-            item_key=decision.item_key,
-            state="verified",
-            target_memory_id=decision.canonical_memory_id,
-            target_hash=item.content_hash,
-        )
-
-
-async def read_sdk_raw_inventory(
-    client: SdkMigrationClient,
-    *,
-    user_id: str = "local",
-    page_size: int = 500,
-) -> tuple[SdkRawInventory, ...]:
-    records: dict[str, SdkRawInventory] = {}
-    offset = 0
-    while True:
-        page = await client.read_raw_inventory(
-            user_id=user_id,
-            limit=page_size,
-            offset=offset,
-            order="asc",
-        )
-        for raw in page.raw:
-            raw_id = str(raw.raw_id)
-            records[raw_id] = SdkRawInventory(
-                raw_id=raw_id,
-                user_id=str(raw.user_id),
-                agent_id=str(raw.agent_id),
-                session_id=str(raw.session_id),
-                source=str(raw.source),
-                content_hash=content_hash(str(raw.content)),
-            )
-        consumed = len(page.raw)
-        offset += consumed
-        if consumed == 0 or offset >= page.total:
-            break
-    return tuple(records[key] for key in sorted(records))
-
-
-def classify_raw_inventory(
-    legacy: tuple[LegacyRawInventory, ...],
-    sdk: tuple[SdkRawInventory, ...],
-) -> tuple[RawMigrationDecision, ...]:
-    sdk_by_identity: dict[tuple[str, str, str, str, str], list[SdkRawInventory]] = {}
-    for raw in sdk:
-        identity = (
-            raw.user_id,
-            raw.agent_id,
-            raw.session_id,
-            raw.source,
-            raw.content_hash,
-        )
-        sdk_by_identity.setdefault(identity, []).append(raw)
-    claimed: set[str] = set()
-    decisions: list[RawMigrationDecision] = []
-    for item in legacy:
-        identity = (
-            item.user_id,
-            item.agent_id,
-            item.session_id,
-            item.source,
-            item.content_hash,
-        )
-        candidates = sorted(
-            sdk_by_identity.get(identity, ()),
-            key=lambda candidate: candidate.raw_id,
-        )
-        existing = next(
-            (candidate for candidate in candidates if candidate.raw_id not in claimed),
-            None,
-        )
-        if existing is not None:
-            claimed.add(existing.raw_id)
-            decisions.append(
-                RawMigrationDecision(
-                    item_key=f"raw:{item.evidence_id}",
-                    action="matched",
-                    evidence_id=item.evidence_id,
-                    raw_id=existing.raw_id,
-                    reason="same_content_and_scope",
-                )
-            )
-        else:
-            decisions.append(
-                RawMigrationDecision(
-                    item_key=f"raw:{item.evidence_id}",
-                    action="import_exact",
-                    evidence_id=item.evidence_id,
-                    raw_id=f"evidence:{item.evidence_id}",
-                    reason="evidence_only",
-                )
-            )
-    for raw in sdk:
-        if raw.raw_id not in claimed:
-            decisions.append(
-                RawMigrationDecision(
-                    item_key=f"sdk-raw:{raw.raw_id}",
-                    action="adopt",
-                    evidence_id=None,
-                    raw_id=raw.raw_id,
-                    reason="sdk_only",
-                )
-            )
-    return tuple(decisions)
-
-
-def record_raw_migration_plan(
-    store: SdkDataPlaneMigrationStore,
-    *,
-    migration_id: str,
-    legacy: tuple[LegacyRawInventory, ...],
-    sdk: tuple[SdkRawInventory, ...],
-) -> tuple[RawMigrationDecision, ...]:
-    decisions = classify_raw_inventory(legacy, sdk)
-    legacy_by_id = {item.evidence_id: item for item in legacy}
-    sdk_by_id = {item.raw_id: item for item in sdk}
-    for decision in decisions:
-        source_hash = (
-            legacy_by_id[decision.evidence_id].content_hash
-            if decision.evidence_id is not None
-            else sdk_by_id[decision.raw_id].content_hash
-        )
-        operation_key = hashlib.sha256(
-            f"{migration_id}:{decision.item_key}:{decision.action}:{source_hash}".encode()
-        ).hexdigest()
-        store.upsert_item(
-            migration_id=migration_id,
-            item_key=decision.item_key,
-            source_kind=f"raw:{decision.action}",
-            source_id=decision.evidence_id or decision.raw_id,
-            source_hash=source_hash,
-            operation_key=operation_key,
-        )
-    return decisions
-
-
-async def execute_raw_migration_plan(
-    client: SdkMigrationClient,
-    store: SdkDataPlaneMigrationStore,
-    *,
-    migration_id: str,
-    legacy: tuple[LegacyRawInventory, ...],
-    decisions: tuple[RawMigrationDecision, ...],
-) -> None:
-    legacy_by_id = {item.evidence_id: item for item in legacy}
-    for decision in decisions:
-        if decision.action == "adopt":
-            store.settle_item(
-                migration_id=migration_id,
-                item_key=decision.item_key,
-                state="adopted",
-                target_memory_id=decision.raw_id,
-            )
-            continue
-        item = legacy_by_id.get(decision.evidence_id or "")
-        if item is None:
-            raise RuntimeError(f"missing legacy raw inventory for {decision.item_key}")
-        if decision.action == "import_exact":
-            request_id = hashlib.sha256(
-                f"{migration_id}:{decision.item_key}:write_legacy_raw".encode()
-            ).hexdigest()
-            await client.write_legacy_raw(
-                content=item.content,
-                raw_id=decision.raw_id,
-                user_id=item.user_id,
-                agent_id=item.agent_id,
-                session_id=item.session_id,
-                source=item.source,
-                memory_at=item.memory_at,
-                metadata=item.metadata,
-                request_id=request_id,
-            )
-        store.settle_item(
-            migration_id=migration_id,
-            item_key=decision.item_key,
-            state="verified",
-            target_memory_id=decision.raw_id,
-            target_hash=item.content_hash,
-        )
-        store.record_raw_alias(
-            migration_id=migration_id,
-            legacy_evidence_id=item.evidence_id,
-            canonical_raw_id=decision.raw_id,
-        )

memhub/apps/core/src/one_memory/sdk_migration_runner.py

@@ -1,210 +1,153 @@
-"""Offline, resumable backfill runner for SDK memory authority."""
+"""Offline startup cutover from the legacy Governance mirror to SDK authority.
+
+Every released build wrote memory and raw turns through the SDK first; the
+Evidence/Governance tables only mirrored them. The cutover therefore copies no
+memory. It makes the surviving write queue independent of the legacy tables,
+commits SDK authority and drops those tables.
+"""
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
 import json
+import os
 import shutil
 import sqlite3
-from collections.abc import Callable
-from contextlib import suppress
+from collections.abc import Callable, Iterator
+from contextlib import AbstractContextManager, contextmanager
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Any
 
 from one_memory.database_access import main_database
+from one_memory.imports.models import EvidenceRecord
 from one_memory.imports.storage import EvidenceStore
 from one_memory.memory_server.contracts import (
     BatchIdentity,
     WriteRequest,
     internal_session_id_for,
 )
-from one_memory.memory_server.durable_queue import DurableWriteQueue
+from one_memory.memory_server.durable_queue import (
+    CANCELLED,
+    COMPLETED,
+    FAILED,
+    REAPED_PAYLOAD_MARKER,
+    DurableWriteQueue,
+)
+from one_memory.memory_server.unified_entry import UnifiedWriteIntake
+from one_memory.model_profiles import ModelProfileSnapshot
 from one_memory.sdk_data_plane_migration import (
     CutoverMarker,
     DataPlaneMigrationError,
-    MigrationProgress,
+    MemoryAuthority,
     SdkDataPlaneMigrationStore,
 )
 from one_memory.sdk_migration_bundle import inspect_sdk_migration_bundle
-from one_memory.sdk_migration_inventory import (
-    SdkMigrationClient,
-    execute_memory_migration_plan,
-    execute_raw_migration_plan,
-    read_legacy_memory_inventory,
-    read_legacy_raw_inventory,
-    read_sdk_memory_inventory,
-    read_sdk_raw_inventory,
-    record_memory_migration_plan,
-    record_raw_migration_plan,
+from one_memory.sdk_migration_verify import (
+    MemoryReadbackFailed,
+    ReadbackReport,
+    legacy_memory_ids_from,
+    legacy_memory_ids_from_bundle,
+    read_active_profile,
+    runtime_sdk_client,
+    verify_readback,
 )
-from one_memory.target_schema import drop_legacy_tables
+from one_memory.sdk_vector_reattach import (
+    ReattachedCollection,
+    legacy_vector_collections,
+    reattach_legacy_vectors,
+)
+from one_memory.target_schema import drop_legacy_tables, legacy_tables_present
+
+PREPARED_PHASE = "prepared"
 
 
 @dataclass(frozen=True)
-class BackfillResult:
-    migration_id: str
-    state: str
-    phase: str
-    counts: dict[str, int]
-    memory_items: int
-    raw_items: int
+class CutoverPreparation:
+    """What the cutover did to legacy work, for the receipt and the dialog."""
+
+    frozen_jobs: int
+    moved_tasks: int
+    moved_dispatches: int
+    abandoned: dict[str, int]
+
+    def as_document(self) -> dict[str, object]:
+        return {
+            "frozen_jobs": self.frozen_jobs,
+            "moved_tasks": self.moved_tasks,
+            "moved_dispatches": self.moved_dispatches,
+            "abandoned": dict(self.abandoned),
+        }
 
 
-@dataclass(frozen=True)
-class SdkMigrationStaging:
-    migration_id: str
-    staging_home: Path
-    live_root: Path
-    staging_root: Path
-    protective_root: Path
-    state_file: Path
+def _tables(database: sqlite3.Connection) -> set[str]:
+    return {
+        str(row[0])
+        for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    }
 
 
-def sdk_migration_staging(
-    *,
-    data_dir: Path,
-    migration_id: str,
-    namespace: str,
-) -> SdkMigrationStaging:
-    """Return same-filesystem paths without trusting the migration ID as a path."""
-    token = hashlib.sha256(migration_id.encode()).hexdigest()[:16]
+def recover_abandoned_staging(*, data_dir: Path, database_path: Path, namespace: str) -> None:
+    """Undo or discard the SDK staging copy that builds through 19 created.
+
+    Those builds staged a full SDK copy and swapped directories at cutover. The
+    live root was untouched until the swap, so before SDK authority the
+    protective root is the original store; after it, the protective root is
+    stale.
+    """
     home = data_dir.resolve()
-    staging_home = home / ".sdk-migration" / token
     live_root = home / "hy-memory" / namespace
-    return SdkMigrationStaging(
-        migration_id=migration_id,
-        staging_home=staging_home,
-        live_root=live_root,
-        staging_root=staging_home / "hy-memory" / namespace,
-        protective_root=live_root.parent / f".{namespace}.pre-migration-{token}",
-        state_file=staging_home / "swap-state.json",
-    )
-
-
-def _write_staging_state(paths: SdkMigrationStaging, phase: str) -> None:
-    paths.staging_home.mkdir(parents=True, exist_ok=True)
-    temporary = paths.state_file.with_suffix(".tmp")
-    temporary.write_text(
-        json.dumps(
-            {"schema_version": 1, "migration_id": paths.migration_id, "phase": phase},
-            sort_keys=True,
-        ),
-        encoding="utf-8",
-    )
-    temporary.replace(paths.state_file)
-
-
-def recover_sdk_migration_swap(
-    *,
-    paths: SdkMigrationStaging,
-    database_path: Path,
-) -> None:
-    """Recover an interrupted directory swap before opening any SDK store."""
-    if not paths.protective_root.exists():
-        return
     authority = SdkDataPlaneMigrationStore(database_path).authority()
-    if authority.value == "sdk":
-        if not paths.live_root.is_dir():
-            raise DataPlaneMigrationError("sdk_migration_committed_store_missing")
-        shutil.rmtree(paths.protective_root)
-        shutil.rmtree(paths.staging_home, ignore_errors=True)
-        return
-    if paths.live_root.exists():
-        if paths.staging_root.exists():
-            raise DataPlaneMigrationError("sdk_migration_swap_state_ambiguous")
-        paths.staging_root.parent.mkdir(parents=True, exist_ok=True)
-        paths.live_root.replace(paths.staging_root)
-    paths.protective_root.replace(paths.live_root)
-    _write_staging_state(paths, "recovered")
+    protective_roots = sorted(live_root.parent.glob(f".{namespace}.pre-migration-*"))
+    if authority.value != "sdk" and len(protective_roots) > 1:
+        raise DataPlaneMigrationError("sdk_migration_swap_state_ambiguous")
+    for protective_root in protective_roots:
+        if authority.value == "sdk":
+            if not live_root.is_dir():
+                raise DataPlaneMigrationError("sdk_migration_committed_store_missing")
+            shutil.rmtree(protective_root)
+            continue
+        if live_root.exists():
+            shutil.rmtree(live_root)
+        protective_root.replace(live_root)
+    shutil.rmtree(home / ".sdk-migration", ignore_errors=True)
 
 
-def prepare_sdk_migration_staging(
-    *,
-    paths: SdkMigrationStaging,
-    database_path: Path,
-) -> Path:
-    """Build a private copy; incomplete retries restart from the unchanged live root."""
-    recover_sdk_migration_swap(paths=paths, database_path=database_path)
-    migration_store = SdkDataPlaneMigrationStore(database_path)
-    try:
-        progress = migration_store.progress(paths.migration_id)
-        reuse_ready = progress.state == "ready" and progress.phase == "reconciled"
-    except Exception:
-        reuse_ready = False
-    if paths.staging_root.exists():
-        if paths.staging_root.is_symlink() or not paths.staging_root.is_dir():
-            raise DataPlaneMigrationError("sdk_migration_staging_invalid")
-        if reuse_ready:
-            return paths.staging_home
-        shutil.rmtree(paths.staging_root)
-    paths.live_root.mkdir(parents=True, exist_ok=True)
-    from one_memory.hy_sdk_migration import snapshot_data_plane
-
-    snapshot_data_plane(
-        source_root=paths.live_root,
-        target_root=paths.staging_root,
-        writers_stopped=True,
+def abandoned_staging_present(*, data_dir: Path, namespace: str) -> bool:
+    home = data_dir.resolve()
+    return (home / ".sdk-migration").exists() or any(
+        (home / "hy-memory").glob(f".{namespace}.pre-migration-*")
     )
-    _write_staging_state(paths, "prepared")
-    return paths.staging_home
-
-
-def activate_sdk_migration_staging(
-    *,
-    paths: SdkMigrationStaging,
-    database_path: Path,
-    commit: Callable[[], CutoverMarker],
-) -> CutoverMarker:
-    """Atomically expose staged SDK files and roll them back if cutover fails."""
-    recover_sdk_migration_swap(paths=paths, database_path=database_path)
-    if not paths.staging_root.is_dir():
-        raise DataPlaneMigrationError("sdk_migration_staging_missing")
-    paths.live_root.mkdir(parents=True, exist_ok=True)
-    _write_staging_state(paths, "activating")
-    paths.live_root.replace(paths.protective_root)
-    try:
-        paths.staging_root.replace(paths.live_root)
-        _write_staging_state(paths, "staging_live")
-        marker = commit()
-    except BaseException:
-        if paths.live_root.exists():
-            paths.staging_root.parent.mkdir(parents=True, exist_ok=True)
-            paths.live_root.replace(paths.staging_root)
-        if paths.protective_root.exists():
-            paths.protective_root.replace(paths.live_root)
-        _write_staging_state(paths, "rolled_back")
-        raise
-    with suppress(OSError):
-        _write_staging_state(paths, "committed")
-    # Authority is already committed; leftover paths are detected by
-    # sdk-migration-status and removed by sdk-cleanup on the next startup.
-    shutil.rmtree(paths.protective_root, ignore_errors=True)
-    shutil.rmtree(paths.staging_home, ignore_errors=True)
-    return marker
 
 
 def prepare_legacy_write_queue_for_cutover(database_path: Path) -> tuple[int, int]:
     """Freeze old jobs and move unstarted task rows into the surviving queue."""
     with main_database(database_path).read(row_factory=sqlite3.Row) as database:
-        tables = {
-            str(row["name"])
-            for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
-        }
+        tables = _tables(database)
         if "evidence" not in tables:
             return (0, 0)
         jobs = (
-            tuple(database.execute("SELECT job_id,payload_json FROM memory_write_jobs"))
+            tuple(
+                database.execute("SELECT job_id,state,payload_json FROM memory_write_jobs")
+            )
             if "memory_write_jobs" in tables
             else ()
         )
     evidence = EvidenceStore(database_path)
     frozen = 0
     for row in jobs:
-        request = WriteRequest.model_validate_json(str(row["payload_json"]))
+        decoded = json.loads(str(row["payload_json"]))
+        if isinstance(decoded, dict) and REAPED_PAYLOAD_MARKER in decoded:
+            # A finished job whose input the queue already released.
+            continue
+        request = WriteRequest.model_validate(decoded)
         if request.input_records:
             continue
         records = tuple(evidence.get(identifier) for identifier in request.evidence_ids)
         if any(record is None for record in records):
+            if str(row["state"]) in {COMPLETED, FAILED, CANCELLED}:
+                continue
             raise DataPlaneMigrationError(f"sdk_data_plane_write_job_input_missing:{row['job_id']}")
         materialized = tuple(record for record in records if record is not None)
         updated = request.model_copy(update={"input_records": materialized})
@@ -277,30 +220,55 @@ def prepare_legacy_write_queue_for_cutover(database_path: Path) -> tuple[int, in
     return (frozen, moved)
 
 
-def sdk_cutover_blockers(database_path: Path) -> dict[str, int]:
-    """Count legacy work that could write or duplicate data after cutover."""
-    blockers: dict[str, int] = {}
+def move_pending_dispatches(database_path: Path) -> int:
+    """Queue Evidence that was waiting for its source file to finish.
+
+    Its source cursor has already moved past these rows, so History catch-up
+    after the cutover would not read them again.
+    """
     with main_database(database_path).read(row_factory=sqlite3.Row) as database:
-        tables = {
-            str(row["name"])
-            for row in database.execute("SELECT name FROM sqlite_master WHERE type='table'")
-        }
+        if not {"evidence", "evidence_pending_memory_dispatch"} <= _tables(database):
+            return 0
+        identifiers = tuple(
+            str(row["evidence_id"])
+            for row in database.execute(
+                "SELECT DISTINCT pending.evidence_id "
+                "FROM evidence_pending_memory_dispatch pending "
+                "JOIN evidence USING(evidence_id) "
+                "ORDER BY COALESCE(evidence.created_at,evidence.imported_at),"
+                "COALESCE(evidence.ingest_seq,evidence.rowid),pending.evidence_id"
+            )
+        )
+    evidence = EvidenceStore(database_path)
+    conversations: dict[tuple[str, str, str, str | None], list[EvidenceRecord]] = {}
+    for identifier in identifiers:
+        record = evidence.get(identifier)
+        if record is None:
+            continue
+        key = (record.user_id, record.source_id, record.conversation_id, record.workspace)
+        conversations.setdefault(key, []).append(record)
+    intake = UnifiedWriteIntake(DurableWriteQueue(database_path))
+    for records in conversations.values():
+        intake.submit_records(tuple(records))
+    return len(conversations)
+
+
+def abandoned_legacy_work(database_path: Path) -> dict[str, int]:
+    """Count legacy work that ends with the cutover instead of blocking it.
+
+    A task that may have reached the SDK is not replayed: the SDK already holds
+    whatever it committed. Pending user deletions are not carried over.
+    """
+    abandoned: dict[str, int] = {}
+    with main_database(database_path).read(row_factory=sqlite3.Row) as database:
+        tables = _tables(database)
         checks = (
             (
-                "memory_tasks",
+                "invoked_tasks",
                 "memory_tasks",
                 "status='unknown' OR (status='running' AND hy_invocation_count > 0)",
             ),
-            (
-                "deletions",
-                "memory_deletions",
-                "status IN ('pending','running')",
-            ),
-            (
-                "pending_dispatch",
-                "evidence_pending_memory_dispatch",
-                "1=1",
-            ),
+            ("deletions", "memory_deletions", "status IN ('pending','running')"),
         )
         for name, table, where in checks:
             if table not in tables:
@@ -309,11 +277,23 @@ def sdk_cutover_blockers(database_path: Path) -> dict[str, int]:
                 database.execute(f"SELECT COUNT(*) FROM {table} WHERE {where}").fetchone()[0]
             )
             if count:
-                blockers[name] = count
-    return blockers
+                abandoned[name] = count
+    return abandoned
 
 
-def commit_reconciled_cutover(
+def prepare_legacy_work_for_cutover(database_path: Path) -> CutoverPreparation:
+    """Make the surviving queue self-contained. Safe to repeat."""
+    frozen, moved = prepare_legacy_write_queue_for_cutover(database_path)
+    dispatches = move_pending_dispatches(database_path)
+    return CutoverPreparation(
+        frozen_jobs=frozen,
+        moved_tasks=moved,
+        moved_dispatches=dispatches,
+        abandoned=abandoned_legacy_work(database_path),
+    )
+
+
+def commit_sdk_cutover(
     *,
     data_dir: Path,
     database_path: Path,
@@ -322,38 +302,25 @@ def commit_reconciled_cutover(
     sdk_version: str,
     store_version: str = "1",
 ) -> CutoverMarker:
-    """Verify backup/work quiescence and atomically commit SDK authority."""
+    """Verify the backup, settle legacy work and commit SDK authority."""
     inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=bundle_id)
-    prepare_legacy_write_queue_for_cutover(database_path)
-    blockers = sdk_cutover_blockers(database_path)
-    if blockers:
-        raise DataPlaneMigrationError(
-            "sdk_data_plane_cutover_work_pending:"
-            + json.dumps(blockers, sort_keys=True, separators=(",", ":"))
-        )
-    with main_database(database_path).read(row_factory=sqlite3.Row) as database:
-        rows = database.execute(
-            "SELECT item_key,state,target_memory_id,target_hash "
-            "FROM sdk_data_plane_migration_items WHERE migration_id=? "
-            "ORDER BY item_key",
-            (migration_id,),
-        ).fetchall()
+    # Repeated here because a CLI-managed Runtime may have run on legacy
+    # authority since `sdk-migrate`.
+    preparation = prepare_legacy_work_for_cutover(database_path)
     digest = hashlib.sha256(
         json.dumps(
-            [dict(row) for row in rows],
+            preparation.as_document(),
             sort_keys=True,
             separators=(",", ":"),
         ).encode()
     ).hexdigest()
-    marker = SdkDataPlaneMigrationStore(database_path).commit_cutover(
+    return SdkDataPlaneMigrationStore(database_path).commit_cutover(
         migration_id=migration_id,
         sdk_version=sdk_version,
         store_version=store_version,
         backup_bundle_id=bundle_id,
         reconciliation_digest=digest,
     )
-    inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=marker.backup_bundle_id)
-    return marker
 
 
 def resume_legacy_cleanup(*, data_dir: Path, database_path: Path) -> tuple[str, ...]:
@@ -367,87 +334,255 @@ def resume_legacy_cleanup(*, data_dir: Path, database_path: Path) -> tuple[str,
             data_dir=data_dir,
             bundle_id=marker.backup_bundle_id,
         )
-        from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
+    from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
 
-        recover_sdk_migration_swap(
-            paths=sdk_migration_staging(
-                data_dir=data_dir,
-                migration_id=marker.migration_id,
-                namespace=HY_MEMORY_NAMESPACE,
-            ),
-            database_path=database_path,
-        )
+    recover_abandoned_staging(
+        data_dir=data_dir,
+        database_path=database_path,
+        namespace=HY_MEMORY_NAMESPACE,
+    )
     return drop_legacy_tables(database_path)
 
 
-async def run_sdk_data_plane_backfill(
+MIGRATION_PREFIX = "sdk-authority-v2"
+CAPTURE_FENCE = ".sdk-migration-capture-fence.json"
+
+
+@dataclass(frozen=True)
+class StartupMigrationResult:
+    """`not_required`, `cutover` (this run switched authority) or `complete`."""
+
+    state: str
+    migration_id: str | None = None
+    bundle_id: str | None = None
+    preparation: CutoverPreparation | None = None
+    dropped_tables: tuple[str, ...] = ()
+    reattached: tuple[ReattachedCollection, ...] = ()
+    readback: ReadbackReport | None = None
+
+    def as_document(self) -> dict[str, object]:
+        document: dict[str, object] = {"schema_version": 1, "state": self.state}
+        if self.migration_id is not None:
+            document["migration_id"] = self.migration_id
+            document["bundle_id"] = self.bundle_id
+        if self.preparation is not None:
+            document.update(self.preparation.as_document())
+        if self.reattached:
+            document["reattached_vectors"] = [
+                {
+                    "collection": item.current,
+                    "legacy_count": item.legacy_count,
+                    "merged_count": item.merged_count,
+                    "final_count": item.final_count,
+                }
+                for item in self.reattached
+            ]
+        if self.readback is not None:
+            document["readback"] = self.readback.as_document()
+        if self.dropped_tables:
+            document["dropped_tables"] = list(self.dropped_tables)
+        return document
+
+
+def database_generation(database_path: Path) -> str:
+    stat = database_path.stat()
+    return hashlib.sha256(f"{stat.st_size}:{stat.st_mtime_ns}".encode()).hexdigest()[:16]
+
+
+def _resumable_identity(database_path: Path) -> tuple[str, str]:
+    """Resume the newest unfinished run, including one build 19 left blocked."""
+    with main_database(database_path).read(row_factory=sqlite3.Row) as database:
+        row = (
+            database.execute(
+                "SELECT migration_id,backup_bundle_id FROM sdk_data_plane_migrations "
+                "WHERE state IN ('running','blocked','ready') AND migration_id LIKE ? "
+                "ORDER BY created_at DESC LIMIT 1",
+                (f"{MIGRATION_PREFIX}-%",),
+            ).fetchone()
+            if "sdk_data_plane_migrations" in _tables(database)
+            else None
+        )
+    if row is not None and row["backup_bundle_id"] == f"{row['migration_id']}-pre-cutover":
+        return str(row["migration_id"]), str(row["backup_bundle_id"])
+    migration_id = f"{MIGRATION_PREFIX}-{database_generation(database_path)}"
+    return migration_id, f"{migration_id}-pre-cutover"
+
+
+@contextmanager
+def capture_fence(data_dir: Path) -> Iterator[None]:
+    """Host capture hooks return silently while this process is alive."""
+    marker = data_dir / CAPTURE_FENCE
+    temporary = marker.with_name(f"{marker.name}.{os.getpid()}.tmp")
+    temporary.write_text(
+        json.dumps({"schema_version": 1, "pid": os.getpid()}) + "\n",
+        encoding="utf-8",
+    )
+    os.chmod(temporary, 0o600)
+    temporary.replace(marker)
+    try:
+        yield
+    finally:
+        marker.unlink(missing_ok=True)
+        temporary.unlink(missing_ok=True)
+
+
+def _ensure_bundle(*, data_dir: Path, bundle_id: str, runtime_build: str) -> dict[str, object]:
+    from one_memory.sdk_migration_bundle import (
+        MigrationBundleRejected,
+        create_sdk_migration_bundle,
+    )
+
+    try:
+        create_sdk_migration_bundle(
+            data_dir=data_dir,
+            bundle_id=bundle_id,
+            runtime_build=runtime_build,
+            runtime_stopped=True,
+        )
+    except MigrationBundleRejected as error:
+        if error.code != "sdk_migration_bundle_exists":
+            raise
+    return inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=bundle_id)
+
+
+def _vacuum(database_path: Path) -> None:
+    """Return the dropped tables' pages to the disk; a full disk only skips it."""
+    try:
+        with main_database(database_path).write(autocommit=True) as database:
+            database.execute("VACUUM")
+    except sqlite3.OperationalError:
+        return
+
+
+def run_startup_migration(
     *,
-    database_path: Path,
-    client: SdkMigrationClient,
-    migration_id: str,
-    bundle_id: str,
-    source_runtime_build: str,
-    target_sdk_version: str,
-    source_schema_version: int,
-    namespace_fingerprint: str,
-    backend_fingerprint: str,
-) -> BackfillResult:
-    """Backfill both lanes but never commit the authority cutover marker."""
+    data_dir: Path,
+    runtime_build: str,
+    sdk_version: str,
+    namespace: str,
+    progress: Callable[[str], None] = lambda _step: None,
+    open_client: Callable[[ModelProfileSnapshot], AbstractContextManager[Any]] | None = None,
+) -> StartupMigrationResult:
+    """Bring one stopped home to SDK authority. Safe to repeat after any failure.
+
+    The caller must stop Runtime and keep its service from restarting.
+    `open_client` yields a public SDK client configured as the Runtime will be;
+    tests replace it.
+    """
+    database_path = data_dir / "one-memory.sqlite3"
+    if not database_path.is_file():
+        return StartupMigrationResult(state="not_required")
+
+    def client_for(profile: ModelProfileSnapshot) -> AbstractContextManager[Any]:
+        if open_client is not None:
+            return open_client(profile)
+        return runtime_sdk_client(database_path, data_dir, profile)
+
     store = SdkDataPlaneMigrationStore(database_path)
-    store.start(
-        migration_id=migration_id,
-        source_runtime_build=source_runtime_build,
-        target_sdk_version=target_sdk_version,
-        source_schema_version=source_schema_version,
-        namespace_fingerprint=namespace_fingerprint,
-        backend_fingerprint=backend_fingerprint,
-        backup_bundle_id=bundle_id,
-    )
-    store.set_phase(migration_id, phase="inventory", state="running")
+    authority, authority_migration = store.authority_state()
+    profile = read_active_profile(database_path)
+    if authority is MemoryAuthority.SDK:
+        vectors_waiting = profile is not None and bool(legacy_vector_collections(data_dir, profile))
+        leftovers = bool(legacy_tables_present(database_path)) or abandoned_staging_present(
+            data_dir=data_dir, namespace=namespace
+        )
+        if not vectors_waiting and not leftovers:
+            return StartupMigrationResult(state="not_required")
+        reattached: tuple[ReattachedCollection, ...] = ()
+        readback: ReadbackReport | None = None
+        if vectors_waiting:
+            assert profile is not None
+            progress("vectors")
+            reattached = reattach_legacy_vectors(data_dir, profile)
+            legacy = legacy_memory_ids_from_bundle(data_dir, authority_migration)
+            if legacy is not None and not legacy.empty:
+                # Authority already moved, so nothing is left to protect by
+                # refusing; record what the repaired home reads back.
+                progress("verify")
+                with client_for(profile) as client:
+                    readback = verify_readback(client, legacy)
+        dropped: tuple[str, ...] = ()
+        if leftovers:
+            progress("cleanup")
+            dropped = resume_legacy_cleanup(data_dir=data_dir, database_path=database_path)
+            if dropped:
+                _vacuum(database_path)
+        return StartupMigrationResult(
+            state="complete",
+            migration_id=authority_migration,
+            bundle_id=f"{authority_migration}-pre-cutover" if authority_migration else None,
+            dropped_tables=dropped,
+            reattached=reattached,
+            readback=readback,
+        )
 
-    legacy_memories = read_legacy_memory_inventory(database_path)
-    sdk_memories = await read_sdk_memory_inventory(client)
-    memory_decisions = record_memory_migration_plan(
-        store,
-        migration_id=migration_id,
-        legacy=legacy_memories,
-        sdk=sdk_memories,
-    )
-    legacy_raw = read_legacy_raw_inventory(database_path)
-    sdk_raw = await read_sdk_raw_inventory(client)
-    raw_decisions = record_raw_migration_plan(
-        store,
-        migration_id=migration_id,
-        legacy=legacy_raw,
-        sdk=sdk_raw,
-    )
+    from one_memory.database_access import close_main_databases
+    from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
+    from one_memory.memory_server.migrations import MIGRATIONS
 
-    store.set_phase(migration_id, phase="backfill", state="running")
-    await execute_memory_migration_plan(
-        client,
-        store,
+    migration_id, bundle_id = _resumable_identity(database_path)
+    legacy = legacy_memory_ids_from(database_path)
+    if profile is None and not legacy.empty:
+        # Without the embedding profile the Runtime's collections cannot be
+        # named, so neither reattached nor read back. Legacy stays intact.
+        raise DataPlaneMigrationError("sdk_migration_model_profile_required")
+    with capture_fence(data_dir):
+        progress("bundle")
+        manifest = _ensure_bundle(
+            data_dir=data_dir,
+            bundle_id=bundle_id,
+            runtime_build=runtime_build,
+        )
+        progress("prepare")
+        asyncio.run(migrate(SqliteMigrationStore(database_path), MIGRATIONS))
+        close_main_databases()
+        recover_abandoned_staging(
+            data_dir=data_dir,
+            database_path=database_path,
+            namespace=namespace,
+        )
+        store.start(
+            migration_id=migration_id,
+            source_runtime_build=str(manifest.get("runtime_build") or ""),
+            target_sdk_version=sdk_version,
+            source_schema_version=MIGRATIONS[-1].version,
+            namespace_fingerprint=hashlib.sha256(namespace.encode()).hexdigest(),
+            backend_fingerprint=hashlib.sha256(b"chroma:kuzu:sqlite").hexdigest(),
+            backup_bundle_id=bundle_id,
+        )
+        preparation = prepare_legacy_work_for_cutover(database_path)
+        reattached = ()
+        readback = None
+        if profile is not None:
+            progress("vectors")
+            reattached = reattach_legacy_vectors(data_dir, profile)
+        if not legacy.empty:
+            assert profile is not None
+            progress("verify")
+            with client_for(profile) as client:
+                readback = verify_readback(client, legacy)
+            if not readback.passed:
+                store.set_phase(migration_id, phase="readback_failed", state="blocked")
+                raise MemoryReadbackFailed(readback)
+        store.set_phase(migration_id, phase=PREPARED_PHASE, state="ready")
+        progress("cutover")
+        commit_sdk_cutover(
+            data_dir=data_dir,
+            database_path=database_path,
+            migration_id=migration_id,
+            bundle_id=bundle_id,
+            sdk_version=sdk_version,
+        )
+        progress("cleanup")
+        dropped = resume_legacy_cleanup(data_dir=data_dir, database_path=database_path)
+        if dropped:
+            _vacuum(database_path)
+    return StartupMigrationResult(
+        state="cutover",
         migration_id=migration_id,
-        legacy=legacy_memories,
-        decisions=memory_decisions,
-    )
-    await execute_raw_migration_plan(
-        client,
-        store,
-        migration_id=migration_id,
-        legacy=legacy_raw,
-        decisions=raw_decisions,
-    )
-    progress: MigrationProgress = store.progress(migration_id)
-    if progress.counts.get("quarantine", 0):
-        store.set_phase(migration_id, phase="review_required", state="blocked")
-    else:
-        store.set_phase(migration_id, phase="reconciled", state="ready")
-    progress = store.progress(migration_id)
-    return BackfillResult(
-        migration_id=migration_id,
-        state=progress.state,
-        phase=progress.phase,
-        counts=progress.counts,
-        memory_items=len(memory_decisions),
-        raw_items=len(raw_decisions),
+        bundle_id=bundle_id,
+        preparation=preparation,
+        dropped_tables=dropped,
+        reattached=reattached,
+        readback=readback,
     )

memhub/apps/core/src/one_memory/sdk_migration_verify.py

@@ -0,0 +1,232 @@
+"""Read legacy memory IDs back through the SDK before (and after) the cutover.
+
+The cutover is only safe if the SDK, configured exactly as the new Runtime
+will configure it, returns the memories the legacy mirror lists. Checking that
+the backup bundle is complete says nothing about it: build 20 dropped the
+legacy tables while its Runtime read an empty vector collection.
+
+Reads go through the public `HyMemoryClient` (`list_memories`, `get`).
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from collections.abc import Iterator
+from contextlib import closing, contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from one_memory.database_access import main_database
+from one_memory.model_profiles import ModelProfileSnapshot
+from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
+
+_PRODUCTIVITY_KINDS = frozenset({"coding", "working"})
+_PRODUCTIVITY_LAYERS = frozenset(
+    {"coding_experience", "working_experience", "coding_fact", "working_fact"}
+)
+_SDK_STATUSES = ("active", "superseded", "shadow", "negated", "conflicted", "archived")
+_PAGE = 500
+
+
+@dataclass(frozen=True)
+class LegacyMemoryIds:
+    chat: tuple[str, ...]
+    productivity: tuple[str, ...]
+    users: tuple[str, ...]
+
+    @property
+    def empty(self) -> bool:
+        return not self.chat and not self.productivity
+
+
+@dataclass(frozen=True)
+class ReadbackReport:
+    chat_expected: int
+    chat_readable: int
+    productivity_expected: int
+    productivity_readable: int
+
+    @property
+    def passed(self) -> bool:
+        # Every chat memory is an SDK node by construction, so all must read
+        # back. The SDK's Coding reconciler deletes experiences it retires, so
+        # a partial productivity count is normal; none at all is not.
+        return self.chat_readable == self.chat_expected and (
+            self.productivity_expected == 0 or self.productivity_readable > 0
+        )
+
+    def as_document(self) -> dict[str, object]:
+        return {
+            "chat_expected": self.chat_expected,
+            "chat_readable": self.chat_readable,
+            "productivity_expected": self.productivity_expected,
+            "productivity_readable": self.productivity_readable,
+            "passed": self.passed,
+        }
+
+
+class MemoryReadbackFailed(DataPlaneMigrationError):
+    def __init__(self, report: ReadbackReport) -> None:
+        super().__init__("sdk_migration_memory_unreadable")
+        self.report = report
+
+
+def _tables(connection: sqlite3.Connection) -> set[str]:
+    return {
+        str(row[0])
+        for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    }
+
+
+def read_legacy_memory_ids(connection: sqlite3.Connection) -> LegacyMemoryIds:
+    """Confirmed legacy memories, by the SDK id the old bridge recorded."""
+    tables = _tables(connection)
+    if "governed_memories" not in tables:
+        return LegacyMemoryIds(chat=(), productivity=(), users=())
+    mapped: dict[str, tuple[str, str | None]] = {}
+    if "memory_task_outputs" in tables:
+        for (body,) in connection.execute("SELECT body FROM memory_task_outputs"):
+            try:
+                document = json.loads(body)
+            except (TypeError, ValueError):
+                continue
+            legacy_id, sdk_id = document.get("memory_id"), document.get("hy_memory_id")
+            if isinstance(legacy_id, str) and isinstance(sdk_id, str):
+                layer = document.get("layer")
+                mapped[legacy_id] = (sdk_id, layer if isinstance(layer, str) else None)
+    chat: set[str] = set()
+    productivity: set[str] = set()
+    users: set[str] = set()
+    for memory_id, user_id, kind in connection.execute(
+        "SELECT memory_id,user_id,kind FROM governed_memories WHERE status='confirmed'"
+    ):
+        sdk_id, layer = mapped.get(str(memory_id), (str(memory_id), None))
+        users.add(str(user_id))
+        if str(kind) in _PRODUCTIVITY_KINDS or layer in _PRODUCTIVITY_LAYERS:
+            productivity.add(sdk_id)
+        else:
+            chat.add(sdk_id)
+    return LegacyMemoryIds(
+        chat=tuple(sorted(chat)),
+        productivity=tuple(sorted(productivity)),
+        users=tuple(sorted(users)),
+    )
+
+
+def legacy_memory_ids_from(database_path: Path) -> LegacyMemoryIds:
+    with main_database(database_path).read() as connection:
+        return read_legacy_memory_ids(connection)
+
+
+def legacy_memory_ids_from_bundle(
+    data_dir: Path, migration_id: str | None
+) -> LegacyMemoryIds | None:
+    """The same list from the pre-cutover bundle, for a home already migrated."""
+    if not migration_id:
+        return None
+    database = (
+        data_dir
+        / "backups"
+        / f"{migration_id}-pre-cutover.sdk-bundle"
+        / "management"
+        / "one-memory.sqlite3"
+    )
+    if not database.is_file():
+        return None
+    with closing(sqlite3.connect(f"file:{database}?mode=ro", uri=True)) as connection:
+        return read_legacy_memory_ids(connection)
+
+
+def read_active_profile(database_path: Path) -> ModelProfileSnapshot | None:
+    """The active model profile, read only; the status probe runs beside Core."""
+    with main_database(database_path).read(row_factory=sqlite3.Row) as connection:
+        if not {"model_profiles", "model_profile_versions"} <= _tables(connection):
+            return None
+        row = connection.execute(
+            "SELECT version.* FROM model_profiles profile "
+            "JOIN model_profile_versions version "
+            "ON version.profile_id=profile.profile_id AND version.revision=profile.revision "
+            "WHERE profile.is_active=1"
+        ).fetchone()
+    if row is None:
+        return None
+    return ModelProfileSnapshot(
+        profile_id=row["profile_id"],
+        revision=row["revision"],
+        provider_kind=row["provider_kind"],
+        base_url=row["base_url"],
+        model=row["model"],
+        embedding_model=row["embedding_model"],
+        embedding_provider_kind=row["embedding_provider_kind"],
+        embedding_base_url=row["embedding_base_url"],
+        embedding_dimensions=row["embedding_dimensions"],
+        timeout_seconds=row["timeout_seconds"],
+        max_concurrency=row["max_concurrency"],
+        context_window=row["context_window"],
+        has_api_key=row["secret_ref"] is not None,
+        disable_thinking=bool(row["disable_thinking"]),
+    )
+
+
+@contextmanager
+def runtime_sdk_client(
+    database_path: Path, data_dir: Path, profile: ModelProfileSnapshot
+) -> Iterator[Any]:
+    """A public SDK client configured exactly as the Runtime configures it."""
+    from one_memory.hy_sdk_runtime import acquire_client
+    from one_memory.model_profiles import ModelProfileStore
+    from one_memory.secrets import KeyringSecretStore, SecretStoreUnavailableError
+
+    try:
+        secret = ModelProfileStore(database_path).resolve_snapshot_secret(
+            profile.profile_id, profile.revision, KeyringSecretStore()
+        )
+    except SecretStoreUnavailableError:
+        raise DataPlaneMigrationError("sdk_migration_secure_storage_unavailable") from None
+    if profile.has_api_key and secret is None:
+        raise DataPlaneMigrationError("sdk_migration_model_secret_unavailable")
+    lease = acquire_client(profile, api_key=secret, data_dir=data_dir)
+    try:
+        yield lease.client
+    finally:
+        lease.close()
+
+
+def verify_readback(client: Any, legacy: LegacyMemoryIds) -> ReadbackReport:
+    listed: set[str] = set()
+    for user_id in legacy.users or ("local",):
+        offset = 0
+        while True:
+            raw = client.list_memories(
+                user_id=user_id,
+                statuses=list(_SDK_STATUSES),
+                limit=_PAGE,
+                offset=offset,
+                order="asc",
+            )
+            bucket = raw.get("vdb") if isinstance(raw, dict) else None
+            rows = bucket.get("memories") if isinstance(bucket, dict) else None
+            total = bucket.get("total") if isinstance(bucket, dict) else None
+            if not isinstance(rows, list) or not isinstance(total, int):
+                raise DataPlaneMigrationError("sdk_migration_memory_list_shape_unknown")
+            listed.update(
+                str(row["memory_id"])
+                for row in rows
+                if isinstance(row, dict) and row.get("memory_id")
+            )
+            offset += len(rows)
+            if not rows or offset >= total:
+                break
+
+    def readable(memory_id: str) -> bool:
+        return memory_id in listed or client.get(memory_id) is not None
+
+    return ReadbackReport(
+        chat_expected=len(legacy.chat),
+        chat_readable=sum(1 for memory_id in legacy.chat if readable(memory_id)),
+        productivity_expected=len(legacy.productivity),
+        productivity_readable=sum(1 for memory_id in legacy.productivity if readable(memory_id)),
+    )
+

memhub/apps/core/src/one_memory/sdk_vector_reattach.py

@@ -0,0 +1,149 @@
+"""Offline reattachment of SDK vector collections named by builds before #145.
+
+Those builds named a profile's embedding space `one_memory_<12 hex>`; #145
+renamed it `memhub_<16 hex>` and dropped the step that carried the old
+collections over, so every later Runtime opened an empty space beside the old
+one. Runtime code only reaches vectors through the SDK. This module runs from
+`sdk-migrate` while the Runtime is stopped, through Chroma's public client.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from collections.abc import Iterator
+from contextlib import closing
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from one_memory.hy_sdk_runtime import vector_collection_bases
+from one_memory.model_profiles import ModelProfileSnapshot
+from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
+
+_BATCH = 500
+
+
+@dataclass(frozen=True)
+class ReattachedCollection:
+    legacy: str
+    current: str
+    legacy_count: int
+    merged_count: int
+    final_count: int
+
+
+def _vector_directory(data_dir: Path) -> Path:
+    return data_dir / "hy-memory" / "governed-v2" / "vector"
+
+
+def _stored_names(data_dir: Path) -> tuple[str, ...]:
+    database = _vector_directory(data_dir) / "chroma.sqlite3"
+    if not database.is_file():
+        return ()
+    with closing(sqlite3.connect(f"file:{database}?mode=ro", uri=True)) as connection:
+        try:
+            rows = connection.execute("SELECT name FROM collections").fetchall()
+        except sqlite3.Error:
+            return ()
+    return tuple(sorted(str(row[0]) for row in rows))
+
+
+def legacy_vector_collections(
+    data_dir: Path, profile: ModelProfileSnapshot
+) -> tuple[tuple[str, str], ...]:
+    """(legacy name, current name) pairs still waiting to be reattached.
+
+    Read-only, so `sdk-migration-status` can ask it.
+    """
+    current, legacy_bases = vector_collection_bases(profile)
+    pairs: list[tuple[str, str]] = []
+    for name in _stored_names(data_dir):
+        for base in legacy_bases:
+            if name == base or name.startswith(f"{base}_"):
+                pairs.append((name, current + name[len(base) :]))
+                break
+    return tuple(pairs)
+
+
+def _records(collection: Any) -> Iterator[dict[str, Any]]:
+    offset = 0
+    while True:
+        page = collection.get(
+            include=["embeddings", "documents", "metadatas"],
+            limit=_BATCH,
+            offset=offset,
+        )
+        ids = list(page["ids"])
+        if not ids:
+            return
+        yield {
+            "ids": ids,
+            "embeddings": page["embeddings"],
+            "documents": page["documents"],
+            "metadatas": page["metadatas"],
+        }
+        offset += len(ids)
+
+
+def reattach_legacy_vectors(
+    data_dir: Path, profile: ModelProfileSnapshot
+) -> tuple[ReattachedCollection, ...]:
+    """Rename legacy collections to the current names, keeping newer writes.
+
+    A current collection that already exists (written after the rename broke)
+    is merged into the legacy one first; on an id present in both, the newer
+    current record wins. Callers must have stopped every SDK writer.
+    """
+    pairs = legacy_vector_collections(data_dir, profile)
+    if not pairs:
+        return ()
+    import chromadb
+    from chromadb.config import Settings
+
+    client = chromadb.PersistentClient(
+        path=str(_vector_directory(data_dir)),
+        settings=Settings(anonymized_telemetry=False),
+    )
+    existing = set(_stored_names(data_dir))
+    done: list[ReattachedCollection] = []
+    for legacy_name, current_name in pairs:
+        legacy = client.get_collection(legacy_name)
+        legacy_count = legacy.count()
+        merged = 0
+        if current_name in existing:
+            current = client.get_collection(current_name)
+            legacy_ids = set(_ids(legacy))
+            current_ids: set[str] = set()
+            for batch in _records(current):
+                legacy.upsert(**batch)
+                current_ids.update(batch["ids"])
+            merged = len(current_ids)
+            if legacy.count() != len(legacy_ids | current_ids):
+                raise DataPlaneMigrationError("sdk_migration_vector_merge_incomplete")
+            client.delete_collection(current_name)
+        legacy.modify(name=current_name)
+        final = client.get_collection(current_name).count()
+        if final < legacy_count:
+            raise DataPlaneMigrationError("sdk_migration_vector_reattach_incomplete")
+        existing.discard(legacy_name)
+        existing.add(current_name)
+        done.append(
+            ReattachedCollection(
+                legacy=legacy_name,
+                current=current_name,
+                legacy_count=legacy_count,
+                merged_count=merged,
+                final_count=final,
+            )
+        )
+    return tuple(done)
+
+
+def _ids(collection: Any) -> Iterator[str]:
+    offset = 0
+    while True:
+        ids = list(collection.get(include=[], limit=_BATCH, offset=offset)["ids"])
+        if not ids:
+            return
+        yield from ids
+        offset += len(ids)

memhub/apps/core/src/one_memory/target_schema.py

@@ -54,6 +54,8 @@ LEGACY_TABLES_IN_DROP_ORDER = (
     "evidence_origins",
     "evidence_versions",
     "evidence_repairs",
+    # FTS5 index over Evidence text; dropping it removes its shadow tables.
+    "evidence_search",
     "evidence",
     "memory_id_aliases",
     "raw_id_aliases",

memhub/apps/core/tests/TEST_INDEX.md

@@ -54,6 +54,7 @@
 | `test_claude_adoption_probe.py` | `contract` | The external-use verifier must distinguish a read from merely mentioning it. |
 | `test_cli_commands.py` | `contract` | 覆盖 status reports the verified runtime as json、doctor fails when runtime identity cannot be verified、serve and legacy empty command share the same entry 等 4 个场景。 |
 | `test_cli_lifecycle.py` | `integration` | 覆盖 cli starts and stops without residual runtime 场景。 |
+| `test_cli_migration_error_codes.py` | `unit` | The startup dialog can only act on the code it is handed. |
 | `test_cli_restore.py` | `contract` | 覆盖 restore requires a source、restore refuses while the runtime is running、restore previews without confirmation 等 4 个场景。 |
 | `test_codex_header_provenance.py` | `contract` | 覆盖 header workspace versions and native ids reach file origin、new adapter rereads header after a real checkpoint、machine and unknown session sources do not establish human or agent sender 等 14 个场景。 |
 | `test_connection_lifetime.py` | `integration` | Every SQLite connection a request opens must be closed when the request ends. |
@@ -138,7 +139,6 @@
 | `test_live_watch.py` | `contract` | 覆盖 a new turn lands well inside the ten second target、start prime catches a change that predates the watcher、a run that predates the change is followed by a fresh one 等 15 个场景。 |
 | `test_live_watch_grants.py` | `integration` | 覆盖 explicit sync grant starts watch without restart and revocation stops it 场景。 |
 | `test_local_embedding.py` | `contract` | Guards for the built-in embedding model. |
-| `test_local_embedding_offsets.py` | `contract` | 覆盖 real locked tokenizer offsets are unicode codepoints、known emoji and combining offsets are not utf8 or utf16 units、offsets are not truncated or padded and do not create onnx session 等 9 个场景。 |
 | `test_local_onnx_provider.py` | `contract` | Expect local ONNX embedding to be the default and to feed Hy Memory. |
 | `test_log_file.py` | `contract` | 覆盖 what the watchdog says survives a dock launch、the log lives where the data does、a machine left running for months does not fill up 等 4 个场景。 |
 | `test_log_privacy.py` | `integration` | The file log is shipped to us in bug reports; it must not carry originals. |
@@ -231,21 +231,8 @@
 | `test_recall_scope_control.py` | `integration` | 覆盖 external scope cannot override shared or host workspace boundary、scope only mutation is versioned without changing status or reindexing 场景。 |
 | `test_recall_scope_off_loop.py` | `contract` | The workspace lookup behind an external recall must not run on the loop. |
 | `test_recent_first_import.py` | `contract` | Recent-file priority survives restart through a selection-bound frozen copy. |
-| `test_retrieval_chunking.py` | `contract` | 覆盖 default 384 body 64 overlap and exact source slices、all prefix and special tokens count toward strict 512 budget、real tokenizer covers long chinese middle and tail object locations 等 20 个场景。 |
-| `test_retrieval_eligibility_benchmark.py` | `contract` | 覆盖 scale probe proves composite authority index changes the plan 场景。 |
-| `test_retrieval_index_store.py` | `contract` | 覆盖 constructor and reads do not create database or schema、explicit initialization rolls back and never grants model policy、consumer and jobs commit together 等 19 个场景。 |
-| `test_retrieval_index_supervisor.py` | `contract` | 覆盖 stop waits for native exit and does not close resources、async cancellation repeatedly waits for native thread、idle no trigger no spin and duplicate start rejected 等 14 个场景。 |
-| `test_retrieval_index_worker.py` | `contract` | 覆盖 tick indexes complete original and reopen can search、change during native encoding never publishes、cancel after encode does not publish and is reclaimable 等 10 个场景。 |
 | `test_retrieval_lifespan.py` | `integration` | 覆盖 retrieval defaults off and does not create index、enabling retrieval requires runtime ownership、index enabled serve imports without building an unmanaged app 等 6 个场景。 |
-| `test_retrieval_mapped_content.py` | `contract` | 覆盖 raw authors keep exact body and do not mutate evidence、raw and author qualification rejects without tokenizing、ambiguous generated and body hygiene are not admitted 等 22 个场景。 |
-| `test_retrieval_mapped_integration.py` | `contract` | 覆盖 empty materialization rechecks excluded map、completed excluded missing active binding is not empty success、old empty completion is invalidated when a new indexed record arrives 等 24 个场景。 |
-| `test_retrieval_provisioning.py` | `contract` | The index has to come up through the product, not through a test helper. |
 | `test_retrieval_query_plan.py` | `contract` | Local QueryPlan: extract URL/path objects. |
-| `test_retrieval_raw_candidates.py` | `contract` | 覆盖 positive only current authorized published chunks are searched、empty authority never reads database or vector、unapproved source never enters vector query 等 15 个场景。 |
-| `test_retrieval_raw_evidence.py` | `contract` | 覆盖 materialized text is exact current original、stale candidates do not authorize original read、budget does not cut a chunk into fake partial object 等 6 个场景。 |
-| `test_retrieval_raw_lane.py` | `contract` | v1 raw lane must not claim complete empty when the derived index is partial. |
-| `test_retrieval_runtime.py` | `contract` | 覆盖 constructor and missing start create nothing、stop releases supervisor worker but preserves immutable snapshot、not initialized or invalid configuration never creates index 等 21 个场景。 |
-| `test_retrieval_vector_store.py` | `contract` | 覆盖 constructor does no io and unopened calls do not create、open existing missing store or collection never creates、real chroma collection has no embedding function or documents 等 32 个场景。 |
 | `test_runtime.py` | `integration` | 覆盖 runtime writes and removes ownership receipt、runtime rejects second owner for same data directory、nonpublishing maintenance owner still excludes a second core 等 4 个场景。 |
 | `test_runtime_client.py` | `contract` | 覆盖 runtime inspection authenticates the discovered instance、runtime inspection rejects untrusted receipts without probing、runtime inspection distinguishes stopped and identity mismatch 场景。 |
 | `test_sanitization_source_map.py` | `contract` | 覆盖 mapping matches existing sanitizer and exact original slices、synthetic replacements never claim original locations、repeated identical suffix is mapped by positions not string search 等 6 个场景。 |
@@ -327,4 +314,4 @@
 | `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
 | `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
 
-共 322 个测试文件。
+共 309 个测试文件。

memhub/apps/core/tests/core_test_classification.py

@@ -14,13 +14,14 @@ from pathlib import Path
 import pytest
 
 PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
-EXPECTED_TEST_FILE_COUNT = 322
-EXPECTED_TEST_FILES_SHA256 = "d242783adb4015ebdbb5c099f3b5aa88a99c828ca6fddc8ca8d6c74c56e48945"
+EXPECTED_TEST_FILE_COUNT = 309
+EXPECTED_TEST_FILES_SHA256 = "ef6165c249971f5a2ab71a0c49f6d859c8c0f3612e3f749fba0acce0eab513ea"
 
 # Pure transformations, validation models, parity checks and state machines.
 # This list is intentionally small: uncertain files default to contract.
 UNIT_FILES = frozenset(
     {
+        "test_cli_migration_error_codes.py",
         "test_context_v2_contract_parity.py",
         "test_daemon_contract.py",
         "test_diagnostic_payload_keys.py",

memhub/apps/core/tests/test_cli_migration_error_codes.py

@@ -0,0 +1,101 @@
+"""The startup dialog can only act on the code it is handed.
+
+A full disk and a damaged backup are different next steps, so they must reach
+the person under different names. These tests pin those names.
+"""
+
+from __future__ import annotations
+
+import errno
+from pathlib import Path
+
+import pytest
+
+from one_memory.cli import migration_failure_code
+
+
+class _Coded(RuntimeError):
+    def __init__(self, code: str) -> None:
+        super().__init__(code)
+        self.code = code
+
+
+def test_running_out_of_disk_is_named_however_deep_it_is_wrapped() -> None:
+    error = OSError(errno.ENOSPC, "No space left on device")
+    wrapped: BaseException = error
+    for _ in range(3):
+        try:
+            raise RuntimeError("wrapper") from wrapped
+        except RuntimeError as caught:
+            wrapped = caught
+
+    assert migration_failure_code(wrapped) == "sdk_migration_disk_space_insufficient"
+    # The explicit context chain counts too: `raise ... from None` discards the
+    # cause but Python still keeps the original as __context__.
+    try:
+        try:
+            raise OSError(errno.ENOSPC, "No space left on device")
+        except OSError:
+            raise RuntimeError("sdk_snapshot_failed") from None
+    except RuntimeError as caught:
+        assert migration_failure_code(caught) == "sdk_migration_disk_space_insufficient"
+
+
+def test_quota_exhaustion_counts_as_no_space(tmp_path: Path) -> None:
+    quota = getattr(errno, "EDQUOT", errno.ENOSPC)
+    assert (
+        migration_failure_code(OSError(quota, "Disk quota exceeded"))
+        == "sdk_migration_disk_space_insufficient"
+    )
+
+
+def test_migration_state_codes_pass_through_because_they_are_already_actionable() -> None:
+    assert (
+        migration_failure_code(_Coded("sdk_data_plane_migration_bundle_conflict"))
+        == "sdk_data_plane_migration_bundle_conflict"
+    )
+    assert (
+        migration_failure_code(_Coded("sdk_migration_swap_state_ambiguous"))
+        == "sdk_migration_swap_state_ambiguous"
+    )
+
+
+def test_unknown_failures_keep_their_message_out_of_the_code() -> None:
+    error = RuntimeError("/Users/someone/private/path blew up")
+    assert migration_failure_code(error) == "sdk_migration_preflight_failed"
+    assert (
+        migration_failure_code(error, fallback="sdk_migration_bundle_copy_failed")
+        == "sdk_migration_bundle_copy_failed"
+    )
+
+
+def test_a_non_code_attribute_never_becomes_a_code() -> None:
+    class Odd(RuntimeError):
+        code = 17
+
+    assert migration_failure_code(Odd("x")) == "sdk_migration_preflight_failed"
+
+
+def test_the_chain_survives_a_cycle() -> None:
+    first = RuntimeError("first")
+    second = RuntimeError("second")
+    first.__cause__ = second
+    second.__cause__ = first
+
+    assert migration_failure_code(first) == "sdk_migration_preflight_failed"
+
+
+@pytest.mark.parametrize(
+    "code",
+    [
+        "sdk_migration_disk_space_insufficient",
+        "sdk_migration_preflight_failed",
+        "sdk_migration_bundle_copy_failed",
+        "sdk_migration_cutover_failed",
+    ],
+)
+def test_the_codes_the_desktop_translates_stay_in_sync(code: str) -> None:
+    """Desktop maps these exact names to sentences; renaming one blinds it."""
+    desktop = Path(__file__).resolve().parents[2] / "desktop" / "src" / "stable-runtime.ts"
+    assert desktop.is_file(), desktop
+    assert f'"{code}"' in desktop.read_text(encoding="utf-8")

memhub/apps/core/tests/test_cli_restore.py

@@ -1,10 +1,14 @@
+import asyncio
 import io
+import json
 import sqlite3
-from types import SimpleNamespace
 
 from one_memory import cli
 from one_memory.config import Settings
+from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
+from one_memory.memory_server.migrations import MIGRATIONS
 from one_memory.runtime_client import RuntimeStatus
+from one_memory.sdk_data_plane_migration import SdkDataPlaneMigrationStore
 
 
 def test_restore_requires_a_source(tmp_path, monkeypatch) -> None:
@@ -110,56 +114,7 @@ def test_restore_sdk_bundle_uses_full_data_plane_restore(tmp_path, monkeypatch)
     assert '"restored_files":4' in document
 
 
-def test_migration_bundle_refuses_while_runtime_is_running(tmp_path, monkeypatch) -> None:
-    monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    monkeypatch.setattr(
-        cli,
-        "inspect_runtime",
-        lambda _data_dir: RuntimeStatus("running", "runtime_verified"),
-    )
-    output = io.StringIO()
-
-    code = cli.run(
-        [
-            "migration-bundle",
-            "--bundle-id",
-            "migration-1",
-            "--runtime-build",
-            "build-13",
-            "--yes",
-            "--json",
-        ],
-        output=output,
-    )
-    assert code == 4
-    assert "sdk_migration_bundle_runtime_running" in output.getvalue()
-
-
-def test_migration_bundle_previews_before_writing(tmp_path, monkeypatch) -> None:
-    monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    monkeypatch.setattr(
-        cli,
-        "inspect_runtime",
-        lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
-    )
-    output = io.StringIO()
-
-    code = cli.run(
-        [
-            "migration-bundle",
-            "--bundle-id",
-            "migration-1",
-            "--runtime-build",
-            "build-13",
-            "--json",
-        ],
-        output=output,
-    )
-    assert code == 4
-    assert '"state": "preview"' in output.getvalue() or '"state":"preview"' in output.getvalue()
-
-
-def test_migration_bundle_success_reports_bundle_identity(tmp_path, monkeypatch) -> None:
+def _stopped(monkeypatch, tmp_path) -> None:
     monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
     monkeypatch.setattr(
         cli,
@@ -167,236 +122,22 @@ def test_migration_bundle_success_reports_bundle_identity(tmp_path, monkeypatch)
         lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
     )
 
-    class Result:
-        bundle_id = "migration-1"
-        bundle_path = tmp_path / "backups" / "migration-1.sdk-bundle"
-        file_count = 4
-        total_bytes = 123
 
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_bundle.create_sdk_migration_bundle",
-        lambda **_kwargs: Result(),
-    )
-    output = io.StringIO()
-    code = cli.run(
-        [
-            "migration-bundle",
-            "--bundle-id",
-            "migration-1",
-            "--runtime-build",
-            "build-13",
-            "--yes",
-            "--json",
-        ],
-        output=output,
-    )
-    assert code == 0
-    document = output.getvalue().replace(" ", "")
-    assert '"state":"created"' in document
-    assert '"bundle_id":"migration-1"' in document
-
-
-def test_migration_bundle_reuses_an_existing_verified_bundle(tmp_path, monkeypatch) -> None:
-    from one_memory.sdk_migration_bundle import MigrationBundleRejected
-
-    monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    monkeypatch.setattr(
-        cli,
-        "inspect_runtime",
-        lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
-    )
-
-    def exists(**_kwargs):
-        raise MigrationBundleRejected("sdk_migration_bundle_exists")
-
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_bundle.create_sdk_migration_bundle",
-        exists,
-    )
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_bundle.inspect_sdk_migration_bundle",
-        lambda **_kwargs: {"file_count": 4, "total_bytes": 123},
-    )
-    output = io.StringIO()
-
-    code = cli.run(
-        [
-            "migration-bundle",
-            "--bundle-id",
-            "migration-1",
-            "--runtime-build",
-            "build-13",
-            "--yes",
-            "--json",
-        ],
-        output=output,
-    )
-
-    assert code == 0
-    assert '"state": "reused"' in output.getvalue()
-
-
-def test_sdk_migrate_preview_never_commits_cutover(tmp_path, monkeypatch) -> None:
-    monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    monkeypatch.setattr(
-        cli,
-        "inspect_runtime",
-        lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
-    )
-    output = io.StringIO()
-    code = cli.run(
-        [
-            "sdk-migrate",
-            "--migration-id",
-            "migration-1",
-            "--bundle-id",
-            "bundle-1",
-            "--json",
-        ],
-        output=output,
-    )
-    assert code == 4
-    document = output.getvalue().replace(" ", "")
-    assert '"state":"preview"' in document
-    assert '"commits_cutover":false' in document
-
-
-def test_sdk_migrate_runs_offline_backfill_without_cutover(tmp_path, monkeypatch) -> None:
-    monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    monkeypatch.setattr(
-        cli,
-        "inspect_runtime",
-        lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
-    )
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_bundle.inspect_sdk_migration_bundle",
-        lambda **_kwargs: {"runtime_build": "v1.0.4"},
-    )
-
-    async def fake_migrate(*_args, **_kwargs):
-        return None
-
-    monkeypatch.setattr("one_memory.memory_server.migration_runner.migrate", fake_migrate)
-
-    profile = SimpleNamespace(
-        has_api_key=False,
-        embedding_provider_kind="openai",
-        embedding_model="embedding",
-        embedding_dimensions=1024,
-    )
-
-    class Profiles:
-        def __init__(self, _database):
-            pass
-
-        def active_identity(self):
-            return ("profile", 1)
-
-        def snapshot(self, *_identity):
-            return profile
-
-        def resolve_snapshot_secret(self, *_args):
-            return None
-
-    class Secrets:
-        pass
-
-    class Lease:
-        client = SimpleNamespace(
-            _loop_thread=None,
-            _vector_store=None,
-            _embed_service=None,
-            _graph_store=None,
+def _legacy_home(tmp_path) -> None:
+    database = tmp_path / "one-memory.sqlite3"
+    asyncio.run(migrate(SqliteMigrationStore(database), MIGRATIONS))
+    with sqlite3.connect(database) as connection:
+        connection.execute(
+            "INSERT INTO evidence ("
+            "evidence_id,idempotency_key,user_id,source_id,conversation_id,"
+            "content_kind,role,written_by,verbatim,content_hash,revision,"
+            "imported_at,updated_at"
+            ") VALUES ('e1','k','local','cursor','s','raw_conversation',"
+            "'user','person','hello','hash',1,'now','now')"
         )
 
-        def close(self):
-            return None
 
-    async def backfill(**_kwargs):
-        return SimpleNamespace(
-            state="ready",
-            phase="reconciled",
-            migration_id="migration-1",
-            counts={"verified": 2},
-            memory_items=1,
-            raw_items=1,
-        )
-
-    monkeypatch.setattr("one_memory.model_profiles.ModelProfileStore", Profiles)
-    monkeypatch.setattr("one_memory.secrets.KeyringSecretStore", Secrets)
-    monkeypatch.setattr(
-        "one_memory.hy_sdk_runtime.acquire_client",
-        lambda *_args, **_kwargs: Lease(),
-    )
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_runner.run_sdk_data_plane_backfill",
-        backfill,
-    )
-    output = io.StringIO()
-    code = cli.run(
-        [
-            "sdk-migrate",
-            "--migration-id",
-            "migration-1",
-            "--bundle-id",
-            "bundle-1",
-            "--yes",
-            "--json",
-        ],
-        output=output,
-    )
-    assert code == 0
-    document = output.getvalue().replace(" ", "")
-    assert '"state":"ready"' in document
-    assert '"commits_cutover":false' in document
-
-
-def test_sdk_cutover_is_offline_confirmed_and_returns_marker(tmp_path, monkeypatch) -> None:
-    monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    monkeypatch.setattr(
-        cli,
-        "inspect_runtime",
-        lambda _data_dir: RuntimeStatus("stopped", "runtime_receipt_missing"),
-    )
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_runner.commit_reconciled_cutover",
-        lambda **_kwargs: SimpleNamespace(
-            migration_id="migration-1",
-            backup_bundle_id="bundle-1",
-            sdk_version="2.0.5",
-            completed_at="2026-10-21T00:00:00Z",
-        ),
-    )
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_runner.activate_sdk_migration_staging",
-        lambda *, commit, **_kwargs: commit(),
-    )
-    monkeypatch.setattr(
-        "one_memory.sdk_migration_runner.resume_legacy_cleanup",
-        lambda **_kwargs: (),
-    )
-    output = io.StringIO()
-
-    code = cli.run(
-        [
-            "sdk-cutover",
-            "--migration-id",
-            "migration-1",
-            "--bundle-id",
-            "bundle-1",
-            "--yes",
-            "--json",
-        ],
-        output=output,
-    )
-
-    assert code == 0
-    document = output.getvalue().replace(" ", "")
-    assert '"state":"cutover"' in document
-    assert '"sdk_version":"2.0.5"' in document
-
-
-def test_sdk_cutover_never_runs_while_runtime_is_live(tmp_path, monkeypatch) -> None:
+def test_sdk_migrate_never_runs_while_runtime_is_live(tmp_path, monkeypatch) -> None:
     monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
     monkeypatch.setattr(
         cli,
@@ -404,35 +145,72 @@ def test_sdk_cutover_never_runs_while_runtime_is_live(tmp_path, monkeypatch) ->
         lambda _data_dir: RuntimeStatus("running", "healthy"),
     )
     output = io.StringIO()
+    code = cli.run(["sdk-migrate", "--runtime-build", "20", "--yes", "--json"], output=output)
+    assert code == 4
+    assert json.loads(output.getvalue())["code"] == "sdk_migration_runtime_running"
 
-    code = cli.run(
-        [
-            "sdk-cutover",
-            "--migration-id",
-            "migration-1",
-            "--bundle-id",
-            "bundle-1",
-            "--yes",
-            "--json",
-        ],
-        output=output,
+
+def test_sdk_migrate_previews_without_confirmation(tmp_path, monkeypatch) -> None:
+    _stopped(monkeypatch, tmp_path)
+    _legacy_home(tmp_path)
+    output = io.StringIO()
+    assert cli.run(["sdk-migrate", "--runtime-build", "20", "--json"], output=output) == 4
+    assert json.loads(output.getvalue())["state"] == "preview"
+    assert SdkDataPlaneMigrationStore(tmp_path / "one-memory.sqlite3").authority().value == (
+        "legacy"
     )
 
-    assert code == 4
-    assert '"code": "sdk_cutover_runtime_running"' in output.getvalue()
+
+def test_sdk_migrate_requires_the_runtime_build(tmp_path, monkeypatch) -> None:
+    _stopped(monkeypatch, tmp_path)
+    assert cli.run(["sdk-migrate", "--yes"], output=io.StringIO()) == 2
 
 
-def test_sdk_migration_status_distinguishes_empty_and_legacy_data(tmp_path, monkeypatch) -> None:
+def test_sdk_migrate_reports_each_step_then_the_result(tmp_path, monkeypatch) -> None:
+    _stopped(monkeypatch, tmp_path)
+    _legacy_home(tmp_path)
+    output = io.StringIO()
+
+    code = cli.run(["sdk-migrate", "--runtime-build", "20", "--yes", "--json"], output=output)
+
+    assert code == 0, output.getvalue()
+    lines = [json.loads(line) for line in output.getvalue().splitlines()]
+    assert [line["step"] for line in lines[:-1]] == ["bundle", "prepare", "cutover", "cleanup"]
+    assert lines[-1]["state"] == "cutover"
+    assert lines[-1]["bundle_id"] == f"{lines[-1]['migration_id']}-pre-cutover"
+    assert not (tmp_path / ".sdk-migration-capture-fence.json").exists()
+
+    again = io.StringIO()
+    assert cli.run(["sdk-migrate", "--runtime-build", "20", "--yes", "--json"], output=again) == 0
+    assert json.loads(again.getvalue())["state"] == "not_required"
+
+
+def test_sdk_migrate_failure_is_one_named_code(tmp_path, monkeypatch) -> None:
+    import errno
+
+    _stopped(monkeypatch, tmp_path)
+
+    def full_disk(**_kwargs):
+        raise OSError(errno.ENOSPC, "No space left on device")
+
+    monkeypatch.setattr("one_memory.sdk_migration_runner.run_startup_migration", full_disk)
+    output = io.StringIO()
+    assert cli.run(["sdk-migrate", "--runtime-build", "20", "--yes", "--json"], output=output) == 1
+    assert json.loads(output.getvalue()) == {
+        "schema_version": 1,
+        "state": "failed",
+        "code": "sdk_migration_disk_space_insufficient",
+    }
+
+
+def test_sdk_migration_status_says_whether_migrate_has_work(tmp_path, monkeypatch) -> None:
     monkeypatch.setattr(cli, "load_settings", lambda: Settings(_env_file=None, data_dir=tmp_path))
-    empty = io.StringIO()
-    assert cli.run(["sdk-migration-status", "--json"], output=empty) == 0
-    assert '"state": "empty"' in empty.getvalue()
 
-    database = tmp_path / "one-memory.sqlite3"
-    with sqlite3.connect(database) as connection:
-        connection.execute("CREATE TABLE evidence (evidence_id TEXT PRIMARY KEY)")
-        connection.execute("INSERT INTO evidence VALUES ('e1')")
-    required = io.StringIO()
-    assert cli.run(["sdk-migration-status", "--json"], output=required) == 0
-    assert '"state": "required"' in required.getvalue()
-    assert '"raw": 1' in required.getvalue()
+    def status() -> dict[str, object]:
+        output = io.StringIO()
+        assert cli.run(["sdk-migration-status", "--json"], output=output) == 0
+        return json.loads(output.getvalue())
+
+    assert status()["state"] == "not_required"
+    _legacy_home(tmp_path)
+    assert status() == {"schema_version": 1, "authority": "legacy", "state": "required"}

memhub/apps/core/tests/test_diagnostic_log.py

@@ -18,11 +18,13 @@ from one_memory.diagnostic_log import (
     diagnostic_exception,
     diagnostics_enabled,
     diagnostics_requested,
+    read_trace_events,
     record_database_error,
     reset_diagnostics,
     reset_trace_id,
     resource_snapshot,
 )
+from one_memory.exception_reporting import report_exception
 
 
 def test_debug_events_are_collected_only_at_debug_level(tmp_path, monkeypatch) -> None:
@@ -105,7 +107,31 @@ def test_structured_log_is_private_correlated_and_owner_only(tmp_path):
         assert log.path.stat().st_mode & 0o777 == 0o600
 
 
-def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
+def test_arbitrary_valid_detail_keys_and_scalars_are_recorded(tmp_path):
+    log = DiagnosticLog(tmp_path)
+    details = {
+        "Custom.metric-v1": "ready",
+        "noneValue": None,
+        "boolValue": False,
+        "intValue": 7,
+        "floatValue": 1.5,
+    }
+    log.write(
+        level="info",
+        component="core.lifecycle",
+        event="core.started",
+        outcome="succeeded",
+        details=details,
+    )
+    assert log.flush()
+    assert json.loads(log.path.read_text())["details"] == details
+
+
+@pytest.mark.parametrize(
+    "key",
+    ["__proto__", "prototype", "constructor", "_private", "9startsWithDigit", "a" * 65],
+)
+def test_dangerous_or_malformed_detail_keys_are_rejected(tmp_path, key):
     log = DiagnosticLog(tmp_path)
     with pytest.raises(ValueError, match="diagnostic_detail_key_invalid"):
         log.write(
@@ -113,8 +139,76 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
             component="hy.bridge",
             event="hy.failed",
             outcome="failed",
-            details={"prompt": "private"},
+            details={key: "value"},
         )
+
+
+@pytest.mark.parametrize("value", [{}, [], object()])
+def test_complex_detail_values_are_rejected(tmp_path, value):
+    log = DiagnosticLog(tmp_path)
+    with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            details={"payload": value},
+        )
+
+
+@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
+def test_non_finite_detail_values_are_rejected(tmp_path, value):
+    log = DiagnosticLog(tmp_path)
+    with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            details={"metric": value},
+        )
+
+
+@pytest.mark.parametrize("value", [2**53, -(2**53), float(2**53), -float(2**53)])
+def test_numeric_detail_values_outside_json_safe_range_are_rejected(tmp_path, value):
+    log = DiagnosticLog(tmp_path)
+    with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            details={"count": value},
+        )
+
+
+@pytest.mark.parametrize("value", ["", "nul\0byte", "carriage\rreturn", "line\nfeed", "x" * 257])
+def test_invalid_detail_strings_are_rejected(tmp_path, value):
+    log = DiagnosticLog(tmp_path)
+    with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            details={"message": value},
+        )
+
+
+def test_more_than_32_detail_keys_are_rejected(tmp_path):
+    log = DiagnosticLog(tmp_path)
+    with pytest.raises(ValueError, match="diagnostic_detail_count_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            details={f"key{index}": index for index in range(33)},
+        )
+
+
+def test_absolute_or_credential_bearing_stack_is_rejected(tmp_path):
+    log = DiagnosticLog(tmp_path)
     with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
         log.write(
             level="error",
@@ -123,16 +217,90 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
             outcome="failed",
             safe_stack=("/Users/me/private.py:1",),
         )
+    with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            safe_stack=("one_memory/sk-live-secret-token.py:1:call",),
+        )
 
 
-def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) -> None:
-    """Rejection protects the log; it must not become the caller's outage.
+def test_safe_stack_uses_strict_shared_grammar_and_sixteen_line_limit(tmp_path):
+    log = DiagnosticLog(tmp_path)
+    for line in (
+        "arbitrary stack text",
+        "thread=worker|threading.py:12:wait",
+        "FILE:///Users/me/private.py:1:call",
+        "one_memory/worker.py:9:call /Users/private",
+        "one_memory/GOCSPX-privateCredential.py:9:call",
+        "packages/ASIA1234567890ABCDEF/file.ts:1:1",
+        "packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
+    ):
+        with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
+            log.write(
+                level="error",
+                component="hy.bridge",
+                event="hy.failed",
+                outcome="failed",
+                safe_stack=(line,),
+            )
 
-    Two of our own nodes shipped with unlisted `details` keys. Both sat on a
-    request path, so the rejection surfaced as a 500 to the person using the
-    app — including the sedimentation switch, whose every request failed
-    because logging the toggle raised before the endpoint could answer.
-    """
+    accepted = (
+        "apps/desktop/src/main.ts:80:1",
+        "one_memory/diagnostic_log.py:895:_safe_stack_line",
+        "threading.py:1045:_bootstrap_inner",
+        "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+    )
+    log.write(
+        level="error",
+        component="hy.bridge",
+        event="hy.failed",
+        outcome="failed",
+        safe_stack=accepted,
+    )
+    with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
+        log.write(
+            level="error",
+            component="hy.bridge",
+            event="hy.failed",
+            outcome="failed",
+            safe_stack=tuple(f"one_memory/worker.py:{index + 1}:call" for index in range(17)),
+        )
+    assert log.flush()
+    assert json.loads(log.path.read_text())["safe_stack"] == list(accepted)
+
+
+def test_invalid_level_is_strict_but_diagnostic_event_remains_fail_open(
+    tmp_path, monkeypatch
+) -> None:
+    log = DiagnosticLog(tmp_path)
+    with pytest.raises(ValueError, match="diagnostic_level_invalid"):
+        log.write(
+            level="fatal",  # type: ignore[arg-type]
+            component="core.lifecycle",
+            event="core.failed",
+            outcome="failed",
+        )
+
+    monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
+    receipt = configure_diagnostics(tmp_path / "fail-open")
+    try:
+        diagnostic_event(
+            level="fatal",
+            component="core.lifecycle",
+            event="core.failed",
+            outcome="failed",
+        )
+    finally:
+        reset_diagnostics()
+    assert not receipt.path.exists()
+
+
+def test_a_rejected_payload_does_not_break_the_caller_or_later_events(
+    tmp_path, monkeypatch
+) -> None:
     monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
     receipt = configure_diagnostics(tmp_path)
     try:
@@ -141,20 +309,122 @@ def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) ->
             component="memory.sedimentation",
             event="memory_sedimentation.toggled",
             outcome="succeeded",
-            details={"prompt": "private"},
+            details={"payload": ["not", "scalar"]},
+        )
+        diagnostic_event(
+            level="info",
+            component="memory.sedimentation",
+            event="memory_sedimentation.toggled",
+            outcome="succeeded",
+            details={"custom.metric": 1},
         )
     finally:
         reset_diagnostics()
-    # Rejected, so nothing is written — but the call returned.
-    assert not receipt.path.exists() or receipt.path.read_text() == ""
+    events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
+    assert [event["details"] for event in events] == [{"custom.metric": 1}]
+
+
+def test_trace_projection_does_not_expose_arbitrary_local_detail_keys(tmp_path):
+    log = DiagnosticLog(tmp_path)
+    log.write(
+        level="info",
+        component="core.api",
+        event="memory.request.completed",
+        outcome="succeeded",
+        request_id="request-1",
+        details={
+            "phase": "recall",
+            "reason": "/Users/private/query.txt",
+            "status": "sk-live-secret-token",
+            "custom.private": "local-only",
+        },
+    )
+    assert log.flush()
+    events = read_trace_events(tmp_path, request_id="request-1")
+    assert events[0]["details"] == {"phase": "recall"}
+
+
+def test_trace_projection_rejects_symlinked_directory_and_redacts_top_level_secrets(
+    tmp_path,
+) -> None:
+    request_id = "request-1"
+    logs = tmp_path / "logs"
+    logs.mkdir()
+    event = {
+        "timestamp": "2026-10-23T01:00:00Z",
+        "request_id": request_id,
+        "component": "secret.token",
+        "event": "credential.failed",
+        "outcome": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+        "error_code": "GOCSPX-clientSecret_A1",
+        "error_type": "ASIA1234567890ABCDEF",
+        "details": {
+            "phase": "recall",
+            "reason": "x_ASIA1234567890ABCDEF",
+            "mode": "x_GOCSPX-clientSecret_A1",
+            "status": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+            "failure_site": "AIzaSyExampleCredential1234567890",
+            "count": 1,
+        },
+    }
+    (logs / "diagnostic-core.jsonl").write_text(json.dumps(event) + "\n")
+    projected = read_trace_events(tmp_path, request_id=request_id)
+    assert projected == (
+        {
+            "timestamp": "2026-10-23T01:00:00Z",
+            "component": "diagnostic.redacted",
+            "event": "diagnostic.redacted",
+            "outcome": "redacted",
+            "details": {"phase": "recall", "count": 1},
+        },
+    )
+
+    external = tmp_path / "external"
+    external.mkdir()
+    (logs / "diagnostic-core.jsonl").unlink()
+    logs.rmdir()
+    logs.symlink_to(external, target_is_directory=True)
+    with pytest.raises(ValueError, match="diagnostic_directory_unsafe"):
+        read_trace_events(tmp_path, request_id=request_id)
+
+
+def test_credential_detection_is_conservative_and_matches_export_rules() -> None:
+    for value in (
+        "ASIA1234567890ABCDEF",
+        "GOCSPX-clientSecret_A1",
+        "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+        "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8_",
+    ):
+        assert diagnostic_log._contains_credential_material(value)
+    for value in (
+        "11111111-1111-4111-8111-111111111111",
+        "abcdef1234567890abcdef1234567890abcdef12",
+        "cursor",
+    ):
+        assert not diagnostic_log._contains_credential_material(value)
+
+
+def test_trace_reader_never_follows_a_diagnostic_file_symlink(tmp_path) -> None:
+    logs = tmp_path / "logs"
+    logs.mkdir()
+    outside = tmp_path / "outside.jsonl"
+    outside.write_text(
+        json.dumps(
+            {
+                "timestamp": "2026-10-23T01:00:00Z",
+                "request_id": "request-1",
+                "component": "core.api",
+                "event": "memory.request.completed",
+                "outcome": "succeeded",
+            }
+        )
+        + "\n"
+    )
+    (logs / "diagnostic-core.jsonl").symlink_to(outside)
+    assert read_trace_events(tmp_path, request_id="request-1") == ()
 
 
 def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
-    """`enabled` is a bounded boolean and has to be a legal detail key.
-
-    Omitting it from the allow-list is what made the switch's own endpoint
-    answer 500 while still saving the new value.
-    """
     monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
     receipt = configure_diagnostics(tmp_path)
     try:
@@ -181,7 +451,7 @@ def test_exception_diagnostic_keeps_code_location_not_message():
             component="hy.bridge",
             event="hy.failed",
             outcome="failed",
-            details={"prompt": secret},
+            details={"prompt": f"{secret}\n"},
         )
     except ValueError as error:
         safe = diagnostic_exception(error)
@@ -191,6 +461,28 @@ def test_exception_diagnostic_keeps_code_location_not_message():
     assert str(__file__) not in json.dumps(safe)
 
 
+def test_report_exception_never_writes_the_exception_message_to_diagnostics(
+    tmp_path, monkeypatch
+) -> None:
+    monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
+    receipt = configure_diagnostics(tmp_path)
+    secret = "sk-live-private-provider-response"
+    try:
+        report_exception(
+            "core.test",
+            "core.test.failed",
+            RuntimeError(f"provider failed with {secret} at /Users/private/query.txt"),
+        )
+    finally:
+        reset_diagnostics()
+    payload = receipt.path.read_text()
+    assert secret not in payload
+    assert "/Users/private" not in payload
+    event = json.loads(payload)
+    assert event["details"] == {"count": 1}
+    assert event["error_type"] == "RuntimeError"
+
+
 def test_old_logs_are_removed_but_symlinks_are_never_followed(tmp_path):
     log = DiagnosticLog(tmp_path)
     log.path.parent.mkdir(parents=True)

memhub/apps/core/tests/test_diagnostic_log_off_loop.py

@@ -449,7 +449,7 @@ def test_invalid_input_is_still_rejected_on_the_caller_side(tmp_path):
             component="hy.bridge",
             event="hy.failed",
             outcome="failed",
-            details={"prompt": "private"},
+            details={"invalid key": "private"},
         )
     log.close()
 

memhub/apps/core/tests/test_diagnostic_payload_keys.py

@@ -1,22 +1,57 @@
 """An instrument must never be able to break the thing it measures.
 
-`diagnostic_event` validates its payload: an unlisted `details` key, a missing
-`level`, or a detail value over 256 characters all raise. Two of our own nodes
-shipped with unlisted keys, and because they sat on a request path and on the
-loop heartbeat, they turned every external recall into a 500 and then killed
-the process. Both were caught only by running the built runtime, not by tests.
+`diagnostic_event` validates its payload: a malformed `details` key, a missing
+`level`, or an unsafe detail value is rejected. A diagnostic payload must never
+be able to break the request path or loop heartbeat it is intended to measure.
 
 So this checks the payloads at their source, by walking every literal call.
 """
 
 import ast
+import re
 from pathlib import Path
 
 import pytest
 
-from one_memory.diagnostic_log import _DETAIL_KEYS
-
 ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
+DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
+REFUSED_DETAIL_KEYS = {"__proto__", "prototype", "constructor"}
+EXPORTED_STRING_DETAIL_KEYS = {
+    "action",
+    "method",
+    "mode",
+    "phase",
+    "platform",
+    "reason",
+    "source_id",
+    "status",
+    "version",
+    "parent_session_id",
+    "runtime_state",
+    "core_state",
+    "broker_state",
+    "instance_id",
+    "operation_kind",
+    "file_kind",
+    "last_probe",
+    "signal",
+}
+PRIVATE_DETAIL_KEYS = {
+    "api_key",
+    "authorization",
+    "content",
+    "credential",
+    "memory",
+    "memory_text",
+    "message",
+    "messages",
+    "password",
+    "prompt",
+    "query",
+    "raw_text",
+    "secret",
+    "token",
+}
 
 #: Every module, not a hand-picked list. The two payloads that caused real
 #: outages were in files nobody thought to add here, so the only shape of this
@@ -33,6 +68,19 @@ def _literal_calls(path: Path):
             yield node
 
 
+def _contains_free_text_conversion(node: ast.AST) -> bool:
+    for item in ast.walk(node):
+        if (
+            isinstance(item, ast.Call)
+            and isinstance(item.func, ast.Name)
+            and item.func.id in {"str", "repr", "_safe_message"}
+        ):
+            return True
+        if isinstance(item, ast.Attribute) and item.attr in {"args", "message"}:
+            return True
+    return False
+
+
 @pytest.mark.parametrize("relative", SOURCES)
 def test_every_diagnostic_payload_is_valid(relative: str) -> None:
     path = ROOT / relative
@@ -45,7 +93,22 @@ def test_every_diagnostic_payload_is_valid(relative: str) -> None:
         for keyword in node.keywords:
             if keyword.arg != "details" or not isinstance(keyword.value, ast.Dict):
                 continue
-            for key in keyword.value.keys:
-                if isinstance(key, ast.Constant) and key.value not in _DETAIL_KEYS:
-                    problems.append(f"{relative}:{node.lineno}: unlisted details key {key.value!r}")
+            for key, expression in zip(keyword.value.keys, keyword.value.values, strict=True):
+                if not isinstance(key, ast.Constant):
+                    continue
+                value = key.value
+                if (
+                    not isinstance(value, str)
+                    or not DETAIL_KEY.fullmatch(value)
+                    or value in REFUSED_DETAIL_KEYS
+                ):
+                    problems.append(f"{relative}:{node.lineno}: invalid details key {value!r}")
+                elif value.lower() in PRIVATE_DETAIL_KEYS:
+                    problems.append(f"{relative}:{node.lineno}: private details key {value!r}")
+                elif value in EXPORTED_STRING_DETAIL_KEYS and _contains_free_text_conversion(
+                    expression
+                ):
+                    problems.append(
+                        f"{relative}:{node.lineno}: exported detail {value!r} uses free text"
+                    )
     assert not problems, "\n".join(problems)

memhub/apps/core/tests/test_fault_matrix.py

@@ -92,11 +92,13 @@ def test_each_fault_outcome_is_realised_by_the_durable_queue(tmp_path) -> None:
             self.error = error
             self.seen: list[str] = []
 
-        async def process_records(self, records, **_kwargs):
+        async def process_sdk_job(self, records, **_kwargs):
+            # Refusals are classified before `before_hy_invocation`, as the
+            # SDK write processor does, so these never cross into HY.
             self.seen.append("called")
             if self.error is not None:
                 raise self.error
-            return ()
+            return type("Result", (), {"raw_ids": (), "memory_ids": ()})()
 
     # hy_succeeded_then_crash: marked invoked, then the lease lapses -> unknown.
     path = tmp_path / "fault.sqlite3"

memhub/apps/core/tests/test_live_capture.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import json
+import os
 import sqlite3
 from contextlib import closing
 from io import StringIO
@@ -20,6 +21,7 @@ from one_memory.memory_server.durable_queue import (
     DurableWriteQueue,
     execute_job,
 )
+from one_memory.sdk_write_processor import SdkWriteReceipt
 
 
 def _grant(tmp_path, source_id: str, *, sync: bool) -> EvidenceStore:
@@ -233,14 +235,14 @@ def test_hook_job_completes_after_file_identity_adoption(tmp_path) -> None:
         ).fetchone()
 
     class IdentityCheckingSupervisor:
-        async def process_records(
+        async def process_sdk_job(
             self,
             records,
             *,
+            request_id,
             before_hy_invocation,
-            source_admitted,
         ):
-            assert source_admitted is True
+            assert request_id
             with closing(sqlite3.connect(database)) as connection:
                 connection.execute("BEGIN")
                 for record in records:
@@ -249,7 +251,7 @@ def test_hook_job_completes_after_file_identity_adoption(tmp_path) -> None:
                     assert ingest_journal.matches_current(connection, current)
                 connection.rollback()
             await before_hy_invocation()
-            return ()
+            return SdkWriteReceipt()
 
     drain = DurableWriteDrain(queue, execute=lambda _job: None)
     assert drain.claim(limit=1) == (job_id,)
@@ -560,6 +562,30 @@ def test_capture_failure_stays_silent(tmp_path, monkeypatch) -> None:
     assert output.getvalue() == ""
 
 
+def test_capture_is_a_silent_noop_during_sdk_migration(tmp_path, monkeypatch) -> None:
+    database = tmp_path / "one-memory.sqlite3"
+    marker = tmp_path / ".sdk-migration-capture-fence.json"
+    marker.write_text(json.dumps({"schema_version": 1, "pid": os.getpid()}))
+    monkeypatch.setattr("one_memory.live_capture.database_path", lambda: database)
+    monkeypatch.setattr(
+        "sys.stdin",
+        StringIO(
+            json.dumps(
+                {
+                    "hook_event_name": "beforeSubmitPrompt",
+                    "conversation_id": "session",
+                    "prompt": "not persisted during migration",
+                }
+            )
+        ),
+    )
+    output = StringIO()
+
+    assert cli.run(["capture", "cursor"], output=output) == 0
+    assert output.getvalue() == ""
+    assert not database.exists()
+
+
 def test_codex_harness_preamble_is_not_a_user_message(tmp_path) -> None:
     database = tmp_path / "one-memory.sqlite3"
     _grant(tmp_path, "codex", sync=True)

memhub/apps/core/tests/test_local_embedding_offsets.py

@@ -1,132 +0,0 @@
-import socket
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import FrozenInstanceError
-
-import pytest
-
-from one_memory import local_embedding
-from one_memory.embedding_assets import AssetVerificationError, resolve_asset_dir
-from one_memory.local_embedding import LocalEmbedder, TokenizedText
-
-
-@pytest.fixture(autouse=True)
-def no_network(monkeypatch):
-    def refused(*args, **kwargs):
-        pytest.fail("local tokenizer/encoder attempted network")
-
-    monkeypatch.setattr(socket.socket, "connect", refused)
-    monkeypatch.setattr(socket, "create_connection", refused)
-
-
-@pytest.mark.parametrize(
-    "text", ["中文😀e\u0301末尾", "👩‍💻链接", "e\u0301 cafe\u0301", "URL路径 /Case/末尾"]
-)
-def test_real_locked_tokenizer_offsets_are_unicode_codepoints(text):
-    embedder = LocalEmbedder()
-    tokenized = embedder.tokenize_with_offsets(text)
-    assert isinstance(tokenized, TokenizedText)
-    assert isinstance(tokenized.token_ids, tuple) and isinstance(tokenized.offsets, tuple)
-    ordinary = [offset for offset in tokenized.offsets if not offset.special]
-    assert ordinary
-    for offset in ordinary:
-        assert 0 <= offset.start_cp < offset.end_cp <= len(text)
-        assert text[offset.start_cp : offset.end_cp]
-    assert all(
-        any(o.start_cp <= i < o.end_cp for o in ordinary)
-        for i, character in enumerate(text)
-        if not character.isspace()
-    )
-    assert embedder._session is None and embedder._tokenizer is None
-    with pytest.raises(FrozenInstanceError):
-        ordinary[0].start_cp = 10
-
-
-def test_known_emoji_and_combining_offsets_are_not_utf8_or_utf16_units():
-    value = LocalEmbedder().tokenize_with_offsets("中文😀e\u0301末尾")
-    offsets = [(o.start_cp, o.end_cp) for o in value.offsets if not o.special]
-    assert offsets == [(0, 1), (1, 2), (2, 5), (5, 6), (6, 7)]
-
-
-def test_offsets_are_not_truncated_or_padded_and_do_not_create_onnx_session(monkeypatch):
-    monkeypatch.setattr(local_embedding, "import_module", lambda *args: pytest.fail("ONNX load"))
-    embedder = LocalEmbedder()
-    long = embedder.tokenize_with_offsets("中" * 1600)
-    short = embedder.tokenize_with_offsets("中")
-    empty = embedder.tokenize_with_offsets("")
-    assert len(long.token_ids) == 1602
-    assert len(short.token_ids) == 3
-    assert len(empty.token_ids) == 2
-    assert all(o.special for o in empty.offsets)
-    assert long.offsets[-2].end_cp == 1600
-    assert embedder._session is None
-
-
-def test_parallel_offset_calls_share_one_lazy_tokenizer_without_shared_encoder_mutation():
-    embedder = LocalEmbedder()
-    texts = ["中" * 700, "😀中e\u0301", "short"] * 8
-    with ThreadPoolExecutor(max_workers=8) as pool:
-        actual = list(pool.map(embedder.tokenize_with_offsets, texts))
-    assert actual == [embedder.tokenize_with_offsets(text) for text in texts]
-    assert embedder._session is None and embedder._tokenizer is None
-
-
-def test_offset_tokenizer_does_not_modify_encode_state_or_vectors():
-    embedder = LocalEmbedder()
-    baseline = embedder.encode(["原有编码必须保持", "短句"])
-    session, tokenizer = embedder._session, embedder._tokenizer
-    truncation, padding = tokenizer.truncation.copy(), tokenizer.padding.copy()
-    with ThreadPoolExecutor(max_workers=4) as pool:
-        tokenized = pool.submit(embedder.tokenize_with_offsets, "中" * 1100)
-        encoded = pool.submit(embedder.encode, ["原有编码必须保持", "短句"])
-        assert len(tokenized.result().token_ids) == 1102
-        assert encoded.result() == baseline
-    assert embedder._session is session and embedder._tokenizer is tokenizer
-    assert tokenizer.truncation == truncation and tokenizer.padding == padding
-    assert len(tokenizer.encode("中" * 1100).ids) == 512
-
-
-def test_offset_assets_reject_unlocked_tokenizer_without_loading_weights(tmp_path, monkeypatch):
-    (tmp_path / "tokenizer.json").write_bytes(b'{"invalid":"not locked"}')
-    monkeypatch.setattr(local_embedding, "resolve_asset_dir", lambda: tmp_path)
-    with pytest.raises(AssetVerificationError):
-        LocalEmbedder().tokenize_with_offsets("中")
-
-
-def test_offset_tokenization_uses_only_tokenizer_asset(tmp_path, monkeypatch):
-    # Tokenization must not require another ONNX session or even the weights file.
-    (tmp_path / "tokenizer.json").write_bytes((resolve_asset_dir() / "tokenizer.json").read_bytes())
-    monkeypatch.setattr(local_embedding, "resolve_asset_dir", lambda: tmp_path)
-    assert len(LocalEmbedder().tokenize_with_offsets("中文").token_ids) == 4
-
-
-def test_real_adjacent_wordpieces_and_long_ascii_unknown_keep_original_offsets():
-    embedder = LocalEmbedder()
-    word = "unaffordable"
-    parts = [o for o in embedder.tokenize_with_offsets(word).offsets if not o.special]
-    assert len(parts) > 1
-    assert parts[0].start_cp == 0 and parts[-1].end_cp == len(word)
-    assert all(a.end_cp == b.start_cp for a, b in zip(parts, parts[1:], strict=False))
-    long_word = "x" * 6000
-    unknown = embedder.tokenize_with_offsets(long_word)
-    offsets = [o for o in unknown.offsets if not o.special]
-    assert len(offsets) == 1 and unknown.token_ids[1] == 100  # locked BERT [UNK]
-    assert (offsets[0].start_cp, offsets[0].end_cp) == (0, 6000)
-
-
-def test_parallel_initial_load_constructs_one_offsets_tokenizer(monkeypatch):
-    from tokenizers import Tokenizer
-
-    calls = []
-    original = Tokenizer.from_str
-
-    def counted(value):
-        calls.append(1)
-        return original(value)
-
-    monkeypatch.setattr(Tokenizer, "from_str", counted)
-    embedder = LocalEmbedder()
-    with ThreadPoolExecutor(max_workers=8) as pool:
-        results = list(pool.map(embedder.tokenize_with_offsets, ["中😀e\u0301"] * 16))
-    assert len(calls) == 1
-    assert all(value == results[0] for value in results)
-    assert embedder._session is None

memhub/apps/core/tests/test_loop_watchdog.py

@@ -102,6 +102,8 @@ def test_stall_captures_a_safe_stack_for_every_thread() -> None:
 
     assert watchdog.stall_stacks
     captured = watchdog.stall_stacks[0]
-    assert any(line.startswith("thread=MainThread|") for line in captured)
-    assert any(line.startswith("thread=source-reader-test|") for line in captured)
+    assert captured
+    assert len(captured) <= 16
+    assert all(line.startswith("thread|") for line in captured)
+    assert all("source-reader-test" not in line and "MainThread" not in line for line in captured)
     assert all("/Users/" not in line and "\n" not in line for line in captured)

memhub/apps/core/tests/test_main_database_entry.py

@@ -87,9 +87,15 @@ ALLOWED = {
         "SQLite's own defaults, read once from a connection that opens no file",
         opened_with='":memory:"',
     ),
-    ("retrieval/vector_store.py", "RawVectorStore._preflight"): Allowed(
-        "Chroma's own chroma.sqlite3, read-only, checked before Chroma opens it",
-        opened_with='"?mode=ro"',
+    ("sdk_migration_verify.py", "legacy_memory_ids_from_bundle"): Allowed(
+        "the pre-cutover bundle's copy of the legacy database, read-only, to list "
+        "the memory ids a migrated home must still read back",
+        opened_with="?mode=ro",
+    ),
+    ("sdk_vector_reattach.py", "_stored_names"): Allowed(
+        "Chroma's own chroma.sqlite3, read-only, to list collection names before "
+        "the offline migration opens Chroma",
+        opened_with="?mode=ro",
     ),
     ("imports/sqlite_access.py", "open_sqlite_readonly"): Allowed(
         "a host app's own SQLite (Cursor state.vscdb / store.db): one direct "

memhub/apps/core/tests/test_memory_server_unified_entry.py

@@ -6,6 +6,7 @@ import asyncio
 import json
 import sqlite3
 from datetime import UTC, datetime
+from types import SimpleNamespace
 
 import pytest
 
@@ -249,16 +250,17 @@ def test_old_branch_job_executes_its_frozen_revision_after_harness_rewind(tmp_pa
     captured = []
 
     class Supervisor:
-        async def process_records(
+        async def process_sdk_job(
             self,
             records,
             *,
+            request_id,
             before_hy_invocation,
-            source_admitted,
         ):
-            assert source_admitted is True
+            assert request_id == old_job.request_id
             captured.extend(records)
             await before_hy_invocation()
+            return SimpleNamespace(raw_ids=(), memory_ids=())
 
     asyncio.run(
         execute_job(

memhub/apps/core/tests/test_model_creation_validation_boundary.py

@@ -85,7 +85,6 @@ async def application(tmp_path: Path):
         live_sync_enabled=False,
         live_sync_watch_enabled=False,
         automatic_memory_processing=False,
-        retrieval_index_enabled=False,
     )
     app = create_app(settings, source_catalog={}, secret_store=secrets)
     async with (

memhub/apps/core/tests/test_model_probe_operations.py

@@ -592,7 +592,6 @@ def test_probe_api_optional_body_lookup_and_authentication(tmp_path, monkeypatch
                 automatic_memory_processing=False,
                 live_sync_enabled=False,
                 live_sync_watch_enabled=False,
-                retrieval_index_enabled=False,
             ),
             source_catalog={},
             secret_store=secrets,

memhub/apps/core/tests/test_raw_search_without_model.py

@@ -1,15 +1,13 @@
 """Search imported conversations with no generative model configured at all.
 
-Every other retrieval test either builds the index by hand or stubs the
-encoder, so none of them answers the question a user would ask: I imported my
-history and I never set up a model, can One Memory find things I said in
-different words? This goes through the product's own entry points only:
-authorize the source, run an import job, search `/api/v1/context/search`.
+This answers the question a user would ask: I imported my history and I
+never set up a model, can One Memory find things I said? This goes through the
+product's own entry points only: authorize the source, run an import job,
+search `/api/v1/context/search`.
 """
 
 import asyncio
 import json
-import time
 from datetime import UTC, datetime
 
 import pytest
@@ -18,7 +16,6 @@ from pydantic import SecretStr
 
 from one_memory.api import create_app
 from one_memory.config import Settings
-from one_memory.embedding_assets import assets_present
 from one_memory.imports.sources.registry import build_source_catalog
 from one_memory.secrets import InMemorySecretStore
 from tests.source_consent import authorized_request, thaw_source
@@ -29,14 +26,7 @@ LICENCE = "我们最后决定 One Memory 用 MIT 协议发布，不再考虑 GPL
 BACKUP = "备份先写到本地磁盘，再同步到对象存储。"
 SYNONYM = "开源软件的授权条款最后怎么挑的"
 
-pytestmark = [
-    pytest.mark.integration,
-    pytest.mark.serial,
-    pytest.mark.requires_embedding_assets,
-    pytest.mark.skipif(
-        not assets_present(), reason="built-in embedding weights required for the semantic lane"
-    ),
-]
+pytestmark = [pytest.mark.integration, pytest.mark.serial]
 
 
 def transcript(root, name, *texts):
@@ -57,7 +47,7 @@ def transcript(root, name, *texts):
     (root / f"rollout-{name}.jsonl").write_text("\n".join(lines) + "\n")
 
 
-def application(tmp_path, *, index: bool):
+def application(tmp_path):
     home = tmp_path / "home"
     root = home / ".codex/sessions"
     root.mkdir(parents=True)
@@ -73,10 +63,8 @@ def application(tmp_path, *, index: bool):
             api_token=SecretStr(TOKEN),
             automatic_memory_processing=False,
             live_sync_enabled=False,
-            retrieval_index_enabled=index,
         ),
         source_catalog={"codex": build_source_catalog(home=home, environment={})["codex"]},
-        manage_runtime=index,
         secret_store=InMemorySecretStore(),
     )
 
@@ -106,20 +94,6 @@ async def search(client, query, *, attempts=1):
     raise AssertionError("unreachable")
 
 
-def lexical_only(app, query):
-    """What the word-matching reader alone returns, for the same request."""
-    from one_memory.context_models import ContextSearchRequest
-    from one_memory.context_raw import read_raw_context
-
-    result = read_raw_context(
-        app.state.evidence_store.database_path,
-        ContextSearchRequest(query=query, kinds=("raw",)),
-        source_ids=frozenset({"codex"}),
-        deadline=time.monotonic() + 5,
-    )
-    return tuple(result.items)
-
-
 def texts(body):
     return [
         message["text"]
@@ -131,7 +105,7 @@ def texts(body):
 
 def test_shared_words_are_found_without_any_model_or_index(tmp_path):
     async def scenario():
-        app = application(tmp_path, index=False)
+        app = application(tmp_path)
         async with (
             app.router.lifespan_context(app),
             AsyncClient(
@@ -144,38 +118,8 @@ def test_shared_words_are_found_without_any_model_or_index(tmp_path):
             assert any(LICENCE in text for text in texts(body))
 
             missed = await search(client, SYNONYM)
-            assert not texts(missed), (
-                "the lexical lane already answers this phrasing, so the semantic "
-                "test below would prove nothing"
-            )
-
-    asyncio.run(scenario())
-
-
-def test_a_semantic_only_candidate_is_not_released_without_confirmation(tmp_path):
-    async def scenario():
-        app = application(tmp_path, index=True)
-        async with (
-            app.router.lifespan_context(app),
-            AsyncClient(
-                transport=ASGITransport(app=app), base_url="http://test", headers=AUTH
-            ) as client,
-        ):
-            assert app.state.retrieval_runtime.snapshot().state == "running"
-            await imported(app, client)
-            deadline = time.monotonic() + 30
-            while time.monotonic() < deadline:
-                snapshot = app.state.retrieval_runtime.snapshot().supervisor
-                if snapshot is not None and snapshot.slices >= 4:
-                    break
-                await asyncio.sleep(1)
-            assert snapshot is not None and snapshot.slices >= 4 and snapshot.failures == 0
-            body = await search(client, SYNONYM)
-            assert body["paths"]["raw"]["reason"] == "lexical_only", body
-            assert not texts(body), "a semantic-only candidate must not release original text"
-            assert not lexical_only(app, SYNONYM), "lexical alone answers this; nothing is proven"
-            assert any(LICENCE in text for text in texts(await search(client, "MIT 协议"))), (
-                "the index must not cost the lexical lane its own hits"
-            )
+            # Different words are not a lexical match, and there is no private
+            # vector index to pretend otherwise.
+            assert not texts(missed)
 
     asyncio.run(scenario())

memhub/apps/core/tests/test_retrieval_chunking.py

@@ -1,273 +0,0 @@
-from dataclasses import FrozenInstanceError
-
-import pytest
-
-from one_memory.local_embedding import LocalEmbedder, TokenizedText, TokenOffset
-from one_memory.retrieval.chunking import ChunkingError, chunk_evidence
-
-
-class Characters:
-    def tokenize_with_offsets(self, text):
-        offsets = (
-            TokenOffset(0, 0, True),
-            *(TokenOffset(i, i + 1) for i in range(len(text))),
-            TokenOffset(0, 0, True),
-        )
-        return TokenizedText(tuple(range(len(offsets))), offsets)
-
-
-def assert_coverage(text, chunks, tokenizer, prefix=""):
-    assert chunks
-    assert [chunk.ordinal for chunk in chunks] == list(range(len(chunks)))
-    covered = bytearray(len(text))
-    previous_start = -1
-    for chunk in chunks:
-        assert 0 <= chunk.start_cp < chunk.end_cp <= len(text)
-        assert chunk.start_cp > previous_start
-        previous_start = chunk.start_cp
-        assert chunk.text == text[chunk.start_cp : chunk.end_cp]
-        assert chunk.token_count == len(
-            tokenizer.tokenize_with_offsets(prefix + chunk.text).token_ids
-        )
-        assert chunk.token_count <= 512
-        covered[chunk.start_cp : chunk.end_cp] = b"\x01" * (chunk.end_cp - chunk.start_cp)
-    assert all(covered[i] for i, c in enumerate(text) if not c.isspace())
-    assert chunks[-1].end_cp == len(text)
-
-
-def test_default_384_body_64_overlap_and_exact_source_slices():
-    text = "中" * 1000
-    chunks = chunk_evidence(text, Characters())
-    assert [(c.start_cp, c.end_cp) for c in chunks] == [(0, 384), (320, 704), (640, 1000)]
-    assert chunks[0].token_count == 386
-    assert_coverage(text, chunks, Characters())
-    with pytest.raises(FrozenInstanceError):
-        chunks[0].text = "replacement"
-
-
-def test_all_prefix_and_special_tokens_count_toward_strict_512_budget():
-    prefix = "P" * 200
-    text = "中" * 900
-    chunks = chunk_evidence(text, Characters(), prefix=prefix)
-    assert chunks[0].end_cp == 310
-    assert chunks[0].token_count == 512
-    assert_coverage(text, chunks, Characters(), prefix)
-    with pytest.raises(ChunkingError, match="budget"):
-        chunk_evidence("中", Characters(), prefix="P" * 510)
-
-
-def test_real_tokenizer_covers_long_chinese_middle_and_tail_object_locations():
-    middle = "https://example.test/Middle?Key=Value#Anchor"
-    tail = "/CaseSensitive/路径/final.txt"
-    text = "中文😀e\u0301\n" + "中" * 740 + "\n" + middle + "\n" + "尾" * 800 + "\n" + tail
-    tokenizer = LocalEmbedder()
-    chunks = chunk_evidence(text, tokenizer)
-    assert len(chunks) > 3
-    assert_coverage(text, chunks, tokenizer)
-    for value in (middle, tail):
-        location = text.index(value)
-        assert any(c.start_cp <= location and c.end_cp >= location + len(value) for c in chunks)
-    assert chunks[0].text.startswith("中文😀e\u0301\n")
-
-
-def test_prefers_paragraph_boundary_near_target_without_losing_text():
-    text = "中" * 300 + "\n\n" + "尾" * 500
-    chunks = chunk_evidence(text, Characters())
-    assert chunks[0].end_cp == 302
-    assert_coverage(text, chunks, Characters())
-
-
-@pytest.mark.parametrize("text", ["", " \n\t\r "])
-def test_empty_or_whitespace_has_no_index_chunks(text):
-    assert chunk_evidence(text, Characters()) == ()
-
-
-@pytest.mark.parametrize(
-    "text", ["😀" * 300, "e\u0301" * 400, "甲\n乙\t丙", "原样 <user_query>原文</user_query>"]
-)
-def test_unicode_and_wrapper_text_identity_is_preserved(text):
-    chunks = chunk_evidence(text, Characters())
-    assert_coverage(text, chunks, Characters())
-
-
-@pytest.mark.parametrize(
-    "offsets,ids",
-    [
-        ((TokenOffset(0, 4),), (1,)),
-        ((TokenOffset(-1, 1),), (1,)),
-        ((TokenOffset(1, 2), TokenOffset(0, 1)), (1, 2)),
-        ((TokenOffset(0, 1), TokenOffset(2, 3)), (1, 2)),
-        ((TokenOffset(0, 0),), (1,)),
-        ((TokenOffset(0, 1, True),), (1,)),
-        ((TokenOffset(0, 3),), (1, 2)),
-        ((), ()),
-    ],
-)
-def test_bad_or_incomplete_offsets_fail_explicitly(offsets, ids):
-    class Broken:
-        def tokenize_with_offsets(self, text):
-            return TokenizedText(ids, offsets)
-
-    with pytest.raises(ChunkingError):
-        chunk_evidence("abc", Broken())
-
-
-@pytest.mark.parametrize(
-    "kwargs",
-    [
-        {"target_tokens": 64},
-        {"overlap_tokens": -1},
-        {"max_tokens": 513},
-        {"max_tokens": True},
-        {"target_tokens": 0},
-        {"overlap_tokens": 384},
-    ],
-)
-def test_invalid_budgets_fail_before_tokenizing(kwargs):
-    with pytest.raises(ChunkingError):
-        chunk_evidence("content", Characters(), **kwargs)
-
-
-def test_retokenizes_actual_prefix_plus_slice_instead_of_assuming_additive_counts():
-    class ContextSensitive(Characters):
-        def tokenize_with_offsets(self, text):
-            result = super().tokenize_with_offsets(text)
-            if text.startswith("P"):
-                # Same valid offsets may carry multiple pieces. Budget must include them all.
-                return TokenizedText(
-                    result.token_ids * 2,
-                    tuple(o for o in result.offsets if o.special)
-                    + tuple(o for o in result.offsets if o.special)
-                    + tuple(o for o in result.offsets if not o.special for _ in range(2)),
-                )
-            return result
-
-    tokenizer = ContextSensitive()
-    chunks = chunk_evidence("中" * 600, tokenizer, prefix="P")
-    assert_coverage("中" * 600, chunks, tokenizer, "P")
-
-
-def test_overlapping_token_offsets_are_not_split_inside_a_character():
-    class Repeated(Characters):
-        def tokenize_with_offsets(self, text):
-            return TokenizedText(
-                tuple(range(2 * len(text))),
-                tuple(TokenOffset(i, i + 1) for i in range(len(text)) for _ in range(2)),
-            )
-
-    tokenizer = Repeated()
-    text = "😀" * 500
-    chunks = chunk_evidence(text, tokenizer)
-    assert_coverage(text, chunks, tokenizer)
-
-
-def test_single_indivisible_offset_over_budget_is_not_truncated():
-    class Indivisible:
-        def tokenize_with_offsets(self, text):
-            return TokenizedText(
-                tuple(range(600)), tuple(TokenOffset(0, len(text)) for _ in range(600))
-            )
-
-    with pytest.raises(ChunkingError, match="budget"):
-        chunk_evidence("😀", Indivisible())
-
-
-def test_real_long_continuous_ascii_unknown_preserves_all_6000_characters():
-    tokenizer = LocalEmbedder()
-    text = " \t" + "x" * 6000 + "\n末尾链接 https://example.test/Case?A=B#C\t"
-    chunks = chunk_evidence(text, tokenizer)
-    assert len(chunks) == 1
-    assert chunks[0].text == text
-    assert_coverage(text, chunks, tokenizer)
-
-
-def test_real_subword_and_whitespace_boundaries_have_no_gaps():
-    tokenizer = LocalEmbedder()
-    text = " \t\n" + ("unaffordable embeddings\t中\u0301文  a\u0301b\n\n" * 180) + "\t\n "
-    prefix = "本段用于检索："
-    chunks = chunk_evidence(text, tokenizer, prefix=prefix)
-    assert len(chunks) > 3
-    assert chunks[0].start_cp == 0 and chunks[-1].end_cp == len(text)
-    assert_coverage(text, chunks, tokenizer, prefix)
-    for first, second in zip(chunks, chunks[1:], strict=False):
-        assert second.start_cp <= first.end_cp
-
-
-def test_partially_overlapping_offsets_for_combining_cluster_are_indivisible():
-    class Combining(Characters):
-        def tokenize_with_offsets(self, text):
-            offsets = []
-            i = 0
-            while i < len(text):
-                if text.startswith("e\u0301", i):
-                    offsets.extend((TokenOffset(i, i + 2), TokenOffset(i + 1, i + 2)))
-                    i += 2
-                else:
-                    offsets.append(TokenOffset(i, i + 1))
-                    i += 1
-            return TokenizedText(tuple(range(len(offsets))), tuple(offsets))
-
-    text = "中" * 383 + "e\u0301" + "尾" * 500
-    tokenizer = Combining()
-    chunks = chunk_evidence(text, tokenizer)
-    assert_coverage(text, chunks, tokenizer)
-    assert all(c.start_cp != 384 and c.end_cp != 384 for c in chunks)
-
-
-def test_sparse_whitespace_gaps_are_retained_not_rebuilt_from_tokens():
-    tokenizer = LocalEmbedder()
-    text = " \n" + "中" * 380 + "\n \t\n" + "尾" * 380 + " \t\r\n"
-    chunks = chunk_evidence(text, tokenizer)
-    assert_coverage(text, chunks, tokenizer)
-    assert chunks[0].text.startswith(" \n")
-    assert chunks[-1].text.endswith(" \t\r\n")
-
-
-def test_real_prefix_retokenization_never_exceeds_encoder_window():
-    tokenizer = LocalEmbedder()
-    prefix = "检索前缀" * 50
-    text = ("中文tokenization😀e\u0301\n" * 120) + "https://example.test/Tail"
-    chunks = chunk_evidence(text, tokenizer, prefix=prefix)
-    assert len(chunks) > 1
-    assert_coverage(text, chunks, tokenizer, prefix)
-
-
-def test_tokenizer_failure_is_explicit_not_partial_output():
-    class Broken:
-        def tokenize_with_offsets(self, text):
-            raise RuntimeError("tokenizer unavailable")
-
-    with pytest.raises(ChunkingError, match="tokenization_failed"):
-        chunk_evidence("中", Broken())
-
-
-@pytest.mark.parametrize(
-    "text", ["e\u0301", "\u0301", "中\u0301文", "\n\r\t中\n尾\n", "👩‍💻", "👩‍👩‍👧‍👦", "✈️"]
-)
-def test_locked_normalizer_supported_unicode_examples_cover_original(text):
-    tokenizer = LocalEmbedder()
-    assert_coverage(text, chunk_evidence(text, tokenizer), tokenizer)
-
-
-@pytest.mark.parametrize("text", ["中\u200d文", "\u200d", "中\u200b文", "中\x00文", "\x00"])
-def test_locked_normalizer_dropped_nonwhitespace_is_explicitly_unsupported(text):
-    tokenizer = LocalEmbedder()
-    offsets = tokenizer.tokenize_with_offsets(text).offsets
-    assert any(
-        not character.isspace()
-        and not any(
-            not offset.special and offset.start_cp <= i < offset.end_cp for offset in offsets
-        )
-        for i, character in enumerate(text)
-    )
-    with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):
-        chunk_evidence(text, tokenizer)
-
-
-def test_another_normalizer_dropping_trailing_combining_mark_is_not_reported_complete():
-    class NormalizesAccent:
-        def tokenize_with_offsets(self, text):
-            return TokenizedText((1,), (TokenOffset(0, 1),))
-
-    with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):
-        chunk_evidence("e\u0301", NormalizesAccent())

memhub/apps/core/tests/test_retrieval_eligibility_benchmark.py

@@ -1,38 +0,0 @@
-import importlib.util
-import sqlite3
-import sys
-from pathlib import Path
-
-import pytest
-
-pytestmark = [pytest.mark.performance, pytest.mark.serial]
-
-SCRIPT = Path(__file__).resolve().parents[3] / "scripts" / "bench_retrieval_eligibility_scale.py"
-if not SCRIPT.exists():
-    pytest.skip(
-        f"benchmark script is not part of this checkout: {SCRIPT.name}",
-        allow_module_level=True,
-    )
-spec = importlib.util.spec_from_file_location("bench_retrieval_eligibility_scale", SCRIPT)
-benchmark = importlib.util.module_from_spec(spec)
-sys.modules["bench_retrieval_eligibility_scale"] = benchmark
-assert spec.loader is not None
-spec.loader.exec_module(benchmark)
-
-
-def test_scale_probe_proves_composite_authority_index_changes_the_plan(tmp_path):
-    db = sqlite3.connect(tmp_path / "probe.sqlite3", cached_statements=0)
-    try:
-        benchmark.create_schema(db)
-        benchmark.fill_to(db, 0, 10_000)
-        before = benchmark.measure(db, benchmark.FILTERED_SQL, ("g", "allowed"), repeat=3)
-        db.execute(
-            "CREATE INDEX retrieval_authority_jobs "
-            "ON retrieval_documents(generation_id,state,source_id,event_seq)"
-        )
-        after = benchmark.measure(db, benchmark.FILTERED_SQL, ("g", "allowed"), repeat=3)
-    finally:
-        db.close()
-    assert before["rows"] == after["rows"] == 100
-    assert not any("source_id" in step for step in before["plan"])
-    assert any("retrieval_authority_jobs" in step for step in after["plan"])

memhub/apps/core/tests/test_retrieval_index_store.py

@@ -1,452 +0,0 @@
-import sqlite3
-from contextlib import closing
-from datetime import UTC, datetime
-
-import pytest
-
-from one_memory import ingest_journal
-from one_memory.imports.grants import SourceGrantStore
-from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
-from one_memory.imports.sources.registry import RootCandidateKind
-from one_memory.imports.storage import EvidenceStore
-from one_memory.memory_server.config_state import ConfigStore
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.index_store import EncoderIdentity, RetrievalIndexStore
-from one_memory.retrieval.models import TimeRange
-
-
-@pytest.fixture
-def fixture(tmp_path):
-    path = tmp_path / "data.sqlite3"
-    evidence = EvidenceStore(path)
-    grants = SourceGrantStore(path, home=tmp_path)
-    source = tmp_path / "source"
-    source.mkdir()
-    grants.grant(
-        user_id="local", source_id="codex", root=source, candidate_kind=RootCandidateKind.DEFAULT
-    )
-    store = RetrievalIndexStore(path)
-    identity = EncoderIdentity("test", "a" * 64, "b" * 64, 3)
-    return path, evidence, grants, store, identity
-
-
-def put(evidence, text="original text", message="one"):
-    return evidence.put(
-        EvidenceInput(
-            source_id="codex",
-            conversation_id="thread",
-            message_id=message,
-            role="user",
-            written_by=WrittenBy.PERSON,
-            verbatim=text,
-            origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
-        )
-    ).record
-
-
-def initialize(f):
-    path, _, _, store, identity = f
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        store.initialize(db)
-        return store.create_generation(db, identity)
-
-
-def test_constructor_and_reads_do_not_create_database_or_schema(tmp_path):
-    store = RetrievalIndexStore(tmp_path / "missing")
-    assert not store.database_path.exists()
-    with pytest.raises(sqlite3.Error):
-        store.status("missing")
-    assert not store.database_path.exists()
-
-
-def test_explicit_initialization_rolls_back_and_never_grants_model_policy(fixture):
-    path, evidence, _, store, _ = fixture
-    row = put(evidence)
-    with closing(sqlite3.connect(path)) as db:
-        with pytest.raises(RuntimeError):
-            store.initialize(db)
-        db.execute("BEGIN IMMEDIATE")
-        store.initialize(db)
-        event = ingest_journal.current(db, row.evidence_id)
-        assert not event.generation_eligible and event.kind == "migration_baseline"
-        db.rollback()
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
-        ).fetchone()
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='evidence_ingest_events'"
-        ).fetchone()
-    initialize(fixture)
-    with closing(sqlite3.connect(path)) as db:
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='continuous_policy_history'"
-        ).fetchone()
-
-
-def test_consumer_and_jobs_commit_together(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    with closing(sqlite3.connect(path)) as db:
-        db.execute("BEGIN IMMEDIATE")
-        assert store.consume(db, generation) == 1
-        assert db.execute("SELECT COUNT(*) FROM retrieval_documents").fetchone()[0] == 1
-        db.rollback()
-    assert store.status(generation)["after_seq"] == 0
-    assert store.consume_pending(generation) == 1
-    assert store.consume_pending(generation) == 0
-    lease = store.claim(generation, "worker", now=100)
-    assert lease.evidence_id == row.evidence_id
-
-
-def test_lease_expiry_old_owner_and_publish_manifest(fixture):
-    _, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "one", now=100, lease_seconds=10)
-    assert store.claim(generation, "two", now=105) is None
-    replacement = store.claim(generation, "two", now=111)
-    chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
-    with pytest.raises(ValueError, match="lease"):
-        store.publish(lease, chunks, vectors_verified=True, now=112)
-    with pytest.raises(ValueError, match="vectors"):
-        store.publish(replacement, chunks, vectors_verified=False, now=112)
-    store.publish(replacement, chunks, vectors_verified=True, now=112)
-    assert store.status(generation)["ready"] == 1
-    assert store.active_chunks(generation) == ()
-    store.activate_generation(generation)
-    assert len(store.active_chunks(generation)) == 1
-    assert store.claim(generation, "three", now=113) is None
-
-
-@pytest.mark.parametrize("change", ["revision", "unrecorded", "revoke", "deleted"])
-def test_changed_inputs_cannot_publish_or_be_read(fixture, change):
-    path, evidence, grants, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    if change == "revision":
-        put(evidence, "updated text")
-    elif change == "unrecorded":
-        with closing(sqlite3.connect(path)) as db, db:
-            db.execute(
-                "UPDATE evidence SET verbatim='unrecorded' WHERE evidence_id=?", (row.evidence_id,)
-            )
-    elif change == "revoke":
-        with closing(sqlite3.connect(path)) as db, db:
-            db.execute("UPDATE source_grants SET revoked_at='2026-09-19' WHERE source_id='codex'")
-    else:
-        with closing(sqlite3.connect(path)) as db, db:
-            db.execute("DELETE FROM evidence_origins WHERE evidence_id=?", (row.evidence_id,))
-            db.execute("DELETE FROM evidence WHERE evidence_id=?", (row.evidence_id,))
-            ingest_journal.record(db, row.evidence_id, cause="delete")
-    with pytest.raises(ValueError):
-        store.publish(
-            lease,
-            (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
-            vectors_verified=True,
-            now=101,
-        )
-    assert store.active_chunks(generation) == ()
-
-
-def test_current_read_rejects_revised_published_content_before_consumer_catches_up(fixture):
-    _, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    store.publish(
-        lease,
-        (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
-        vectors_verified=True,
-        now=101,
-    )
-    store.activate_generation(generation)
-    assert store.active_chunks(generation)
-    put(evidence, "new version")
-    assert store.active_chunks(generation) == ()
-    store.consume_pending(generation)
-    updated = store.claim(generation, "worker", now=102)
-    assert updated.build_id != lease.build_id
-
-
-def test_pausing_harness_sync_does_not_hide_published_memory(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    store.publish(
-        lease,
-        (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
-        vectors_verified=True,
-        now=101,
-    )
-    store.activate_generation(generation)
-    before = store.active_chunks(generation)
-    assert before
-
-    config = ConfigStore(path.parent)
-    config.set_sync("codex", True)
-    config.set_sync("codex", False)
-
-    assert store.active_chunks(generation) == before
-
-
-def test_partial_or_fabricated_chunk_manifest_never_publishes(fixture):
-    _, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    for chunks in [
-        (),
-        (TextChunk(0, 0, 3, 3, "ori"),),
-        (TextChunk(0, 0, len(row.verbatim), 4, "not original"),),
-    ]:
-        with pytest.raises(ValueError, match="chunk"):
-            store.publish(lease, chunks, vectors_verified=True, now=101)
-    assert store.status(generation)["ready"] == 0
-
-
-def test_incomplete_index_cannot_replace_existing_active_generation(fixture):
-    path, evidence, _, store, identity = fixture
-    first = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(first)
-    lease = store.claim(first, "worker", now=100)
-    chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
-    store.publish(lease, chunks, vectors_verified=True, now=101)
-    store.activate_generation(first)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        second = store.create_generation(db, EncoderIdentity("second", "c" * 64, "d" * 64, 3))
-    with pytest.raises(ValueError, match="caught_up"):
-        store.activate_generation(second)
-    assert store.active_chunks(first)
-    store.consume_pending(second)
-    with pytest.raises(ValueError, match="incomplete"):
-        store.activate_generation(second)
-    claim = store.claim(second, "worker", now=102)
-    store.publish(claim, chunks, vectors_verified=True, now=103)
-    store.activate_generation(second)
-    assert store.active_chunks(first) == () and store.active_chunks(second)
-
-
-def test_block_and_explicit_retry_require_same_lease_and_authority(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    put(evidence)
-    store.consume_pending(generation)
-    first = store.claim(generation, "one", now=100, lease_seconds=1)
-    second = store.claim(generation, "two", now=102)
-    store.block(first)
-    assert store.status(generation)["leased"] == 1
-    store.block(second)
-    assert store.status(generation)["blocked"] == 1
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("UPDATE source_grants SET revoked_at='revoked'")
-    assert store.retry_blocked(generation) == 0
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("UPDATE source_grants SET revoked_at=NULL")
-    assert store.retry_blocked(generation) == 1
-    assert store.claim(generation, "three", now=103) is not None
-
-
-def test_scope_filter_and_revocation_apply_before_exposing_manifest(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    assert store.read_lease(lease, now=101).verbatim == row.verbatim
-    store.publish(
-        lease,
-        (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),),
-        vectors_verified=True,
-        now=101,
-    )
-    store.activate_generation(generation)
-    assert store.active_chunks(generation, source_ids=frozenset()) == ()
-    assert store.active_chunks(generation, source_ids=frozenset({"codex"}))
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("UPDATE source_grants SET revision=revision+1")
-    assert store.active_chunks(generation) == ()
-
-
-def test_source_and_time_eligibility_are_applied_before_manifest_validation(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
-    store.publish(lease, chunks, vectors_verified=True, now=101)
-    store.activate_generation(generation)
-
-    assert store.active_chunks(generation, source_ids=frozenset()) == ()
-    assert store.active_chunks(generation, source_ids=frozenset({"other"})) == ()
-    assert len(store.active_chunks(generation, source_ids=frozenset({"codex"}))) == 1
-    assert (
-        store.active_chunks(
-            generation,
-            time_range=TimeRange(
-                start=datetime(2030, 1, 1, tzinfo=UTC),
-                end=datetime(2030, 1, 2, tzinfo=UTC),
-            ),
-        )
-        == ()
-    )
-
-
-def test_generation_and_metadata_survive_reopen_without_body_copy(fixture):
-    path, evidence, _, store, identity = fixture
-    generation = initialize(fixture)
-    put(evidence, "sensitive-body-must-not-copy")
-    store.consume_pending(generation)
-    reopened = RetrievalIndexStore(path)
-    assert reopened.status(generation) == store.status(generation)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        reopened.initialize(db)
-        assert reopened.create_generation(db, identity) == generation
-        rows = db.execute("SELECT * FROM retrieval_documents").fetchall()
-        assert "sensitive-body-must-not-copy" not in str(rows)
-
-
-def test_partial_schema_is_not_blessed_as_new_instance(fixture):
-    path, _, _, store, _ = fixture
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("CREATE TABLE retrieval_chunks (unexpected TEXT)")
-    with closing(sqlite3.connect(path)) as db:
-        db.execute("BEGIN IMMEDIATE")
-        with pytest.raises(ValueError, match="partial_initialization"):
-            store.initialize(db)
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_state'"
-        ).fetchone()
-
-
-def test_lease_renewal_rechecks_authority_and_rejects_old_expiry(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    renewed = store.renew(lease, now=125)
-    assert renewed.expires_at == 185
-    with pytest.raises(ValueError, match="lease_changed"):
-        store.read_lease(lease, now=126)
-    assert store.read_lease(renewed, now=126)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("UPDATE source_grants SET revision=revision+1")
-    with pytest.raises(ValueError, match="authority"):
-        store.renew(renewed, now=127)
-
-
-def test_regrant_requeues_and_reuses_immutable_manifest(fixture):
-    path, evidence, grants, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
-    store.publish(lease, chunks, vectors_verified=True, now=101)
-    store.activate_generation(generation)
-    assert len(store.active_chunks(generation)) == 1
-    grants.revoke(user_id="local", source_id="codex")
-    assert store.active_chunks(generation) == ()
-    grants.grant(
-        user_id="local",
-        source_id="codex",
-        root=path.parent / "source",
-        candidate_kind=RootCandidateKind.DEFAULT,
-    )
-    assert store.active_chunks(generation) == ()
-    assert store.retry_blocked(generation) == 1
-    renewed = store.claim(generation, "worker", now=102)
-    assert renewed.build_id == lease.build_id
-    store.publish(renewed, chunks, vectors_verified=True, now=103)
-    assert len(store.active_chunks(generation)) == 1
-
-
-def test_retry_cursor_passes_ineligible_prefix_across_reopen(fixture):
-    path, evidence, grants, store, _ = fixture
-    generation = initialize(fixture)
-    for index in range(257):
-        put(evidence, message=str(index))
-    store.consume_pending(generation)
-    store.consume_pending(generation)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("UPDATE retrieval_documents SET state='blocked',source_id='revoked'")
-        last_id = db.execute(
-            "SELECT evidence_id FROM retrieval_documents ORDER BY event_seq DESC LIMIT 1"
-        ).fetchone()[0]
-        db.execute(
-            "UPDATE retrieval_documents SET source_id='codex' WHERE evidence_id=?", (last_id,)
-        )
-    assert store.retry_blocked(generation) == 0
-    reopened = RetrievalIndexStore(path)
-    assert reopened.retry_blocked(generation) == 1
-    assert reopened.claim(generation, "worker", now=100).evidence_id == last_id
-
-
-def test_same_build_replay_requires_exact_complete_manifest(fixture):
-    path, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    row = put(evidence)
-    store.consume_pending(generation)
-    lease = store.claim(generation, "worker", now=100)
-    chunks = (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),)
-    store.publish(lease, chunks, vectors_verified=True, now=101)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("UPDATE retrieval_documents SET state='pending',active_build_id=NULL")
-    replay = store.claim(generation, "worker", now=102)
-    with pytest.raises(ValueError, match="manifest"):
-        store.publish(
-            replay,
-            (TextChunk(0, 0, len(row.verbatim), 5, row.verbatim),),
-            vectors_verified=True,
-            now=103,
-        )
-    assert store.status(generation)["ready"] == 0
-    store.publish(replay, chunks, vectors_verified=True, now=103)
-    assert store.status(generation)["ready"] == 1
-
-
-def test_origin_metadata_roundtrip_reuses_old_build_without_losing_new_event(fixture):
-    _, evidence, _, store, _ = fixture
-    generation = initialize(fixture)
-    builds = []
-    seqs = []
-    for version in ("a", "b", "a"):
-        row = evidence.put(
-            EvidenceInput(
-                source_id="codex",
-                conversation_id="thread",
-                message_id="one",
-                role="user",
-                written_by=WrittenBy.PERSON,
-                verbatim="unchanged original",
-                origin_meta={EvidenceOrigin.FILE: OriginMetadata(source_version=version)},
-            )
-        ).record
-        assert row.revision == 1
-        assert store.consume_pending(generation) == 1
-        lease = store.claim(generation, "worker", now=100)
-        builds.append(lease.build_id)
-        seqs.append(lease.event_seq)
-        store.publish(
-            lease,
-            (TextChunk(0, 0, len(row.verbatim), 5, row.verbatim),),
-            vectors_verified=True,
-            now=101,
-        )
-    assert builds[0] == builds[2] and builds[0] != builds[1]
-    assert seqs[0] < seqs[1] < seqs[2]
-    store.activate_generation(generation)
-    assert len(store.active_chunks(generation)) == 1

memhub/apps/core/tests/test_retrieval_index_supervisor.py

@@ -1,357 +0,0 @@
-import asyncio
-import socket
-import threading
-import time
-from dataclasses import FrozenInstanceError
-
-import pytest
-
-from one_memory.retrieval.index_supervisor import RetrievalIndexSupervisor
-from one_memory.retrieval.index_worker import IndexWorkResult
-
-
-@pytest.fixture(autouse=True)
-def no_network(monkeypatch):
-    def reject(*args, **kwargs):
-        raise AssertionError("supervisor tests must remain offline")
-
-    monkeypatch.setattr(socket.socket, "connect", reject)
-    monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-class Worker:
-    def __init__(self, state="idle", blocking=False):
-        self.state = state
-        self.entered = threading.Event()
-        self.release = threading.Event()
-        self.exited = threading.Event()
-        self.cancel = None
-        self.blocking = blocking
-        self.calls = []
-        self.thread_ids = set()
-        self.active = 0
-        self.maximum = 0
-        self.closed = False
-
-    def run_once(self, *, cancel=None):
-        self.cancel = cancel
-        self.active += 1
-        self.maximum = max(self.maximum, self.active)
-        self.calls.append(time.monotonic())
-        self.thread_ids.add(threading.get_ident())
-        self.entered.set()
-        try:
-            if self.blocking:
-                assert self.release.wait(3), "test cleanup failed"
-            if self.state == "exception":
-                raise RuntimeError("SECRET_RAW_ERROR")
-            if self.state == "invalid":
-                return object()
-            return IndexWorkResult(self.state, "SECRET_REASON")
-        finally:
-            self.active -= 1
-            self.exited.set()
-
-    def close(self):
-        self.closed = True
-
-
-async def until(predicate):
-    deadline = time.monotonic() + 2
-    while not predicate():
-        assert time.monotonic() < deadline
-        await asyncio.sleep(0.002)
-
-
-def test_stop_waits_for_native_exit_and_does_not_close_resources():
-    async def run():
-        worker = Worker(blocking=True)
-        supervisor = RetrievalIndexSupervisor(worker)
-        supervisor.start()
-        await until(worker.entered.is_set)
-        stopping = asyncio.create_task(supervisor.stop())
-        try:
-            await until(lambda: worker.cancel.is_set())
-            await asyncio.sleep(0.03)
-            assert not stopping.done() and not worker.exited.is_set()
-            assert supervisor.snapshot().thread_alive
-        finally:
-            worker.release.set()
-            await stopping
-        assert worker.exited.is_set()
-        assert not supervisor.snapshot().thread_alive
-        assert not worker.closed
-        assert supervisor.snapshot().state == "stopped"
-
-    asyncio.run(run())
-
-
-@pytest.mark.parametrize("method", ["stop", "wait"])
-def test_async_cancellation_repeatedly_waits_for_native_thread(method):
-    async def run():
-        worker = Worker(blocking=True)
-        supervisor = RetrievalIndexSupervisor(worker)
-        supervisor.start()
-        await until(worker.entered.is_set)
-        task = asyncio.create_task(getattr(supervisor, method)())
-        await asyncio.sleep(0)
-        task.cancel()
-        try:
-            await until(lambda: worker.cancel.is_set())
-            task.cancel()
-            await asyncio.sleep(0.03)
-            assert not task.done()
-            assert not worker.exited.is_set()
-        finally:
-            worker.release.set()
-            with pytest.raises(asyncio.CancelledError):
-                await task
-            await supervisor.stop()
-        assert worker.exited.is_set() and not supervisor.snapshot().thread_alive
-        assert not worker.closed
-
-    asyncio.run(run())
-
-
-def test_idle_no_trigger_no_spin_and_duplicate_start_rejected():
-    async def run():
-        worker = Worker()
-        supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.3)
-        assert not supervisor.trigger()
-        assert worker.calls == []
-        supervisor.start()
-        try:
-            with pytest.raises(RuntimeError, match="index_supervisor_already_started"):
-                supervisor.start()
-            await until(lambda: len(worker.calls) == 1)
-            await asyncio.sleep(0.06)
-            assert len(worker.calls) == 1
-            assert supervisor.snapshot().last_result == "idle"
-        finally:
-            await supervisor.stop()
-        with pytest.raises(RuntimeError, match="index_supervisor_already_started"):
-            supervisor.start()
-        assert not supervisor.trigger()
-
-    asyncio.run(run())
-
-
-def test_trigger_storm_coalesces_during_native_call_without_parallel_workers():
-    async def run():
-        worker = Worker(blocking=True)
-        supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.5)
-        supervisor.start()
-        await until(worker.entered.is_set)
-        try:
-            for _ in range(200):
-                assert supervisor.trigger()
-            worker.release.set()
-            await until(lambda: len(worker.calls) >= 2)
-            await asyncio.sleep(0.05)
-            assert len(worker.calls) == 2
-            assert worker.maximum == 1 and len(worker.thread_ids) == 1
-        finally:
-            worker.release.set()
-            await supervisor.stop()
-
-    asyncio.run(run())
-
-
-def test_interactive_activity_pauses_index_slices_then_resumes() -> None:
-    async def run() -> None:
-        worker = Worker()
-        interactive = True
-        supervisor = RetrievalIndexSupervisor(
-            worker,
-            yield_wait_seconds=0.02,
-            background_should_yield=lambda: interactive,
-        )
-        supervisor.start()
-        try:
-            await asyncio.sleep(0.06)
-            assert worker.calls == []
-            interactive = False
-            supervisor.trigger()
-            await until(lambda: len(worker.calls) == 1)
-        finally:
-            await supervisor.stop()
-
-    asyncio.run(run())
-
-
-@pytest.mark.parametrize(
-    "state,code",
-    [
-        ("exception", "index_worker_failed"),
-        ("invalid", "index_worker_invalid_result"),
-        ("blocked", "index_worker_blocked"),
-    ],
-)
-def test_failures_have_fixed_error_and_backoff_even_with_triggers(state, code):
-    async def run():
-        worker = Worker(state)
-        supervisor = RetrievalIndexSupervisor(worker, failure_wait_seconds=0.2)
-        supervisor.start()
-        try:
-            await until(lambda: supervisor.snapshot().error_code is not None)
-            for _ in range(100):
-                supervisor.trigger()
-            await asyncio.sleep(0.05)
-            assert len(worker.calls) == 1
-            snapshot = supervisor.snapshot()
-            assert snapshot.error_code == code
-            assert "SECRET" not in repr(snapshot)
-            await until(lambda: len(worker.calls) >= 2)
-            assert worker.calls[1] - worker.calls[0] >= 0.18
-        finally:
-            await supervisor.stop()
-
-    asyncio.run(run())
-
-
-@pytest.mark.parametrize("state", ["ready", "excluded"])
-def test_backlog_yields_after_maximum_slices(state):
-    async def run():
-        worker = Worker(state)
-        supervisor = RetrievalIndexSupervisor(
-            worker, max_consecutive_slices=3, yield_wait_seconds=0.15
-        )
-        supervisor.start()
-        try:
-            await until(lambda: len(worker.calls) >= 3)
-            for _ in range(50):
-                supervisor.trigger()
-            await asyncio.sleep(0.04)
-            assert len(worker.calls) == 3
-            await until(lambda: len(worker.calls) >= 4)
-            assert worker.calls[3] - worker.calls[2] >= 0.13
-        finally:
-            await supervisor.stop()
-        assert worker.maximum == 1
-
-    asyncio.run(run())
-
-
-def test_concurrent_stop_is_idempotent_and_stop_before_start_is_terminal():
-    async def run():
-        worker = Worker()
-        supervisor = RetrievalIndexSupervisor(worker)
-        supervisor.start()
-        await asyncio.gather(supervisor.stop(), supervisor.stop())
-        assert not supervisor.snapshot().thread_alive
-        unused = RetrievalIndexSupervisor(worker)
-        await unused.stop()
-        with pytest.raises(RuntimeError, match="index_supervisor_already_started"):
-            unused.start()
-
-    asyncio.run(run())
-
-
-@pytest.mark.parametrize(
-    "kwargs",
-    [
-        {"idle_wait_seconds": 0},
-        {"failure_wait_seconds": float("nan")},
-        {"yield_wait_seconds": 0},
-        {"max_consecutive_slices": True},
-        {"max_consecutive_slices": 257},
-    ],
-)
-def test_invalid_configuration_never_starts_worker(kwargs):
-    worker = Worker()
-    with pytest.raises(ValueError, match="index_supervisor_invalid_config"):
-        RetrievalIndexSupervisor(worker, **kwargs)
-    assert worker.calls == []
-
-
-def test_idle_timeout_polls_again_and_stop_interrupts_long_wait():
-    async def run():
-        worker = Worker()
-        supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.04)
-        supervisor.start()
-        try:
-            await until(lambda: len(worker.calls) >= 2)
-            assert worker.calls[1] - worker.calls[0] >= 0.035
-        finally:
-            await supervisor.stop()
-        long_idle = RetrievalIndexSupervisor(Worker(), idle_wait_seconds=300)
-        long_idle.start()
-        await until(lambda: long_idle.snapshot().slices == 1)
-        started = time.monotonic()
-        await long_idle.stop()
-        assert time.monotonic() - started < 0.5
-
-    asyncio.run(run())
-
-
-def test_failure_clears_on_recovery_without_leaking_worker_reason():
-    async def run():
-        worker = Worker("exception")
-        supervisor = RetrievalIndexSupervisor(worker, failure_wait_seconds=0.06)
-        supervisor.start()
-        try:
-            await until(lambda: supervisor.snapshot().failures == 1)
-            worker.state = "idle"
-            await until(lambda: supervisor.snapshot().slices == 2)
-            current = supervisor.snapshot()
-            assert current.error_code is None and current.last_result == "idle"
-            assert current.failures == 1
-            assert "SECRET" not in repr(current)
-            with pytest.raises(FrozenInstanceError):
-                current.slices = 0
-        finally:
-            await supervisor.stop()
-
-    asyncio.run(run())
-
-
-def test_thread_start_failure_is_fixed_and_terminal(monkeypatch):
-    def fail(_self):
-        raise OSError("SECRET_THREAD_ERROR")
-
-    monkeypatch.setattr(threading.Thread, "start", fail)
-    supervisor = RetrievalIndexSupervisor(Worker())
-    with pytest.raises(RuntimeError, match="^index_supervisor_start_failed$"):
-        supervisor.start()
-    assert not supervisor.snapshot().thread_alive
-    assert supervisor.snapshot().state == "stopped"
-    assert "SECRET" not in repr(supervisor.snapshot())
-    asyncio.run(supervisor.stop())
-
-
-@pytest.mark.parametrize("state", ["deferred", "cancelled"])
-def test_nonprogress_results_do_not_drain_in_a_tight_loop(state):
-    async def run():
-        worker = Worker(state)
-        supervisor = RetrievalIndexSupervisor(worker, idle_wait_seconds=0.3)
-        supervisor.start()
-        try:
-            await until(lambda: supervisor.snapshot().slices == 1)
-            await asyncio.sleep(0.05)
-            assert len(worker.calls) == 1
-        finally:
-            await supervisor.stop()
-
-    asyncio.run(run())
-
-
-def test_thread_is_non_daemon_and_joined_without_default_executor_threads():
-    async def run():
-        before = set(threading.enumerate())
-        worker = Worker(blocking=True)
-        supervisor = RetrievalIndexSupervisor(worker)
-        supervisor.start()
-        await until(worker.entered.is_set)
-        own = [t for t in threading.enumerate() if t.ident in worker.thread_ids]
-        assert len(own) == 1 and not own[0].daemon
-        try:
-            task = asyncio.create_task(supervisor.stop())
-            await until(lambda: worker.cancel.is_set())
-            assert set(threading.enumerate()) - before == set(own)
-        finally:
-            worker.release.set()
-            await task
-        assert not own[0].is_alive()
-        assert set(threading.enumerate()) == before
-
-    asyncio.run(run())

memhub/apps/core/tests/test_retrieval_index_worker.py

@@ -1,222 +0,0 @@
-import socket
-import sqlite3
-import threading
-import time
-from contextlib import closing
-
-import pytest
-
-from one_memory.embedding_assets import assets_present
-from one_memory.local_embedding import TokenizedText, TokenOffset
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder, RetrievalIndexWorker
-from one_memory.retrieval.vector_store import RawVectorStore, VectorStoreError
-from tests.test_retrieval_index_store import fixture, initialize, put  # noqa: F401
-
-
-@pytest.fixture(autouse=True)
-def forbid_network(monkeypatch):
-    def reject(*args, **kwargs):
-        raise AssertionError("index worker must stay offline")
-
-    monkeypatch.setattr(socket.socket, "connect", reject)
-    monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-class Encoder:
-    def __init__(self, identity):
-        self.identity = identity
-        self.calls = []
-        self.after_encode = lambda: None
-
-    def tokenize_with_offsets(self, text):
-        return TokenizedText(
-            tuple(range(len(text))), tuple(TokenOffset(i, i + 1) for i in range(len(text)))
-        )
-
-    def encode_documents(self, texts):
-        self.calls.append(texts)
-        self.after_encode()
-        return [[1.0, 0.5, 0.25] for _ in texts]
-
-
-@pytest.fixture
-def setup(fixture, tmp_path):  # noqa: F811
-    generation = initialize(fixture)
-    store = fixture[3]
-    encoder = Encoder(fixture[4])
-    vectors = RawVectorStore.create(tmp_path / "vectors", generation_id=generation, dimensions=3)
-    worker = RetrievalIndexWorker(store, vectors, encoder, generation_id=generation, owner="test")
-    try:
-        yield fixture, generation, encoder, vectors, worker
-    finally:
-        vectors.close()
-
-
-def test_tick_indexes_complete_original_and_reopen_can_search(setup):
-    f, generation, encoder, vectors, worker = setup
-    record = put(f[1], "start " + "middle " * 180 + " end")
-    assert worker.run_once().state == "ready"
-    assert worker.run_once().state == "idle"
-    assert all(len(batch) <= 8 for batch in encoder.calls)
-    assert "".join(encoder.calls[0])  # Nonempty positive encoding control.
-    f[3].activate_generation(generation)
-    chunks = RetrievalIndexStore(f[0]).active_chunks(generation)
-    assert len(chunks) > 1 and chunks[0].start_cp == 0
-    assert chunks[-1].end_cp == len(record.verbatim)
-    result = vectors.search(
-        [1.0, 0.5, 0.25], [c.chunk_id for c in chunks], 8, time.monotonic() + 10
-    )
-    assert result.complete and result.hits
-    with closing(sqlite3.connect(f[0])) as db:
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='continuous_policy_history'"
-        ).fetchone()
-
-
-@pytest.mark.parametrize("change", ["revision", "revoke"])
-def test_change_during_native_encoding_never_publishes(setup, change):
-    f, generation, encoder, vectors, worker = setup
-    put(f[1])
-
-    def mutate():
-        if change == "revision":
-            put(f[1], "changed original")
-        else:
-            with closing(sqlite3.connect(f[0])) as db, db:
-                db.execute("UPDATE source_grants SET revoked_at='revoked'")
-
-    encoder.after_encode = mutate
-    result = worker.run_once()
-    assert result.state == "blocked"
-    assert f[3].status(generation)["ready"] == 0
-    assert f[3].active_chunks(generation) == ()
-
-
-def test_cancel_after_encode_does_not_publish_and_is_reclaimable(setup):
-    f, generation, encoder, _, worker = setup
-    put(f[1])
-    cancel = threading.Event()
-    encoder.after_encode = cancel.set
-    assert worker.run_once(cancel=cancel).state == "cancelled"
-    assert f[3].status(generation)["leased"] == 1
-    assert f[3].claim(generation, "replacement", now=time.time() + 61)
-
-
-def test_cancel_before_tick_does_not_consume_or_encode(setup):
-    f, generation, encoder, _, worker = setup
-    put(f[1])
-    cancel = threading.Event()
-    cancel.set()
-    assert worker.run_once(cancel=cancel).state == "cancelled"
-    assert f[3].status(generation)["after_seq"] == 0
-    assert not encoder.calls
-
-
-def test_partial_vector_write_can_retry_same_build_without_duplicates(setup, monkeypatch):
-    f, generation, encoder, vectors, worker = setup
-    put(f[1], "long message " * 90)
-    original = vectors.upsert
-
-    def partial(records):
-        original(records[:1])
-        raise VectorStoreError("vector_write_unavailable")
-
-    monkeypatch.setattr(vectors, "upsert", partial)
-    assert worker.run_once().state == "blocked"
-    assert f[3].status(generation)["ready"] == 0
-    monkeypatch.setattr(vectors, "upsert", original)
-    assert f[3].retry_blocked(generation) == 1
-    assert worker.run_once().state == "ready"
-    f[3].activate_generation(generation)
-    assert len(f[3].active_chunks(generation)) > 1
-
-
-def test_bad_vector_count_or_failed_verification_never_publishes(setup, monkeypatch):
-    f, generation, encoder, vectors, worker = setup
-    put(f[1])
-    monkeypatch.setattr(encoder, "encode_documents", lambda _: [])
-    assert worker.run_once().state == "blocked"
-    assert f[3].status(generation)["ready"] == 0
-    assert f[3].retry_blocked(generation) == 1
-    monkeypatch.setattr(
-        encoder, "encode_documents", lambda texts: [[1.0, 0.5, 0.25] for _ in texts]
-    )
-    monkeypatch.setattr(
-        vectors, "verify", lambda _: (_ for _ in ()).throw(VectorStoreError("failure"))
-    )
-    assert worker.run_once().state == "blocked"
-    assert f[3].status(generation)["ready"] == 0
-
-
-@pytest.mark.parametrize(
-    "text",
-    ["token=example-secret-value", "<environment_context>system wrapper</environment_context>"],
-)
-def test_content_requiring_source_mapping_is_not_encoded_or_marked_ready(setup, text):
-    f, generation, encoder, _, worker = setup
-    put(f[1], text)
-    result = worker.run_once()
-    assert result.state == "blocked" and result.reason == "content_mapping_required"
-    assert not encoder.calls
-    assert f[3].status(generation)["ready"] == 0
-
-
-def test_deadline_after_native_encode_discards_late_work(setup):
-    f, generation, encoder, vectors, _ = setup
-    put(f[1])
-    clock = [1.0]
-    encoder.after_encode = lambda: clock.__setitem__(0, 50.0)
-    worker = RetrievalIndexWorker(
-        f[3],
-        vectors,
-        encoder,
-        generation_id=generation,
-        owner="test",
-        clock=lambda: clock[0],
-        max_job_seconds=10,
-    )
-    assert worker.run_once().state == "deferred"
-    assert f[3].status(generation)["ready"] == 0
-
-
-def test_generation_encoder_mismatch_rejected_before_claim(setup):
-    f, generation, encoder, vectors, _ = setup
-    from dataclasses import replace
-
-    encoder.identity = replace(encoder.identity, encoder_digest="f" * 64)
-    with pytest.raises(ValueError, match="encoder_identity"):
-        RetrievalIndexWorker(f[3], vectors, encoder, generation_id=generation, owner="other")
-    assert f[3].status(generation)["leased"] == 0
-
-
-@pytest.mark.skipif(not assets_present(), reason="locked local assets absent")
-def test_real_local_encoder_chroma_full_text_chain(fixture, tmp_path):  # noqa: F811
-    path, evidence, _, store, _ = fixture
-    encoder = LocalRetrievalEncoder(content_version=1)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        store.initialize(db)
-        generation = store.create_generation(db, encoder.identity)
-    text = (
-        "A long source message about source control and releases. " * 80
-        + " Final artifact at /project/design/final.png"
-    )
-    record = put(evidence, text)
-    vectors = RawVectorStore.create(
-        tmp_path / "real-vectors", generation_id=generation, dimensions=encoder.identity.dimensions
-    )
-    try:
-        worker = RetrievalIndexWorker(
-            store, vectors, encoder, generation_id=generation, owner="real"
-        )
-        assert worker.run_once().state == "ready"
-        store.activate_generation(generation)
-        chunks = store.active_chunks(generation)
-        assert len(chunks) > 1 and chunks[-1].end_cp == len(text)
-        query = encoder.encode_documents([text[chunks[-1].start_cp : chunks[-1].end_cp]])[0]
-        result = vectors.search(query, [c.chunk_id for c in chunks], 8, time.monotonic() + 10)
-        assert result.complete and result.hits[0].chunk_id == chunks[-1].chunk_id
-        assert record.verbatim.endswith("final.png")
-    finally:
-        vectors.close()

memhub/apps/core/tests/test_retrieval_lifespan.py

@@ -1,23 +1,14 @@
 import asyncio
-import json
-import os
-import socket
 import sqlite3
-import subprocess
-import sys
-import time
-import urllib.request
 from contextlib import closing
-from pathlib import Path
-
-import pytest
 
 from one_memory.api import create_app
 from one_memory.config import Settings
 from one_memory.secrets import InMemorySecretStore
 
 
-def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch):
+def test_startup_does_not_create_a_private_retrieval_index(tmp_path, monkeypatch):
+    """Raw reads go through the SDK and lexical FTS; MemHub keeps no vector copy."""
     monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
 
     async def scenario():
@@ -29,10 +20,10 @@ def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch)
             automatic_memory_processing=False,
             live_sync_enabled=False,
         )
-        assert settings.retrieval_index_enabled is False
         app = create_app(settings, secret_store=InMemorySecretStore())
         async with app.router.lifespan_context(app):
-            assert app.state.retrieval_runtime is None
+            assert not hasattr(app.state, "retrieval_runtime")
+            assert not (tmp_path / "retrieval-vectors").exists()
             with closing(sqlite3.connect(tmp_path / "one-memory.sqlite3")) as db:
                 assert (
                     db.execute(
@@ -42,188 +33,3 @@ def test_retrieval_defaults_off_and_does_not_create_index(tmp_path, monkeypatch)
                 )
 
     asyncio.run(scenario())
-
-
-def test_enabling_retrieval_requires_runtime_ownership(tmp_path):
-    with pytest.raises(ValueError, match="retrieval_runtime_ownership_required"):
-        create_app(
-            Settings(_env_file=None, data_dir=tmp_path, retrieval_index_enabled=True),
-            secret_store=InMemorySecretStore(),
-        )
-
-
-def test_index_enabled_serve_imports_without_building_an_unmanaged_app(tmp_path, monkeypatch):
-    """The real serve entry must not be killed by an import-time default app.
-
-    `cli.serve()` imports `create_app` and then builds the app with
-    `manage_runtime=True`. If this module also builds a default app at import
-    time, enabling the index makes that throw before `serve()` ever binds.
-    Run the public entry in a subprocess, not through `create_app()` directly,
-    because the regression exists only in the import/entry-point combination.
-    """
-    monkeypatch.setenv("ONE_MEMORY_DATA_DIR", str(tmp_path))
-    monkeypatch.setenv("ONE_MEMORY_RETRIEVAL_INDEX_ENABLED", "1")
-    monkeypatch.setenv("ONE_MEMORY_RETRIEVAL_MINIMUM_FREE_BYTES", "0")
-    with socket.socket() as probe:
-        probe.bind(("127.0.0.1", 0))
-        port = probe.getsockname()[1]
-    monkeypatch.setenv("ONE_MEMORY_HOST", "127.0.0.1")
-    monkeypatch.setenv("ONE_MEMORY_PORT", str(port))
-    monkeypatch.setenv("HOME", str(tmp_path / "home"))
-    monkeypatch.setenv("ONE_MEMORY_SOURCE_HOME", str(tmp_path / "source-home"))
-    monkeypatch.setenv("ONE_MEMORY_IMPORTS_ENABLED", "0")
-    monkeypatch.setenv("ONE_MEMORY_LIVE_SYNC_ENABLED", "0")
-    monkeypatch.setenv("ONE_MEMORY_AUTOMATIC_MEMORY_PROCESSING", "0")
-
-    process = subprocess.Popen(
-        [sys.executable, "-m", "one_memory.cli", "serve"],
-        env={
-            **os.environ,
-            "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src"),
-        },
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        text=True,
-    )
-    stdout = ""
-    stderr = ""
-    try:
-        deadline = time.monotonic() + 15
-        health = None
-        while time.monotonic() < deadline and process.poll() is None:
-            try:
-                with urllib.request.urlopen(
-                    f"http://127.0.0.1:{port}/api/v1/health", timeout=0.5
-                ) as response:
-                    health = json.loads(response.read())
-                break
-            except OSError:
-                time.sleep(0.1)
-        if health is None:
-            process.terminate()
-            try:
-                stdout, stderr = process.communicate(timeout=5)
-            except subprocess.TimeoutExpired:
-                process.kill()
-                stdout, stderr = process.communicate(timeout=5)
-            pytest.fail(
-                "serve did not become healthy within 15s; "
-                f"exit={process.returncode} stdout={stdout!r} stderr={stderr!r}"
-            )
-        assert health["status"] == "ok"
-        assert health["service"] == "one-memory-core"
-    finally:
-        if process.poll() is None:
-            process.terminate()
-            try:
-                process.communicate(timeout=5)
-            except subprocess.TimeoutExpired:
-                process.kill()
-                process.communicate(timeout=5)
-
-
-def managed(tmp_path):
-    return create_app(
-        Settings(
-            _env_file=None,
-            data_dir=tmp_path,
-            source_home=tmp_path / "home",
-            imports_enabled=False,
-            automatic_memory_processing=False,
-            live_sync_enabled=False,
-            retrieval_index_enabled=True,
-        ),
-        manage_runtime=True,
-        secret_store=InMemorySecretStore(),
-    )
-
-
-def test_managed_start_without_an_encoder_preserves_normal_core_startup(tmp_path, monkeypatch):
-    """No built-in weights is the real reason an index cannot be admitted.
-
-    Core must still come up and the lexical lane must still answer; the runtime
-    says not_initialized rather than taking the process down with it.
-    """
-    monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
-    monkeypatch.setattr(
-        "one_memory.retrieval.index_provisioning.assets_present", lambda *_args: False
-    )
-
-    async def scenario():
-        app = managed(tmp_path)
-        async with app.router.lifespan_context(app):
-            assert app.state.retrieval_runtime.snapshot().state == "not_initialized"
-            assert (tmp_path / "runtime.json").exists()
-            assert not (tmp_path / "retrieval-vectors").exists()
-        assert not (tmp_path / "runtime.json").exists()
-
-    asyncio.run(scenario())
-
-
-def test_enabling_retrieval_provisions_the_index_instead_of_waiting_to_be_handed_one(
-    tmp_path, monkeypatch
-):
-    """The product entry point, which until now only a test helper could reach."""
-    monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
-
-    async def scenario():
-        app = managed(tmp_path)
-        async with app.router.lifespan_context(app):
-            assert app.state.retrieval_runtime.snapshot().state == "running"
-            assert (tmp_path / "retrieval-vectors").is_dir()
-            with closing(sqlite3.connect(tmp_path / "one-memory.sqlite3")) as db:
-                assert [
-                    row[0]
-                    for row in db.execute(
-                        "SELECT state FROM retrieval_index_generations ORDER BY rowid"
-                    )
-                ] == ["active"]
-
-    asyncio.run(scenario())
-
-
-@pytest.mark.parametrize("failure", [RuntimeError, asyncio.CancelledError])
-def test_retrieval_stop_failure_still_drains_other_owned_services(tmp_path, monkeypatch, failure):
-    monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
-    events = []
-
-    class StubRuntime:
-        def __init__(self, *args, **kwargs):
-            pass
-
-        async def start(self):
-            events.append("start")
-
-        async def stop(self):
-            events.append("retrieval_drained")
-            raise failure("after drain")
-
-    monkeypatch.setattr("one_memory.retrieval.runtime.ExistingRetrievalRuntime", StubRuntime)
-
-    async def scenario():
-        app = create_app(
-            Settings(
-                _env_file=None,
-                data_dir=tmp_path,
-                source_home=tmp_path / "home",
-                imports_enabled=False,
-                automatic_memory_processing=False,
-                live_sync_enabled=False,
-                retrieval_index_enabled=True,
-            ),
-            manage_runtime=True,
-            secret_store=InMemorySecretStore(),
-        )
-        with pytest.raises(failure):
-            async with app.router.lifespan_context(app):
-                original = app.state.sdk_memory_runtime.shutdown
-
-                async def shutdown():
-                    await original()
-                    events.append("processing_drained")
-
-                monkeypatch.setattr(app.state.sdk_memory_runtime, "shutdown", shutdown)
-        assert events == ["start", "retrieval_drained", "processing_drained"]
-        assert not (tmp_path / "runtime.json").exists()
-
-    asyncio.run(scenario())

memhub/apps/core/tests/test_retrieval_mapped_content.py

@@ -1,422 +0,0 @@
-import hashlib
-import json
-import socket
-from dataclasses import FrozenInstanceError, replace
-from datetime import UTC, datetime
-
-import pytest
-
-from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
-from one_memory.input_sanitization import SanitizedCopySpan, SanitizedInput, sanitize_model_input
-from one_memory.local_embedding import LocalEmbedder, TokenizedText, TokenOffset
-from one_memory.retrieval import mapped_content as mapped
-from one_memory.retrieval.chunking import ChunkingError
-from one_memory.retrieval.mapped_content import (
-    MappedContentError,
-    chunk_mapped_content,
-    prepare_mapped_content,
-)
-
-
-@pytest.fixture(autouse=True)
-def offline(monkeypatch):
-    def reject(*args, **kwargs):
-        raise AssertionError("mapped content must remain offline")
-
-    monkeypatch.setattr(socket.socket, "connect", reject)
-    monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-def record(text, **changes):
-    values = dict(
-        evidence_id="synthetic-evidence",
-        user_id="local",
-        source_id="synthetic-source",
-        workspace=None,
-        project=None,
-        conversation_id="synthetic-conversation",
-        message_id="synthetic-message",
-        correlation_id=None,
-        source_ordinal=None,
-        content_kind=EvidenceContentKind.RAW_CONVERSATION,
-        role="user",
-        written_by=WrittenBy.PERSON,
-        created_at=None,
-        verbatim=text,
-        origin_meta={},
-        content_hash="outer-manifest-owned",
-        revision=4,
-        imported_at=datetime(2026, 9, 19, tzinfo=UTC),
-        updated_at=datetime(2026, 9, 19, tzinfo=UTC),
-    )
-    return EvidenceRecord(**(values | changes))
-
-
-class Characters:
-    def __init__(self):
-        self.calls = []
-
-    def tokenize_with_offsets(self, text):
-        self.calls.append(text)
-        offsets = (
-            TokenOffset(0, 0, True),
-            *(TokenOffset(i, i + 1) for i in range(len(text))),
-            TokenOffset(0, 0, True),
-        )
-        return TokenizedText(tuple(range(len(offsets))), offsets)
-
-
-def assert_map(original, content):
-    meta = content.metadata
-    assert meta.source_length_cp == len(original)
-    assert meta.sanitized_length_cp == len(content.actual_text)
-    assert meta.sanitized_sha256 == hashlib.sha256(content.actual_text.encode()).hexdigest()
-    output_end = 0
-    copied = []
-    for span in meta.segments:
-        assert span.start_cp == output_end < span.end_cp
-        output_end = span.end_cp
-        if span.kind == "copy":
-            assert (
-                content.actual_text[span.start_cp : span.end_cp]
-                == original[span.source_start_cp : span.source_end_cp]
-            )
-            copied.append((span.source_start_cp, span.source_end_cp))
-        else:
-            assert span.kind == "synthetic"
-            assert span.source_start_cp is span.source_end_cp is None
-    assert output_end == len(content.actual_text)
-    ranges = sorted(copied + [(s.start_cp, s.end_cp) for s in meta.excluded])
-    end = 0
-    for start, right in ranges:
-        assert start == end < right
-        end = right
-    assert end == len(original)
-
-
-def assert_chunks(original, prepared, chunks):
-    assert [c.ordinal for c in chunks] == list(range(len(chunks)))
-    coverage = set()
-    for c in chunks:
-        assert c.text == original[c.start_cp : c.end_cp]
-        assert c.token_count <= 512
-        assert any(
-            s.kind == "copy" and s.source_start_cp <= c.start_cp < c.end_cp <= s.source_end_cp
-            for s in prepared.metadata.segments
-        )
-        coverage.update(range(c.start_cp, c.end_cp))
-    for s in prepared.metadata.segments:
-        if s.kind == "copy":
-            assert all(
-                i in coverage
-                for i in range(s.source_start_cp, s.source_end_cp)
-                if not original[i].isspace()
-            )
-
-
-@pytest.mark.parametrize(
-    "role,author", [("user", WrittenBy.PERSON), ("assistant", WrittenBy.AGENT)]
-)
-def test_raw_authors_keep_exact_body_and_do_not_mutate_evidence(role, author):
-    original = " \t\n\u4e2d\U0001f680e\u0301 same\n same\t "
-    evidence = record(original, role=role, written_by=author)
-    before = evidence.model_dump()
-    content = prepare_mapped_content(evidence)
-    assert content.state == "ready"
-    assert content.actual_text == original.strip()
-    assert evidence.model_dump() == before
-    assert_map(original, content)
-    assert_chunks(original, content, chunk_mapped_content(content, Characters()))
-
-
-@pytest.mark.parametrize(
-    "change",
-    [
-        {"role": "tool", "written_by": WrittenBy.TOOL},
-        {"role": "system"},
-        {"content_kind": EvidenceContentKind.MEMORY_CONTENT},
-    ],
-)
-def test_raw_role_qualification_rejects_without_tokenizing(change):
-    with pytest.raises(MappedContentError, match="mapped_content_ineligible"):
-        prepare_mapped_content(record("ordinary body", **change))
-
-
-@pytest.mark.parametrize("author", list(WrittenBy))
-def test_host_author_guess_does_not_change_protocol_role_qualification(author):
-    content = prepare_mapped_content(record("ordinary body", role="user", written_by=author))
-    assert content.state == "ready"
-
-
-@pytest.mark.parametrize(
-    "text,reason",
-    [
-        ("<user_query>one</user_query><user_query>two</user_query>", "ambiguous"),
-        ("<user_info>unclosed", "ambiguous"),
-        ("<rules>context</rules> unexplained trailing body", "ambiguous"),
-        ("<dynamic_tool_catalog>generated</dynamic_tool_catalog>", "generated"),
-        ('{"request_id":"fixture","type":"write","steps":[]}', "generated"),
-        (
-            "<user_query><dynamic_tool_catalog>generated</dynamic_tool_catalog></user_query>",
-            "generated",
-        ),
-        ("<user_query><environment_context>harness</environment_context></user_query>", "hygiene"),
-    ],
-)
-def test_ambiguous_generated_and_body_hygiene_are_not_admitted(text, reason):
-    with pytest.raises(MappedContentError, match=reason):
-        prepare_mapped_content(record(text))
-
-
-def test_wrapper_offsets_compose_by_position_despite_repeated_text_and_unicode():
-    prefix = " \t<rules>same \U0001f680</rules>\n<user_query> \n"
-    body = "same \U0001f680 token=same same e\u0301"
-    original = prefix + body + " \n</user_query><image_local_path>same</image_local_path> "
-    prepared = prepare_mapped_content(record(original))
-    assert prepared.actual_text == sanitize_model_input(body)
-    assert prepared.metadata.body_start_cp == len(prefix)
-    assert prepared.metadata.body_end_cp == len(prefix) + len(body)
-    assert_map(original, prepared)
-    tokenizer = Characters()
-    chunks = chunk_mapped_content(prepared, tokenizer, target_tokens=10, overlap_tokens=3)
-    assert_chunks(original, prepared, chunks)
-    assert chunks[0].start_cp == len(prefix)
-    assert all("<rules>" not in text and "redacted-secret" not in text for text in tokenizer.calls)
-
-
-@pytest.mark.parametrize(
-    "secret",
-    [
-        "token=private-value",
-        "Authorization: Bearer abcdefghijklmnop",
-        '--password "two words"',
-        "sk-abcdefghijklmnopqrstuvwxyz",
-        "https://alice:private-password@example.test/path",
-        "-----BEGIN PRIVATE KEY-----\nprivate\n-----END PRIVATE KEY-----",
-    ],
-)
-def test_secrets_and_synthetic_replacements_never_reach_tokenizer(secret):
-    original = "left \u4e2d\U0001f680 " + secret + " right e\u0301 /final/file"
-    content = prepare_mapped_content(record(original))
-    tokenizer = Characters()
-    chunks = chunk_mapped_content(content, tokenizer, target_tokens=12, overlap_tokens=4)
-    assert_map(original, content)
-    assert_chunks(original, content, chunks)
-    assert all(secret not in text and "redacted-secret" not in text for text in tokenizer.calls)
-    assert any("left" in c.text for c in chunks) and any("right" in c.text for c in chunks)
-
-
-@pytest.mark.parametrize(
-    "text", ["", " \n\t ", "<rules>only wrapper</rules>", "token=private", " token=x \t token=y "]
-)
-def test_empty_wrapper_or_all_excluded_is_explicit_nonindexable(text):
-    content = prepare_mapped_content(record(text))
-    assert content.state == "no_indexable_content"
-    assert_map(text, content)
-    tokenizer = Characters()
-    assert chunk_mapped_content(content, tokenizer) == ()
-    assert tokenizer.calls == []
-
-
-def test_metadata_is_immutable_canonical_and_never_contains_text_or_secret_hashes():
-    original = "<user_query>prefix token=private-value suffix</user_query>"
-    content = prepare_mapped_content(record(original))
-    encoded = content.metadata.canonical_json()
-    assert encoded == json.dumps(json.loads(encoded), sort_keys=True, separators=(",", ":"))
-    assert content.metadata.digest == hashlib.sha256(encoded.encode()).hexdigest()
-    for text in [
-        "prefix",
-        "suffix",
-        "private-value",
-        "redacted-secret",
-        hashlib.sha256(b"private-value").hexdigest(),
-    ]:
-        assert text not in encoded
-    assert content.metadata == prepare_mapped_content(record(original)).metadata
-    with pytest.raises(FrozenInstanceError):
-        content.metadata.source_length_cp = 1
-    with pytest.raises(FrozenInstanceError):
-        content.metadata.segments[0].start_cp = 1
-    with pytest.raises(FrozenInstanceError):
-        content.actual_text = "other"
-
-
-def test_digest_changes_for_safe_text_or_coordinate_version_not_equal_length_excluded_secret():
-    first = prepare_mapped_content(record("left token=secret-one right"))
-    second = prepare_mapped_content(record("left token=secret-two right"))
-    assert first.metadata.digest == second.metadata.digest
-    assert (
-        first.metadata.digest
-        != prepare_mapped_content(record("LEFT token=secret-one right")).metadata.digest
-    )
-    assert first.metadata.digest != replace(first.metadata, map_version="future-map").digest
-
-
-def test_overlapping_sanitizer_passes_and_literal_placeholder_are_distinct():
-    text = "<redacted-secret> token=sk-abcdefghijklmnopqrstuvwxyz /tail"
-    content = prepare_mapped_content(record(text))
-    assert_map(text, content)
-    chunks = chunk_mapped_content(content, Characters())
-    assert chunks[0].text == "<redacted-secret> "  # Literal source text is copy, not synthetic.
-    assert chunks[-1].text == " /tail"
-
-
-@pytest.mark.parametrize("text", ["\u4e2d\x00\u6587", "\u4e2d\u200b\u6587", "\u4e2d\u200d\u6587"])
-def test_control_characters_are_not_silently_sanitized_or_dropped(text):
-    content = prepare_mapped_content(record(text))
-    assert content.actual_text == text
-    with pytest.raises(ChunkingError, match="uncovered_nonwhitespace"):
-        chunk_mapped_content(content, LocalEmbedder())
-
-
-def test_locked_tokenizer_unicode_coverage_and_prefix_budget():
-    body = ("\u4e2d\u6587\U0001f680e\u0301\n" * 400) + " token=private " + ("\u5c3e\u6587\n" * 400)
-    original = "<user_query>" + body + "</user_query>"
-    content = prepare_mapped_content(record(original))
-    tokenizer = LocalEmbedder()
-    prefix = "\u68c0\u7d22\u524d\u7f00" * 30
-    chunks = chunk_mapped_content(content, tokenizer, prefix=prefix)
-    assert len(chunks) > 4
-    assert_chunks(original, content, chunks)
-    assert all(
-        c.token_count == len(tokenizer.tokenize_with_offsets(prefix + c.text).token_ids)
-        for c in chunks
-    )
-
-
-def test_more_than_8192_chunks_aborts_one_copy_span_without_returning_partial_content():
-    tokenizer = Characters()
-    content = prepare_mapped_content(record("x" * 9000))
-    with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):
-        chunk_mapped_content(content, tokenizer, target_tokens=1, overlap_tokens=0)
-    assert len(tokenizer.calls) <= 8194
-
-
-def test_global_limit_across_copy_spans_and_exact_limit_succeeds():
-    content = prepare_mapped_content(record("a token=private b"))
-    with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):
-        chunk_mapped_content(content, Characters(), max_chunks=1)
-    chunks = chunk_mapped_content(content, Characters(), max_chunks=2)
-    assert [c.ordinal for c in chunks] == [0, 1]
-
-
-@pytest.mark.parametrize("limit", [0, -1, True, 8193])
-def test_chunk_limit_is_not_raiseable_or_coercible(limit):
-    with pytest.raises(MappedContentError, match="mapped_invalid_chunk_limit"):
-        chunk_mapped_content(prepare_mapped_content(record("body")), Characters(), max_chunks=limit)
-
-
-def test_forged_text_and_incomplete_sanitizer_mapping_fail_closed(monkeypatch):
-    prepared = prepare_mapped_content(record("body"))
-    with pytest.raises(MappedContentError, match="mapped_invalid"):
-        chunk_mapped_content(replace(prepared, actual_text="BODY"), Characters())
-    monkeypatch.setattr(
-        mapped,
-        "sanitize_model_input_with_spans",
-        lambda text: SanitizedInput(text, (SanitizedCopySpan(0, 2, 0, 1),), ()),
-    )
-    with pytest.raises(MappedContentError, match="mapped_invalid"):
-        prepare_mapped_content(record("body"))
-
-
-def test_exact_8192_chunks_and_shrink_retries_do_not_spend_extra_chunk_budget():
-    content = prepare_mapped_content(record("x" * 8192))
-    chunks = chunk_mapped_content(content, Characters(), target_tokens=1, overlap_tokens=0)
-    assert len(chunks) == 8192 and chunks[-1].ordinal == 8191
-    content = prepare_mapped_content(record("x" * 10))
-    chunks = chunk_mapped_content(
-        content,
-        Characters(),
-        target_tokens=8,
-        overlap_tokens=0,
-        max_tokens=8,
-        prefix="PP",
-        max_chunks=3,
-    )
-    assert len(chunks) == 3
-    assert_chunks("x" * 10, content, chunks)
-
-
-def test_fragmented_content_limit_is_global_and_rejects_before_next_copy_tokenization():
-    text = " token=private ".join(["x"] * 8193)
-    prepared = prepare_mapped_content(record(text))
-    tokenizer = Characters()
-    with pytest.raises(MappedContentError, match="mapped_chunk_limit_exceeded"):
-        chunk_mapped_content(prepared, tokenizer)
-    assert len(tokenizer.calls) == 2 * 8192
-
-
-def test_random_repeated_wrapper_and_sanitizer_composition():
-    import random
-
-    randomizer = random.Random(20260919)
-    for _ in range(80):
-        body = " ".join(
-            randomizer.choices(
-                [
-                    "same",
-                    "\u4e2d\U0001f680e\u0301",
-                    "token=same",
-                    "--password same",
-                    "Authorization: Bearer abcdefghijklmnop",
-                    "\t",
-                    "https://u:p@host/path",
-                ],
-                k=10,
-            )
-        )
-        original = "<rules>same</rules><user_query> " + body + " </user_query>"
-        content = prepare_mapped_content(record(original))
-        assert_map(original, content)
-        assert_chunks(original, content, chunk_mapped_content(content, Characters()))
-
-
-@pytest.mark.parametrize(
-    "case", ["output_gap", "source_overlap", "wrong_copy", "missing_exclusion"]
-)
-def test_bad_upstream_maps_do_not_authorize_original_slices(monkeypatch, case):
-    samples = {
-        "output_gap": SanitizedInput("body", (SanitizedCopySpan(2, 4, 2, 4),), ()),
-        "source_overlap": SanitizedInput(
-            "bobo", (SanitizedCopySpan(0, 2, 0, 2), SanitizedCopySpan(2, 4, 0, 2)), ((2, 4),)
-        ),
-        "wrong_copy": SanitizedInput("BODY", (SanitizedCopySpan(0, 4, 0, 4),), ()),
-        "missing_exclusion": SanitizedInput("bo", (SanitizedCopySpan(0, 2, 0, 2),), ()),
-    }
-    monkeypatch.setattr(mapped, "sanitize_model_input_with_spans", lambda text: samples[case])
-    with pytest.raises(MappedContentError, match="mapped_invalid"):
-        prepare_mapped_content(record("body"))
-
-
-def test_assistant_context_keeps_existing_hygiene_semantics_but_not_generated_payloads():
-    text = "Discuss <environment_context>literal</environment_context> as context"
-    content = prepare_mapped_content(record(text, role="assistant", written_by=WrittenBy.AGENT))
-    assert content.state == "ready" and content.actual_text == text
-    with pytest.raises(MappedContentError, match="generated"):
-        prepare_mapped_content(
-            record(
-                "<dynamic_tool_catalog>generated</dynamic_tool_catalog>",
-                role="assistant",
-                written_by=WrittenBy.AGENT,
-            )
-        )
-
-
-def test_copy_boundaries_cannot_reconstruct_a_url_across_secret_gap():
-    original = "https://example.test/left token=private /right"
-    content = prepare_mapped_content(record(original))
-    chunks = chunk_mapped_content(content, Characters())
-    assert len(chunks) == 2
-    assert chunks[0].text == "https://example.test/left "
-    assert chunks[1].text == " /right"
-    assert chunks[0].end_cp < chunks[1].start_cp
-    assert_chunks(original, content, chunks)
-
-
-def test_source_size_limit_rejects_before_parser(monkeypatch):
-    def unexpected(_):
-        raise AssertionError("oversize input reached parser")
-
-    monkeypatch.setattr(mapped, "unwrap_host_envelope_with_positions", unexpected)
-    with pytest.raises(MappedContentError, match="mapped_source_limit_exceeded"):
-        prepare_mapped_content(record("x" * 1_000_001))

memhub/apps/core/tests/test_retrieval_mapped_integration.py

@@ -1,549 +0,0 @@
-import socket
-import sqlite3
-import time
-from contextlib import closing
-from dataclasses import replace
-
-import pytest
-
-from one_memory.retrieval.chunking import TextChunk
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import RetrievalIndexWorker
-from one_memory.retrieval.mapped_content import chunk_mapped_content, prepare_mapped_content
-from one_memory.retrieval.raw_candidates import read_raw_semantic_candidates
-from one_memory.retrieval.raw_evidence import materialize_raw_candidates
-from one_memory.retrieval.vector_store import RawVectorStore
-from tests.test_retrieval_index_store import fixture, initialize, put  # noqa: F401
-from tests.test_retrieval_index_worker import Encoder
-
-
-@pytest.fixture(autouse=True)
-def offline(monkeypatch):
-    def fail(*args, **kwargs):
-        raise AssertionError("no network permitted")
-
-    monkeypatch.setattr(socket.socket, "connect", fail)
-    monkeypatch.setattr(socket.socket, "connect_ex", fail)
-
-
-def migrate(f):
-    initialize(f)
-    identity = replace(f[4], normalizer_version=2, chunker_version=2)
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        f[3].migrate_v2(db)
-        generation = f[3].create_generation(db, identity)
-    return generation, identity
-
-
-@pytest.fixture
-def mapped_setup(fixture, tmp_path):  # noqa: F811
-    generation, identity = migrate(fixture)
-    encoder = Encoder(identity)
-    vectors = RawVectorStore.create(
-        tmp_path / "mapped-vectors", generation_id=generation, dimensions=3
-    )
-    worker = RetrievalIndexWorker(
-        fixture[3], vectors, encoder, generation_id=generation, owner="mapped"
-    )
-    try:
-        yield fixture, generation, encoder, vectors, worker
-    finally:
-        vectors.close()
-
-
-def candidates(f, generation, vectors):
-    return read_raw_semantic_candidates(
-        f[3],
-        vectors,
-        [1.0, 0.5, 0.25],
-        generation_id=generation,
-        source_ids=frozenset({"codex"}),
-        deadline=time.monotonic() + 10,
-    )
-
-
-def materialize(f, generation, hits):
-    return materialize_raw_candidates(
-        f[3],
-        hits,
-        generation_id=generation,
-        source_ids=frozenset({"codex"}),
-        deadline=time.monotonic() + 10,
-    )
-
-
-def test_empty_materialization_rechecks_excluded_map(mapped_setup):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "token=private-value")
-    assert worker.run_once().state == "excluded"
-    f[3].activate_generation(generation)
-    empty = candidates(f, generation, vectors)
-    assert empty.complete and not empty.hits
-    assert materialize(f, generation, empty).complete
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("DELETE FROM retrieval_build_maps")
-    assert not materialize(f, generation, empty).complete
-
-
-def test_completed_excluded_missing_active_binding_is_not_empty_success(mapped_setup):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "token=private-value")
-    assert worker.run_once().state == "excluded"
-    f[3].activate_generation(generation)
-    assert candidates(f, generation, vectors).complete
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("UPDATE retrieval_documents SET active_build_id=NULL")
-    result = candidates(f, generation, vectors)
-    assert not result.complete and result.reason == "index_unavailable"
-
-
-def test_old_empty_completion_is_invalidated_when_a_new_indexed_record_arrives(mapped_setup):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "token=private-value")
-    assert worker.run_once().state == "excluded"
-    f[3].activate_generation(generation)
-    empty = candidates(f, generation, vectors)
-    assert empty.complete and not empty.hits
-    put(f[1], "new actual content", message="new")
-    assert worker.run_once().state == "ready"
-    result = materialize(f, generation, empty)
-    assert not result.complete and result.reason == "evidence_changed"
-
-
-def test_schema2_roundtrip_only_encodes_copy_and_separates_original_spans(mapped_setup):
-    f, generation, encoder, vectors, worker = mapped_setup
-    original = (
-        "<rules>private wrapper</rules><user_query>before token=private-value after</user_query>"
-    )
-    record = put(f[1], original)
-    result = worker.run_once()
-    assert result.state == "ready" and result.chunk_count == 2
-    assert encoder.calls == [["before ", " after"]]
-    f[3].activate_generation(generation)
-    hits = candidates(f, generation, vectors)
-    assert hits.complete and len(hits.hits) == 2
-    output = materialize(f, generation, hits)
-    assert output.complete and len(output.items) == 1
-    assert [s.text for s in output.items[0].spans] == ["before ", " after"]
-    assert all(original[s.start_cp : s.end_cp] == s.text for s in output.items[0].spans)
-    with closing(sqlite3.connect(f[0])) as db:
-        rows = db.execute(
-            "SELECT source_map_json,map_digest,coverage_state FROM retrieval_build_maps"
-        ).fetchall()
-        assert len(rows) == 1 and rows[0][2] == "indexable"
-        assert "before" not in rows[0][0] and "private" not in rows[0][0]
-        assert rows[0][1] == prepare_mapped_content(record).metadata.digest
-    assert RetrievalIndexStore(f[0]).active_chunks(generation)
-
-
-def test_explicit_migration_rolls_back_and_v1_remains_readable(fixture):  # noqa: F811
-    f = fixture
-    generation = initialize(f)
-    row = put(f[1], "legacy text")
-    f[3].consume_pending(generation)
-    lease = f[3].claim(generation, "legacy")
-    f[3].publish(
-        lease, (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),), vectors_verified=True
-    )
-    f[3].activate_generation(generation)
-    before = f[3].active_chunks(generation)
-    with closing(sqlite3.connect(f[0])) as db:
-        with pytest.raises(RuntimeError):
-            f[3].migrate_v2(db)
-        db.execute("BEGIN IMMEDIATE")
-        f[3].migrate_v2(db)
-        db.rollback()
-        assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='retrieval_build_maps'"
-        ).fetchone()
-    assert f[3].active_chunks(generation) == before
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        f[3].migrate_v2(db)
-        f[3].migrate_v2(db)
-    assert f[3].active_chunks(generation) == before
-    from one_memory.retrieval.raw_candidates import RawCandidateBatch, RawSemanticCandidate
-
-    output = materialize(
-        f, generation, RawCandidateBatch(tuple(RawSemanticCandidate(c, 0.0) for c in before), True)
-    )
-    assert output.complete and output.items[0].spans[0].text == "legacy text"
-
-
-def test_version2_requires_explicit_schema_upgrade(fixture):  # noqa: F811
-    f = fixture
-    initialize(f)
-    with closing(sqlite3.connect(f[0])) as db:
-        db.execute("BEGIN IMMEDIATE")
-        with pytest.raises(ValueError, match="schema"):
-            f[3].create_generation(db, replace(f[4], normalizer_version=2, chunker_version=2))
-        assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1
-
-
-@pytest.mark.parametrize("text", ["token=private", "<rules>only wrapper</rules>"])
-def test_excluded_completion_has_no_vectors_and_is_not_ready(mapped_setup, text):
-    f, generation, encoder, vectors, worker = mapped_setup
-    put(f[1], text)
-    result = worker.run_once()
-    assert result.state == "excluded" and result.chunk_count == 0
-    assert encoder.calls == []
-    assert f[3].status(generation)["excluded"] == 1
-    assert f[3].status(generation)["ready"] == 0
-    f[3].activate_generation(generation)
-    assert candidates(f, generation, vectors).hits == ()
-
-
-@pytest.mark.parametrize("tamper", ["json", "digest", "offset", "missing_map", "missing_chunk"])
-def test_persisted_tampering_rejected_at_candidate_and_final_read(mapped_setup, tamper):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "before token=private after")
-    assert worker.run_once().state == "ready"
-    f[3].activate_generation(generation)
-    hits = candidates(f, generation, vectors)
-    with closing(sqlite3.connect(f[0])) as db, db:
-        if tamper == "json":
-            db.execute("UPDATE retrieval_build_maps SET source_map_json='{}'")
-        elif tamper == "digest":
-            db.execute("UPDATE retrieval_build_maps SET map_digest=?", ("0" * 64,))
-        elif tamper == "offset":
-            db.execute("UPDATE retrieval_chunks SET end_cp=end_cp+1 WHERE ordinal=0")
-        elif tamper == "missing_map":
-            db.execute("DELETE FROM retrieval_build_maps")
-        else:
-            db.execute("DELETE FROM retrieval_chunks WHERE ordinal=1")
-    assert not candidates(f, generation, vectors).complete
-    final = materialize(f, generation, hits)
-    assert not final.complete and not final.items
-
-
-@pytest.mark.parametrize("change", ["secret", "revoke", "policy"])
-def test_current_source_secret_revision_and_grant_changes_invalidate_hits(mapped_setup, change):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "before token=private-one after")
-    assert worker.run_once().state == "ready"
-    f[3].activate_generation(generation)
-    hits = candidates(f, generation, vectors)
-    if change == "secret":
-        put(f[1], "before token=private-two after")
-    else:
-        with closing(sqlite3.connect(f[0])) as db, db:
-            db.execute(
-                "UPDATE source_grants SET revoked_at='revoked'"
-                if change == "revoke"
-                else "UPDATE source_grants SET revision=revision+1"
-            )
-    assert not materialize(f, generation, hits).items
-    assert not f[3].active_chunks(generation)
-
-
-def test_publish_recomputes_map_and_rejects_cross_gap_or_incomplete_manifest(mapped_setup):
-    f, generation, encoder, _, _ = mapped_setup
-    row = put(f[1], "before token=private after")
-    f[3].consume_pending(generation)
-    lease = f[3].claim(generation, "publisher")
-    prepared = prepare_mapped_content(row)
-    chunks = chunk_mapped_content(prepared, encoder)
-    for bad in [chunks[:1], (TextChunk(0, 0, len(row.verbatim), 10, row.verbatim),)]:
-        with pytest.raises(ValueError, match="manifest"):
-            f[3].publish(lease, bad, vectors_verified=True, source_map=prepared.metadata)
-    with pytest.raises(ValueError, match="map"):
-        f[3].publish(
-            lease,
-            chunks,
-            vectors_verified=True,
-            source_map=replace(prepared.metadata, sanitized_sha256="0" * 64),
-        )
-    assert f[3].status(generation)["ready"] == 0
-
-
-def test_many_copy_regions_remain_partial_not_a_cross_gap_quote(mapped_setup):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], " token=private ".join([f"part-{i}" for i in range(6)]))
-    assert worker.run_once().chunk_count == 6
-    f[3].activate_generation(generation)
-    final = materialize(f, generation, candidates(f, generation, vectors))
-    assert not final.complete and len(final.items[0].spans) == 4
-    assert all("token=" not in s.text for s in final.items[0].spans)
-
-
-def test_local_onnx_chroma_mapped_roundtrip_without_download(fixture, tmp_path):  # noqa: F811
-    from one_memory.retrieval.index_worker import LocalRetrievalEncoder
-
-    f = fixture
-    encoder = LocalRetrievalEncoder()
-    assert encoder.identity.normalizer_version == encoder.identity.chunker_version == 2
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        f[3].initialize(db)
-        f[3].migrate_v2(db)
-        generation = f[3].create_generation(db, encoder.identity)
-    row = put(
-        f[1],
-        "<user_query>\u4e2d\u6587\U0001f680e\u0301\n"
-        + "\u524d\u6587" * 300
-        + " token=private-value "
-        + "\u540e\u6587" * 300
-        + "</user_query>",
-    )
-    vectors = RawVectorStore.create(
-        tmp_path / "native-mapped", generation_id=generation, dimensions=encoder.identity.dimensions
-    )
-    try:
-        worker = RetrievalIndexWorker(
-            f[3], vectors, encoder, generation_id=generation, owner="local"
-        )
-        result = worker.run_once()
-        assert result.state == "ready" and result.chunk_count >= 4
-        f[3].activate_generation(generation)
-        hits = read_raw_semantic_candidates(
-            f[3],
-            vectors,
-            encoder.encode_documents(["\u540e\u6587"])[0],
-            generation_id=generation,
-            source_ids=frozenset({"codex"}),
-            deadline=time.monotonic() + 15,
-        )
-        final = materialize(f, generation, hits)
-        assert final.complete and len(final.items[0].spans) == 2
-        for span in final.items[0].spans:
-            assert span.text == row.verbatim[span.start_cp : span.end_cp]
-            assert "private-value" not in span.text and "redacted-secret" not in span.text
-    finally:
-        vectors.close()
-
-
-@pytest.mark.parametrize(
-    "text",
-    [
-        "<user_query>one</user_query><user_query>two</user_query>",
-        "<dynamic_tool_catalog>generated</dynamic_tool_catalog>",
-    ],
-)
-def test_ambiguous_generated_stay_blocked(mapped_setup, text):
-    f, generation, encoder, _, worker = mapped_setup
-    put(f[1], text)
-    assert worker.run_once().state == "blocked"
-    assert not encoder.calls
-    with pytest.raises(ValueError, match="incomplete"):
-        f[3].activate_generation(generation)
-
-
-@pytest.mark.parametrize("change", ["revoke", "secret"])
-def test_change_during_encoding_never_publishes(mapped_setup, change):
-    f, generation, encoder, _, worker = mapped_setup
-    put(f[1], "before token=private-one after")
-    original = encoder.encode_documents
-
-    def changed(texts):
-        result = original(texts)
-        if change == "secret":
-            put(f[1], "before token=private-two after")
-        else:
-            f[2].revoke(user_id="local", source_id="codex")
-        return result
-
-    encoder.encode_documents = changed
-    assert worker.run_once().state == "blocked"
-    assert f[3].status(generation)["ready"] == 0
-    with closing(sqlite3.connect(f[0])) as db:
-        assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-
-
-def test_cancellation_inside_publish_rolls_back_map_chunks_and_completion(
-    mapped_setup, monkeypatch
-):
-    import threading
-
-    f, generation, _, _, worker = mapped_setup
-    put(f[1], "before token=private after")
-    cancel = threading.Event()
-    original = f[3]._prepare
-
-    def cancelled(*args):
-        prepared = original(*args)
-        cancel.set()
-        return prepared
-
-    monkeypatch.setattr(f[3], "_prepare", cancelled)
-    assert worker.run_once(cancel=cancel).state == "cancelled"
-    assert f[3].status(generation)["leased"] == 1
-    with closing(sqlite3.connect(f[0])) as db:
-        assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-        assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
-
-
-def test_excluded_metadata_is_validated_on_activation_and_candidate_read(mapped_setup):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "token=private")
-    assert worker.run_once().state == "excluded"
-    f[3].activate_generation(generation)
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("DELETE FROM retrieval_build_maps")
-    assert not candidates(f, generation, vectors).complete
-    with pytest.raises(ValueError, match="map"):
-        f[3].activate_generation(generation)
-
-
-def test_migration_preserves_an_existing_v1_lease_and_store_identity(fixture):  # noqa: F811
-    f = fixture
-    generation = initialize(f)
-    row = put(f[1], "legacy leased")
-    f[3].consume_pending(generation)
-    lease = f[3].claim(generation, "legacy")
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        f[3].migrate_v2(db)
-    assert f[3].read_lease(lease) == row
-    f[3].publish(
-        lease, (TextChunk(0, 0, len(row.verbatim), 4, row.verbatim),), vectors_verified=True
-    )
-    f[3].activate_generation(generation)
-    assert f[3].active_chunks(generation)
-
-
-def test_bad_token_offsets_leave_no_partial_mapped_manifest(mapped_setup):
-    from one_memory.local_embedding import TokenizedText, TokenOffset
-
-    f, generation, encoder, _, worker = mapped_setup
-    put(f[1], "before token=private after")
-    encoder.tokenize_with_offsets = lambda text: TokenizedText((1,), (TokenOffset(0, 1),))
-    assert worker.run_once().state == "blocked"
-    assert not encoder.calls
-    assert f[3].status(generation)["ready"] == 0
-
-
-def test_failed_migration_keeps_original_table_lease_and_schema(fixture):  # noqa: F811
-    f = fixture
-    generation = initialize(f)
-    put(f[1], "preserved")
-    f[3].consume_pending(generation)
-    lease = f[3].claim(generation, "legacy")
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("CREATE TABLE retrieval_build_maps (foreign_value TEXT)")
-        db.execute("BEGIN IMMEDIATE")
-        with pytest.raises(sqlite3.Error):
-            f[3].migrate_v2(db)
-        assert db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0] == 1
-        assert not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='retrieval_documents_v2'"
-        ).fetchone()
-    assert f[3].read_lease(lease).verbatim == "preserved"
-
-
-@pytest.mark.parametrize(
-    "text,expected", [("before token=private after", "ready"), ("token=private", "excluded")]
-)
-def test_regrant_reuses_immutable_map_and_manifest_only_after_explicit_retry(
-    mapped_setup, text, expected
-):
-    from one_memory.imports.sources.registry import RootCandidateKind
-
-    f, generation, _, _, worker = mapped_setup
-    put(f[1], text)
-    assert worker.run_once().state == expected
-    f[3].activate_generation(generation)
-    with closing(sqlite3.connect(f[0])) as db:
-        original = db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-    f[2].revoke(user_id="local", source_id="codex")
-    assert not f[3].active_chunks(generation)
-    f[2].grant(
-        user_id="local",
-        source_id="codex",
-        root=f[0].parent / "source",
-        candidate_kind=RootCandidateKind.DEFAULT,
-    )
-    assert not f[3].active_chunks(generation)
-    assert f[3].retry_blocked(generation) == 1
-    assert worker.run_once().state == expected
-    with closing(sqlite3.connect(f[0])) as db:
-        assert db.execute("SELECT * FROM retrieval_build_maps").fetchall() == original
-
-
-def test_map_replay_conflict_rolls_back_new_chunks(mapped_setup):
-    f, generation, encoder, _, _ = mapped_setup
-    row = put(f[1], "before token=private after")
-    f[3].consume_pending(generation)
-    lease = f[3].claim(generation, "publisher")
-    prepared = prepare_mapped_content(row)
-    chunks = chunk_mapped_content(prepared, encoder)
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute(
-            "INSERT INTO retrieval_build_maps VALUES (?,?,?,?)",
-            (lease.build_id, "{}", "0" * 64, "indexable"),
-        )
-    with pytest.raises(ValueError, match="immutable_source_map"):
-        f[3].publish(lease, chunks, vectors_verified=True, source_map=prepared.metadata)
-    with closing(sqlite3.connect(f[0])) as db:
-        assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
-    assert f[3].status(generation)["leased"] == 1
-
-
-def test_deadline_expiring_during_publish_rolls_back_completion(mapped_setup, monkeypatch):
-    f, generation, _, _, worker = mapped_setup
-    put(f[1], "before token=private after")
-    current = [0.0]
-    worker.clock = lambda: current[0]
-    worker.max_job_seconds = 10
-    original = f[3]._prepare
-
-    def expires(*args):
-        result = original(*args)
-        current[0] = 11
-        return result
-
-    monkeypatch.setattr(f[3], "_prepare", expires)
-    result = worker.run_once()
-    assert result.state == "deferred" and result.reason == "deadline"
-    assert f[3].status(generation)["leased"] == 1
-    with closing(sqlite3.connect(f[0])) as db:
-        assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
-        assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-
-
-def test_deadline_and_character_budget_never_release_cross_gap_spans(mapped_setup):
-    f, generation, _, vectors, worker = mapped_setup
-    put(f[1], "before token=private after")
-    assert worker.run_once().state == "ready"
-    f[3].activate_generation(generation)
-    hits = candidates(f, generation, vectors)
-    expired = materialize_raw_candidates(
-        f[3],
-        hits,
-        generation_id=generation,
-        source_ids=frozenset({"codex"}),
-        deadline=time.monotonic() - 1,
-    )
-    assert not expired.items and expired.reason == "deadline"
-    bounded = materialize_raw_candidates(
-        f[3],
-        hits,
-        generation_id=generation,
-        source_ids=frozenset({"codex"}),
-        deadline=time.monotonic() + 10,
-        character_budget=7,
-    )
-    assert not bounded.complete and bounded.reason == "character_budget"
-    assert bounded.used_characters <= 7
-    assert sum(len(item.spans) for item in bounded.items) == 1
-
-
-def test_lease_expiring_inside_publish_cannot_commit(mapped_setup, monkeypatch):
-    f, generation, _, _, worker = mapped_setup
-    put(f[1], "before token=private after")
-    current = [100.0]
-    worker.wall_clock = lambda: current[0]
-    original = f[3]._prepare
-
-    def expires(*args):
-        result = original(*args)
-        current[0] = 1000.0
-        return result
-
-    monkeypatch.setattr(f[3], "_prepare", expires)
-    assert worker.run_once().state == "blocked"
-    with closing(sqlite3.connect(f[0])) as db:
-        assert not db.execute("SELECT * FROM retrieval_chunks").fetchall()
-        assert not db.execute("SELECT * FROM retrieval_build_maps").fetchall()
-    assert f[3].status(generation)["ready"] == 0

memhub/apps/core/tests/test_retrieval_provisioning.py

@@ -1,125 +0,0 @@
-"""The index has to come up through the product, not through a test helper.
-
-Every existing retrieval test builds its generation by hand, so none of them
-can tell whether a user who enables the feature ever gets an index at all.
-"""
-
-import logging
-import sqlite3
-from contextlib import closing
-from pathlib import Path
-
-import pytest
-
-from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval.index_provisioning import provision_retrieval_index
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import LocalRetrievalEncoder
-
-
-def database(tmp_path: Path) -> Path:
-    path = tmp_path / "one-memory.sqlite3"
-    EvidenceStore(path)
-    return path
-
-
-def active(path: Path) -> list[str]:
-    with closing(sqlite3.connect(path)) as db:
-        if not db.execute(
-            "SELECT 1 FROM sqlite_master WHERE name='retrieval_index_generations'"
-        ).fetchone():
-            return []
-        return [
-            row[0]
-            for row in db.execute(
-                "SELECT generation_id FROM retrieval_index_generations WHERE state='active'"
-            )
-        ]
-
-
-def test_first_enable_publishes_one_active_generation_and_its_vector_store(tmp_path):
-    database(tmp_path)
-
-    generation = provision_retrieval_index(tmp_path)
-
-    assert generation is not None
-    assert active(tmp_path / "one-memory.sqlite3") == [generation]
-    assert (tmp_path / "retrieval-vectors").is_dir()
-
-
-def test_second_start_reuses_the_same_generation_rather_than_stacking_one(tmp_path):
-    database(tmp_path)
-
-    first = provision_retrieval_index(tmp_path)
-    second = provision_retrieval_index(tmp_path)
-
-    assert first == second
-    assert active(tmp_path / "one-memory.sqlite3") == [first]
-
-
-def test_a_v1_index_is_left_alone_instead_of_being_migrated_on_startup(tmp_path, caplog):
-    """Falsifiable against the first test, where provisioning does publish one.
-
-    Migrating someone's existing index while they were only starting the app is
-    the one thing the schema note rules out, so this must stay a no-op. The
-    absence of a warning is asserted too: crashing into the same `None` would
-    otherwise read as a deliberate decision.
-    """
-    path = database(tmp_path)
-    store = RetrievalIndexStore(path)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        store.initialize(db)
-
-    with caplog.at_level(logging.WARNING):
-        assert provision_retrieval_index(tmp_path) is None
-    assert active(path) == []
-    assert "retrieval_index_provision_failed" not in caplog.text
-    with closing(sqlite3.connect(path)) as db:
-        version = db.execute("SELECT schema_version FROM retrieval_index_state").fetchone()[0]
-    assert version == 1
-
-
-def test_a_generation_for_another_encoder_is_not_swapped_out_silently(tmp_path):
-    path = database(tmp_path)
-    store = RetrievalIndexStore(path)
-    other = LocalRetrievalEncoder(content_version=1).identity
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        store.initialize(db)
-        store.migrate_v2(db)
-        foreign = store.create_generation(db, other)
-    store.activate_generation(foreign)
-
-    assert provision_retrieval_index(tmp_path) is None
-    assert active(path) == [foreign]
-
-
-def test_without_the_built_in_encoder_no_index_is_created(tmp_path, monkeypatch):
-    path = database(tmp_path)
-    monkeypatch.setattr(
-        "one_memory.retrieval.index_provisioning.assets_present", lambda *_args: False
-    )
-
-    assert provision_retrieval_index(tmp_path) is None
-    assert active(path) == []
-    assert not (tmp_path / "retrieval-vectors").exists()
-
-
-def test_a_missing_database_is_not_created_by_provisioning(tmp_path):
-    assert provision_retrieval_index(tmp_path) is None
-    assert not (tmp_path / "one-memory.sqlite3").exists()
-    assert not (tmp_path / "retrieval-vectors").exists()
-
-
-@pytest.mark.parametrize("failure", ["initialize_v2", "create_generation"])
-def test_a_failed_provision_reports_none_instead_of_half_an_index(tmp_path, monkeypatch, failure):
-    path = database(tmp_path)
-
-    def explode(*_args, **_kwargs):
-        raise sqlite3.OperationalError("synthetic")
-
-    monkeypatch.setattr(RetrievalIndexStore, failure, explode)
-
-    assert provision_retrieval_index(tmp_path) is None
-    assert active(path) == []

memhub/apps/core/tests/test_retrieval_raw_candidates.py

@@ -1,224 +0,0 @@
-import sqlite3
-import time
-from contextlib import closing
-from datetime import datetime
-
-import pytest
-
-from one_memory.imports.models import EvidenceInput
-from one_memory.retrieval.index_worker import RetrievalIndexWorker
-from one_memory.retrieval.models import TimeRange
-from one_memory.retrieval.raw_candidates import (
-    compose_raw_retrieval_path,
-    read_raw_semantic_candidates,
-)
-from one_memory.retrieval.vector_store import RawVectorStore
-from tests.test_retrieval_index_store import fixture, initialize, put  # noqa: F401
-from tests.test_retrieval_index_worker import Encoder, forbid_network  # noqa: F401
-
-
-@pytest.fixture
-def indexed(fixture, tmp_path):  # noqa: F811
-    generation = initialize(fixture)
-    record = put(fixture[1])
-    encoder = Encoder(fixture[4])
-    vectors = RawVectorStore.create(tmp_path / "vectors", generation_id=generation, dimensions=3)
-    worker = RetrievalIndexWorker(
-        fixture[3], vectors, encoder, generation_id=generation, owner="test"
-    )
-    assert worker.run_once().state == "ready"
-    fixture[3].activate_generation(generation)
-    try:
-        yield fixture, generation, record, vectors
-    finally:
-        vectors.close()
-
-
-def search(indexed, **kwargs):
-    f, generation, _, vectors = indexed
-    options = dict(source_ids=frozenset({"codex"}), deadline=time.monotonic() + 10)
-    options.update(kwargs)
-    return read_raw_semantic_candidates(
-        f[3], vectors, [1.0, 0.5, 0.25], generation_id=generation, **options
-    )
-
-
-def test_positive_only_current_authorized_published_chunks_are_searched(indexed):
-    result = search(indexed)
-    assert result.complete and len(result.hits) == 1
-    assert result.hits[0].chunk.evidence_id == indexed[2].evidence_id
-    assert result.coverage == "indexed_only"
-
-
-def test_empty_authority_never_reads_database_or_vector(indexed, monkeypatch):
-    def forbidden(*args, **kwargs):
-        raise AssertionError("unauthorized access")
-
-    monkeypatch.setattr(indexed[0][3], "active_chunks", forbidden)
-    monkeypatch.setattr(indexed[3], "search", forbidden)
-    result = search(indexed, source_ids=frozenset())
-    assert not result.hits and result.reason == "not_authorized"
-
-
-def test_unapproved_source_never_enters_vector_query(indexed, monkeypatch):
-    monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("not eligible"))
-    assert search(indexed, source_ids=frozenset({"cursor"})).hits == ()
-
-
-@pytest.mark.parametrize("change", ["revoke", "revise", "policy"])
-def test_mutation_during_vector_query_discards_stale_candidate(indexed, monkeypatch, change):
-    f, _, _, vectors = indexed
-    original = vectors.search
-
-    def query(*args, **kwargs):
-        result = original(*args, **kwargs)
-        if change == "revise":
-            put(f[1], "changed text")
-        elif change == "revoke":
-            f[2].revoke(user_id="local", source_id="codex")
-        else:
-            with closing(sqlite3.connect(f[0])) as db, db:
-                db.execute("UPDATE source_grants SET revision=revision+1")
-        return result
-
-    monkeypatch.setattr(vectors, "search", query)
-    result = search(indexed)
-    assert not result.hits and not result.complete and result.reason == "evidence_changed"
-
-
-def test_expired_deadline_does_not_read_or_query(indexed, monkeypatch):
-    monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("late query"))
-    result = search(indexed, deadline=time.monotonic() - 1)
-    assert not result.hits and not result.complete and result.reason == "deadline"
-
-
-def test_missing_published_vector_is_error_not_empty(indexed):
-    indexed[3]._collection.delete(ids=[c.chunk_id for c in indexed[0][3].active_chunks(indexed[1])])
-    result = search(indexed)
-    assert not result.hits and not result.complete and result.reason == "vector_unavailable"
-
-
-def test_small_allowlist_uses_query_as_integrity_check_without_duplicate_get(indexed, monkeypatch):
-    monkeypatch.setattr(
-        indexed[3],
-        "verify_candidates",
-        lambda *args, **kwargs: pytest.fail("small allowlist must not be read twice"),
-    )
-    result = search(indexed)
-    assert result.complete and len(result.hits) == 1
-
-
-def test_missing_event_time_never_matches_hard_range(indexed, monkeypatch):
-    window = TimeRange(start="2026-09-18T00:00:00+08:00", end="2026-09-19T00:00:00+08:00")
-    monkeypatch.setattr(indexed[3], "search", lambda *a, **k: pytest.fail("unknown event time"))
-    assert search(indexed, time_range=window).hits == ()
-
-
-@pytest.mark.parametrize("sources", [None, ["codex"], "codex"])
-def test_caller_must_provide_explicit_resolved_authority(indexed, sources):
-    with pytest.raises(ValueError, match="source"):
-        search(indexed, source_ids=sources)
-
-
-def test_event_time_uses_utc_half_open_range_not_import_time(indexed):
-    f, generation, record, vectors = indexed
-    f[1].put(
-        EvidenceInput(
-            source_id=record.source_id,
-            conversation_id=record.conversation_id,
-            message_id=record.message_id,
-            role=record.role,
-            written_by=record.written_by,
-            verbatim=record.verbatim,
-            created_at=datetime.fromisoformat("2026-09-18T16:00:00+00:00"),
-            origin_meta=record.origin_meta,
-        )
-    )
-    worker = RetrievalIndexWorker(
-        f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"
-    )
-    assert worker.run_once().state == "ready"
-    before = TimeRange(start="2026-09-18T00:00:00+08:00", end="2026-09-19T00:00:00+08:00")
-    after = TimeRange(start="2026-09-19T00:00:00+08:00", end="2026-09-20T00:00:00+08:00")
-    assert not search(indexed, time_range=before).hits
-    result = search(indexed, time_range=after)
-    assert result.complete and len(result.hits) == 1
-    assert result.hits[0].chunk.event_time == after.start
-
-
-def test_final_deadline_failure_discards_previously_found_hits(indexed, monkeypatch):
-    from one_memory.retrieval import raw_candidates
-
-    original = raw_candidates._check
-    calls = []
-
-    def check(deadline):
-        calls.append(deadline)
-        if len(calls) == 3:
-            raise TimeoutError("late final revalidation")
-        return original(deadline)
-
-    monkeypatch.setattr(raw_candidates, "_check", check)
-    result = search(indexed)
-    assert len(calls) == 3 and not result.hits and result.reason == "deadline"
-
-
-@pytest.mark.parametrize("corruption", ["all_missing", "tail_missing", "offset", "hash"])
-def test_corrupted_published_manifest_is_unavailable_not_complete_empty(indexed, corruption):
-    f, generation, _, vectors = indexed
-    put(f[1], "long source text " * 90)
-    worker = RetrievalIndexWorker(
-        f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"
-    )
-    assert worker.run_once().state == "ready"
-    assert len(search(indexed).hits) > 1
-    with closing(sqlite3.connect(f[0])) as db, db:
-        if corruption == "all_missing":
-            db.execute("DELETE FROM retrieval_chunks")
-        elif corruption == "tail_missing":
-            db.execute("DELETE FROM retrieval_chunks WHERE ordinal>0")
-        elif corruption == "offset":
-            db.execute("UPDATE retrieval_chunks SET start_cp=3 WHERE ordinal=0")
-        else:
-            db.execute("UPDATE retrieval_chunks SET text_hash='broken'")
-    result = search(indexed)
-    assert not result.complete and not result.hits and result.reason == "index_unavailable"
-
-
-def test_null_stored_grant_with_revocation_never_queries_vectors(indexed, monkeypatch):
-    f, _, _, vectors = indexed
-    assert search(indexed).hits
-    with closing(sqlite3.connect(f[0])) as db, db:
-        db.execute("UPDATE retrieval_documents SET grant_fingerprint=NULL")
-    f[2].revoke(user_id="local", source_id="codex")
-    monkeypatch.setattr(vectors, "search", lambda *a, **k: pytest.fail("revoked vector query"))
-    assert not search(indexed).hits
-
-
-def test_unpublished_authorized_evidence_cannot_claim_complete_empty(indexed):
-    """A partial index with no hits is incomplete, not an honest empty library."""
-    f, generation, _, _vectors = indexed
-    put(f[1], "用户决定 One Memory 采用 MIT 协议发布。", message="unindexed")
-    assert f[3].consume_pending(generation) >= 1
-    pending = f[3].status(generation)
-    assert pending["pending"] >= 1
-    # Published chunks may still match a probe vector. The unpublished
-    # authorized document is what forbids complete-empty, so the candidate
-    # batch is forced empty: that is the lying-empty case, not a miss on
-    # already-indexed text.
-    empty = type(search(indexed))(hits=(), complete=True, reason="no_eligible_chunks")
-    path = compose_raw_retrieval_path(
-        empty, unpublished_authorized=pending["pending"] > 0, lexical_hit_count=0
-    )
-    assert path.coverage == "partial"
-    assert path.status != "empty"
-    assert "semantic" in path.searched_channels
-
-
-def test_complete_index_with_no_hits_may_be_empty(indexed):
-    batch = search(indexed, source_ids=frozenset({"codex"}))
-    # Query vector still matches the only indexed chunk; force a no-hit batch.
-    empty = type(batch)(hits=(), complete=True, reason="no_eligible_chunks")
-    path = compose_raw_retrieval_path(empty, unpublished_authorized=False, lexical_hit_count=0)
-    assert path.status == "empty"
-    assert path.coverage == "complete"

memhub/apps/core/tests/test_retrieval_raw_evidence.py

@@ -1,88 +0,0 @@
-import sqlite3
-import time
-from contextlib import closing
-
-import pytest
-
-from one_memory.retrieval.index_worker import RetrievalIndexWorker
-from one_memory.retrieval.raw_evidence import materialize_raw_candidates
-from tests.test_retrieval_index_store import fixture, put  # noqa: F401
-from tests.test_retrieval_index_worker import Encoder, forbid_network  # noqa: F401
-from tests.test_retrieval_raw_candidates import indexed as indexed_fixture
-from tests.test_retrieval_raw_candidates import search
-
-indexed = indexed_fixture
-
-
-def materialize(indexed, candidates=None, **kwargs):
-    options = dict(
-        source_ids=frozenset({"codex"}), deadline=time.monotonic() + 10, character_budget=6000
-    )
-    options.update(kwargs)
-    return materialize_raw_candidates(
-        indexed[0][3], candidates or search(indexed), generation_id=indexed[1], **options
-    )
-
-
-def test_materialized_text_is_exact_current_original(indexed):
-    result = materialize(indexed)
-    assert result.complete and len(result.items) == 1
-    item = result.items[0]
-    assert item.evidence_id == indexed[2].evidence_id
-    assert item.spans[0].text == indexed[2].verbatim
-    assert item.written_by == "person" and item.matched_by == ("semantic",)
-
-
-@pytest.mark.parametrize("change", ["revision", "revoke", "unrecorded"])
-def test_stale_candidates_do_not_authorize_original_read(indexed, change):
-    candidates = search(indexed)
-    assert candidates.hits
-    f = indexed[0]
-    if change == "revision":
-        put(f[1], "updated original")
-    elif change == "revoke":
-        f[2].revoke(user_id="local", source_id="codex")
-    else:
-        with closing(sqlite3.connect(f[0])) as db, db:
-            db.execute(
-                "UPDATE evidence SET verbatim='unrecorded' WHERE evidence_id=?",
-                (indexed[2].evidence_id,),
-            )
-    result = materialize(indexed, candidates)
-    assert not result.items and not result.complete and result.reason == "evidence_changed"
-
-
-def test_budget_does_not_cut_a_chunk_into_fake_partial_object(indexed):
-    result = materialize(indexed, character_budget=5)
-    assert not result.items and not result.complete and result.reason == "character_budget"
-
-
-def test_forged_offsets_and_modified_manifest_do_not_release_body(indexed):
-    candidates = search(indexed)
-    with closing(sqlite3.connect(indexed[0][0])) as db, db:
-        db.execute("UPDATE retrieval_chunks SET start_cp=2")
-    result = materialize(indexed, candidates)
-    assert not result.items and not result.complete
-
-
-def test_empty_authority_and_expired_deadline_return_no_body(indexed):
-    candidates = search(indexed)
-    assert not materialize(indexed, candidates, source_ids=frozenset()).items
-    assert not materialize(indexed, candidates, deadline=time.monotonic() - 1).items
-
-
-def test_overlapping_unicode_chunks_merge_without_duplicate_characters(indexed):
-    f, generation, _, vectors = indexed
-    text = "\u6c49\u5b57\U0001f680e\u0301 " * 180 + " /project/final-design.png"
-    put(f[1], text)
-    worker = RetrievalIndexWorker(
-        f[3], vectors, Encoder(f[4]), generation_id=generation, owner="test"
-    )
-    assert worker.run_once().state == "ready"
-    candidates = search(indexed)
-    assert len(candidates.hits) > 1
-    result = materialize(indexed, candidates)
-    assert result.complete and len(result.items) == 1
-    assert len(result.items[0].spans) == 1
-    assert result.items[0].spans[0].text == text
-    assert result.used_characters == len(text)

memhub/apps/core/tests/test_retrieval_raw_lane.py

@@ -1,356 +0,0 @@
-"""v1 raw lane must not claim complete empty when the derived index is partial."""
-
-import asyncio
-import sqlite3
-import threading
-import time
-from types import SimpleNamespace
-
-import pytest
-
-from one_memory.context_models import ContextLaneResult, ContextSearchRequest
-from one_memory.context_raw import read_raw_context
-from one_memory.database_access import main_database
-from one_memory.embedding_assets import assets_present
-from one_memory.retrieval.raw_candidates import RawCandidateBatch
-from one_memory.retrieval.raw_lane import fuse_raw_lane, read_raw_indexed_context
-from tests.test_context_raw import database as database  # noqa: F401
-from tests.test_context_raw import put
-
-
-def _require_assets() -> None:
-    """Make an absent local fixture explicit instead of a lease-style hang."""
-    if not assets_present():
-        pytest.skip("_assets_unavailable")
-
-
-def test_unpublished_authorized_evidence_degrades_empty_lexical_lane():
-    lexical = ContextLaneResult(status="empty", reason="lexical_only")
-    result = fuse_raw_lane(
-        lexical,
-        RawCandidateBatch(complete=True, reason="no_eligible_chunks"),
-        unpublished_authorized=True,
-    )
-    assert result.status == "degraded"
-    assert result.reason == "lexical_only"
-    assert result.items == ()
-
-
-def test_complete_index_with_no_hits_stays_empty():
-    lexical = ContextLaneResult(status="empty", reason="lexical_only")
-    result = fuse_raw_lane(
-        lexical,
-        RawCandidateBatch(complete=True, reason="no_eligible_chunks"),
-        unpublished_authorized=False,
-    )
-    assert result.status == "empty"
-    assert result.items == ()
-
-
-def test_interactive_raw_lane_reads_one_snapshot_while_a_writer_holds(database):
-    """The read lane must not wait for an import writer, nor see its uncommitted row.
-
-    This is the C4-owned half of the import-contention BLOCK: WAL readers are
-    expected to proceed from the last committed snapshot even while the single
-    writer holds a transaction. A regression that lets context reads block on
-    the writer - or read its uncommitted row - makes this fail.
-    """
-    with sqlite3.connect(database) as db, db:
-        db.execute("PRAGMA journal_mode=WAL")
-        put(db, "committed", "needle committed row")
-    writer_holds = threading.Event()
-    release_writer = threading.Event()
-
-    def hold_write_transaction() -> None:
-        with main_database(database).write(autocommit=True) as writer:
-            writer.execute("BEGIN IMMEDIATE")
-            writer.execute(
-                """INSERT INTO evidence
-                (evidence_id,idempotency_key,user_id,source_id,conversation_id,role,
-                 written_by,created_at,verbatim,content_hash,revision,imported_at,updated_at)
-                VALUES ('uncommitted','uncommitted','local','cursor','c','user',
-                        'person','2026-09-20','needle uncommitted row','synthetic',
-                        1,'2026-09-20','2026-09-20')"""
-            )
-            writer_holds.set()
-            assert release_writer.wait(5)
-            writer.execute("ROLLBACK")
-
-    thread = threading.Thread(target=hold_write_transaction)
-    thread.start()
-    assert writer_holds.wait(5)
-    try:
-        before = main_database(database).stats()
-        started = time.monotonic()
-        result = read_raw_context(
-            database,
-            ContextSearchRequest(query="needle", kinds=("raw",)),
-            source_ids=frozenset({"cursor"}),
-            deadline=time.monotonic() + 2,
-        )
-        elapsed = time.monotonic() - started
-        after = main_database(database).stats()
-    finally:
-        release_writer.set()
-        thread.join(timeout=5)
-    assert not thread.is_alive()
-    assert elapsed < 1.0
-    assert after.reader_waits == before.reader_waits
-    assert [item.anchor_evidence_id for item in result.items] == ["committed"]
-    assert all(
-        message.evidence_id != "uncommitted" for item in result.items for message in item.messages
-    )
-
-
-def test_authorization_and_unavailability_are_not_rewritten():
-    denied = ContextLaneResult(status="not_authorized", reason="permission_required")
-    assert (
-        fuse_raw_lane(denied, RawCandidateBatch(), unpublished_authorized=True).status
-        == "not_authorized"
-    )
-    missing = ContextLaneResult(status="unavailable", reason="read_unavailable")
-    assert fuse_raw_lane(missing, RawCandidateBatch(), unpublished_authorized=True).status == (
-        "unavailable"
-    )
-
-
-def test_indexed_reader_without_runtime_matches_lexical_empty(database):
-    with sqlite3.connect(database) as db, db:
-        put(db, "e-unrelated", "alpha transcript only")
-    request = ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",))
-    deadline = time.monotonic() + 5
-    lexical = read_raw_context(
-        database, request, source_ids=frozenset({"cursor"}), deadline=deadline
-    )
-    indexed = read_raw_indexed_context(
-        database, request, source_ids=frozenset({"cursor"}), deadline=deadline, index=None
-    )
-    assert lexical.status == "empty"
-    assert indexed == lexical
-    assert indexed.searched_channels == ("lexical",)
-    assert indexed.semantic_status == "not_searched"
-    assert indexed.index_complete is False
-
-
-def test_indexed_reader_with_unpublished_index_is_not_empty(database):
-    with sqlite3.connect(database) as db, db:
-        put(db, "e-unrelated", "alpha transcript only")
-    request = ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",))
-    index = SimpleNamespace(
-        lookup_raw=lambda query, *, source_ids, deadline: (
-            RawCandidateBatch(complete=True, reason="no_eligible_chunks"),
-            True,
-        )
-    )
-    result = read_raw_indexed_context(
-        database,
-        request,
-        source_ids=frozenset({"cursor"}),
-        deadline=time.monotonic() + 5,
-        index=index,
-    )
-    assert result.status == "degraded"
-    assert result.items == ()
-    assert result.searched_channels == ("lexical", "semantic")
-    assert result.semantic_status == "completed"
-    assert result.index_complete is False
-
-
-def test_semantic_hit_without_lexical_overlap_is_not_released(database):
-    from datetime import UTC, datetime
-
-    from one_memory.retrieval.index_store import PublishedChunk
-    from one_memory.retrieval.raw_candidates import RawSemanticCandidate
-
-    with sqlite3.connect(database) as db, db:
-        put(db, "e-mit", "用户决定 One Memory 采用 MIT 协议发布。")
-    chunk = PublishedChunk(
-        chunk_id="c1",
-        build_id="b1",
-        evidence_id="e-mit",
-        revision=1,
-        start_cp=0,
-        end_cp=8,
-        ordinal=0,
-        source_id="cursor",
-        event_time=datetime(2026, 9, 19, tzinfo=UTC),
-    )
-    index = SimpleNamespace(
-        lookup_raw=lambda query, *, source_ids, deadline: (
-            RawCandidateBatch(
-                hits=(RawSemanticCandidate(chunk, 0.1),),
-                complete=True,
-            ),
-            False,
-        )
-    )
-    result = read_raw_indexed_context(
-        database,
-        ContextSearchRequest(query="开源许可证怎么选", kinds=("raw",)),
-        source_ids=frozenset({"cursor"}),
-        deadline=time.monotonic() + 5,
-        index=index,
-    )
-    assert result.status == "degraded"
-    assert result.reason == "lexical_only"
-    assert result.items == ()
-    assert result.searched_channels == ("lexical", "semantic")
-    assert result.semantic_status == "completed"
-    assert result.index_complete is True
-
-
-def test_semantic_candidate_with_lexical_corroboration_is_released(database):
-    from datetime import UTC, datetime
-
-    from one_memory.retrieval.index_store import PublishedChunk
-    from one_memory.retrieval.raw_candidates import RawSemanticCandidate
-
-    with sqlite3.connect(database) as db, db:
-        put(db, "e-mit", "用户决定 One Memory 采用 MIT 协议发布。")
-    chunk = PublishedChunk(
-        chunk_id="c1",
-        build_id="b1",
-        evidence_id="e-mit",
-        revision=1,
-        start_cp=0,
-        end_cp=8,
-        ordinal=0,
-        source_id="cursor",
-        event_time=datetime(2026, 9, 19, tzinfo=UTC),
-    )
-    index = SimpleNamespace(
-        lookup_raw=lambda query, *, source_ids, deadline: (
-            RawCandidateBatch(
-                hits=(RawSemanticCandidate(chunk, 0.1),),
-                complete=True,
-            ),
-            False,
-        )
-    )
-    result = read_raw_indexed_context(
-        database,
-        ContextSearchRequest(query="MIT 协议", kinds=("raw",)),
-        source_ids=frozenset({"cursor"}),
-        deadline=time.monotonic() + 5,
-        index=index,
-    )
-    assert result.status == "ok"
-    assert [item.anchor_evidence_id for item in result.items] == ["e-mit"]
-
-
-def test_raw_semantic_timeout_is_not_reported_as_complete_empty(database):
-    with sqlite3.connect(database) as db, db:
-        put(db, "e-unrelated", "alpha transcript only")
-    index = SimpleNamespace(
-        lookup_raw=lambda query, *, source_ids, deadline: (
-            RawCandidateBatch(reason="deadline"),
-            False,
-        )
-    )
-    result = read_raw_indexed_context(
-        database,
-        ContextSearchRequest(query="no-matching-zzzzzz", kinds=("raw",)),
-        source_ids=frozenset({"cursor"}),
-        deadline=time.monotonic() + 5,
-        index=index,
-    )
-    assert result.status == "degraded"
-    assert result.semantic_status == "timed_out"
-    assert result.index_complete is False
-    assert result.timed_out
-
-
-def test_real_built_index_semantic_only_hit_is_not_released_without_confirmation(tmp_path):
-    """The whole chain for real: ONNX encode, Chroma, published chunks, authority.
-
-    Every other test here hands the lane a hand-written candidate, so none of
-    them can tell whether a real build would ever produce one. The query is
-    chosen to share no term with the stored sentence, and the lexical-only read
-    is asserted empty first, so an `ok` here can only have come from the index.
-    """
-    _require_assets()
-    from one_memory.imports.grants import SourceGrantStore
-    from one_memory.imports.sources.registry import RootCandidateKind
-    from one_memory.retrieval.runtime import ExistingRetrievalRuntime
-    from tests.test_retrieval_index_store import put as put_evidence
-    from tests.test_retrieval_runtime import configured, until
-
-    store, evidence, generation = configured(tmp_path)
-    source = tmp_path / "source"
-    source.mkdir()
-    SourceGrantStore(store.database_path, home=tmp_path).grant(
-        user_id="local",
-        source_id="codex",
-        root=source,
-        candidate_kind=RootCandidateKind.DEFAULT,
-    )
-    put_evidence(evidence, "我们最后决定 One Memory 用 MIT 协议发布。")
-    request = ContextSearchRequest(query="开源软件的授权条款如何挑选", kinds=("raw",))
-    sources = frozenset({"codex"})
-
-    lexical = read_raw_context(
-        store.database_path, request, source_ids=sources, deadline=time.monotonic() + 5
-    )
-    assert lexical.status == "empty", "词法已经命中，这条测试证明不了语义"
-
-    async def scenario():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        assert await handle.start()
-        try:
-            await until(lambda: store.status(generation)["ready"] >= 1)
-            return await asyncio.to_thread(
-                read_raw_indexed_context,
-                store.database_path,
-                request,
-                source_ids=sources,
-                deadline=time.monotonic() + 10,
-                index=handle,
-            )
-        finally:
-            await handle.stop()
-
-    result = asyncio.run(scenario())
-    # The semantic lane found the row, but lexical-only evidence is not enough
-    # confirmation to release it to a host. Keep the candidate visible as a
-    # degraded coverage receipt until a relevance/corroboration gate exists.
-    assert result.status == "degraded"
-    assert result.reason == "lexical_only"
-    assert result.items == ()
-
-
-def test_real_built_index_skips_with_a_named_asset_reason(tmp_path, monkeypatch):
-    """A missing untracked fixture must not look like a lease or worker hang."""
-    monkeypatch.setattr("test_retrieval_raw_lane.assets_present", lambda: False)
-    with pytest.raises(pytest.skip.Exception, match="_assets_unavailable"):
-        test_real_built_index_semantic_only_hit_is_not_released_without_confirmation(tmp_path)
-
-
-def test_revoked_source_semantic_hit_does_not_release_body(database):
-    from datetime import UTC, datetime
-
-    from one_memory.retrieval.index_store import PublishedChunk
-    from one_memory.retrieval.raw_candidates import RawSemanticCandidate
-    from one_memory.retrieval.raw_lane import load_raw_items_from_candidates
-    from tests.test_context_raw import revoke
-
-    with sqlite3.connect(database) as db, db:
-        put(db, "e-revoked", "用户决定 One Memory 采用 MIT 协议发布。")
-        revoke(db, "cursor")
-    chunk = PublishedChunk(
-        chunk_id="c1",
-        build_id="b1",
-        evidence_id="e-revoked",
-        revision=1,
-        start_cp=0,
-        end_cp=8,
-        ordinal=0,
-        source_id="cursor",
-        event_time=datetime(2026, 9, 19, tzinfo=UTC),
-    )
-    items = load_raw_items_from_candidates(
-        database,
-        RawCandidateBatch(hits=(RawSemanticCandidate(chunk, 0.1),), complete=True),
-        source_ids=frozenset({"cursor"}),
-        deadline=time.monotonic() + 5,
-    )
-    assert items == ()

memhub/apps/core/tests/test_retrieval_runtime.py

@@ -1,522 +0,0 @@
-import asyncio
-import json
-import os
-import shutil
-import socket
-import sqlite3
-import threading
-import time
-from contextlib import closing
-
-import pytest
-
-from one_memory.imports.grants import SourceGrantStore
-from one_memory.imports.sources.registry import RootCandidateKind
-from one_memory.imports.storage import EvidenceStore
-from one_memory.retrieval import runtime
-from one_memory.retrieval.index_store import RetrievalIndexStore
-from one_memory.retrieval.index_worker import IndexWorkResult, LocalRetrievalEncoder
-from one_memory.retrieval.runtime import ExistingRetrievalRuntime
-from one_memory.retrieval.vector_store import RawVectorStore
-from tests.test_retrieval_index_store import put
-
-
-@pytest.fixture(autouse=True)
-def offline(monkeypatch):
-    def reject(*args, **kwargs):
-        raise AssertionError("offline runtime test")
-
-    monkeypatch.setattr(socket.socket, "connect", reject)
-    monkeypatch.setattr(socket.socket, "connect_ex", reject)
-
-
-def configured(tmp_path, *, version=2, collection=True):
-    path = tmp_path / "one-memory.sqlite3"
-    evidence = EvidenceStore(path)
-    identity = LocalRetrievalEncoder(content_version=version).identity
-    store = RetrievalIndexStore(path)
-    with closing(sqlite3.connect(path)) as db, db:
-        db.execute("BEGIN IMMEDIATE")
-        store.initialize(db)
-        store.migrate_v2(db)
-        generation = store.create_generation(db, identity)
-    store.activate_generation(generation)
-    if collection:
-        vectors = RawVectorStore.create(
-            tmp_path / "retrieval-vectors", generation_id=generation, dimensions=identity.dimensions
-        )
-        vectors.close()
-    return store, evidence, generation
-
-
-async def until(predicate):
-    end = time.monotonic() + 10
-    while not predicate():
-        assert time.monotonic() < end
-        await asyncio.sleep(0.01)
-
-
-def test_constructor_and_missing_start_create_nothing(tmp_path):
-    target = tmp_path / "missing"
-    handle = ExistingRetrievalRuntime(target)
-    assert not target.exists()
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().state == "not_initialized"
-    assert not target.exists()
-    asyncio.run(handle.stop())
-
-
-def test_stop_releases_supervisor_worker_but_preserves_immutable_snapshot(tmp_path):
-    configured(tmp_path)
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-
-    async def scenario():
-        assert await handle.start()
-        assert handle._supervisor is not None
-        await handle.stop()
-
-    asyncio.run(scenario())
-    assert handle._supervisor is None
-    assert handle.snapshot().supervisor is not None
-    assert not handle.snapshot().supervisor.thread_alive
-
-
-@pytest.mark.parametrize(
-    "kind",
-    ["schema1", "no_active", "missing_collection", "bad_identity", "extra_identity", "two_active"],
-)
-def test_not_initialized_or_invalid_configuration_never_creates_index(tmp_path, kind):
-    store, _, gen = configured(tmp_path, collection=False)
-    with closing(sqlite3.connect(store.database_path)) as db, db:
-        if kind == "schema1":
-            db.execute("UPDATE retrieval_index_state SET schema_version=1")
-        elif kind == "no_active":
-            db.execute("UPDATE retrieval_index_generations SET state='building'")
-        elif kind in ("bad_identity", "extra_identity"):
-            raw = json.loads(
-                db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]
-            )
-            if kind == "bad_identity":
-                raw["encoder_digest"] = "0" * 64
-            else:
-                raw["unexpected"] = "nope"
-            db.execute("UPDATE retrieval_index_generations SET identity_json=?", (json.dumps(raw),))
-        elif kind == "two_active":
-            db.execute("DROP INDEX retrieval_one_active")
-            db.execute(
-                "INSERT INTO retrieval_index_generations SELECT 'other',identity_json,state,"
-                "created_at FROM retrieval_index_generations"
-            )
-        before = db.execute("SELECT * FROM retrieval_index_generations").fetchall()
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().state in {"not_initialized", "failed"}
-    assert not (tmp_path / "retrieval-vectors").exists()
-    with closing(sqlite3.connect(store.database_path)) as db:
-        assert db.execute("SELECT * FROM retrieval_index_generations").fetchall() == before
-    assert str(tmp_path) not in repr(handle.snapshot())
-
-
-def test_low_disk_rejects_before_open_writer(tmp_path, monkeypatch):
-    configured(tmp_path)
-    monkeypatch.setattr(
-        runtime.shutil, "disk_usage", lambda _: shutil._ntuple_diskusage(1000, 999, 1)
-    )
-    monkeypatch.setattr(
-        runtime.RawVectorStore,
-        "open_writer_existing",
-        lambda *a, **k: pytest.fail("opened on low disk"),
-    )
-    handle = ExistingRetrievalRuntime(tmp_path)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().error_code == "retrieval_runtime_low_disk"
-
-
-@pytest.mark.parametrize("kind", ["root", "database", "vectors"])
-def test_symlink_rejected(tmp_path, kind):
-    actual = tmp_path / "actual"
-    actual.mkdir()
-    configured(actual)
-    selected = actual
-    if kind == "root":
-        selected = tmp_path / "link"
-        selected.symlink_to(actual, target_is_directory=True)
-    else:
-        path = actual / ("one-memory.sqlite3" if kind == "database" else "retrieval-vectors")
-        moved = tmp_path / "moved"
-        path.rename(moved)
-        path.symlink_to(moved, target_is_directory=kind == "vectors")
-    handle = ExistingRetrievalRuntime(selected, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"
-
-
-def test_local_existing_integration_indexes_only_after_explicit_start(tmp_path):
-    store, evidence, gen = configured(tmp_path)
-    source = tmp_path / "source"
-    source.mkdir()
-    SourceGrantStore(store.database_path, home=tmp_path).grant(
-        user_id="local", source_id="codex", root=source, candidate_kind=RootCandidateKind.DEFAULT
-    )
-    row = put(evidence, "before token=private after")
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        assert store.status(gen)["ready"] == 0
-        assert await handle.start()
-        try:
-            await until(lambda: store.status(gen)["ready"] == 1)
-            assert len(store.active_chunks(gen)) == 2
-            assert handle.snapshot().generation_id == gen
-            with pytest.raises(RuntimeError, match="already_started"):
-                await handle.start()
-        finally:
-            await handle.stop()
-        assert handle.snapshot().state == "stopped"
-        assert not handle.snapshot().supervisor.thread_alive
-        assert evidence.get(row.evidence_id).verbatim == row.verbatim
-
-    asyncio.run(run())
-
-
-def fake_resources(tmp_path, monkeypatch, *, fail_worker=False, block_open=False, version=2):
-    configured(tmp_path, version=version)
-    entered, release, exited, opened = (threading.Event() for _ in range(4))
-    events = []
-
-    class Vectors:
-        def close(self):
-            events.append("close")
-
-    def open_vectors(*args, **kwargs):
-        opened.set()
-        if block_open:
-            assert release.wait(5)
-        return Vectors()
-
-    class Worker:
-        def __init__(self, *args, **kwargs):
-            if fail_worker:
-                raise RuntimeError("SECRET failure")
-
-        def run_once(self, *, cancel):
-            entered.set()
-            while not cancel.is_set():
-                time.sleep(0.001)
-            events.append("cancel")
-            assert release.wait(5)
-            events.append("exit")
-            exited.set()
-            return IndexWorkResult("cancelled")
-
-    monkeypatch.setattr(runtime.RawVectorStore, "open_writer_existing", open_vectors)
-    monkeypatch.setattr(runtime, "RetrievalIndexWorker", Worker)
-    return entered, release, exited, opened, events
-
-
-def test_partial_start_failure_closes_owned_vectors(tmp_path, monkeypatch):
-    *_, events = fake_resources(tmp_path, monkeypatch, fail_worker=True)
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert events == ["close"]
-    assert handle.snapshot().error_code == "retrieval_runtime_start_failed"
-    assert "SECRET" not in repr(handle.snapshot())
-
-
-def test_cancelled_stop_drains_before_close(tmp_path, monkeypatch):
-    entered, release, exited, _, events = fake_resources(tmp_path, monkeypatch)
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        assert await handle.start()
-        await until(entered.is_set)
-        task = asyncio.create_task(handle.stop())
-        await asyncio.sleep(0)
-        task.cancel()
-        try:
-            await until(lambda: "cancel" in events)
-            task.cancel()
-            await asyncio.sleep(0.03)
-            assert not task.done() and "close" not in events
-        finally:
-            release.set()
-            with pytest.raises(asyncio.CancelledError):
-                await task
-        assert exited.is_set() and events == ["cancel", "exit", "close"]
-        await handle.stop()
-        assert events.count("close") == 1
-
-    asyncio.run(run())
-
-
-def test_cancelled_start_drains_open_and_cleans_resources(tmp_path, monkeypatch):
-    _, release, _, opened, events = fake_resources(
-        tmp_path, monkeypatch, fail_worker=True, block_open=True
-    )
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        task = asyncio.create_task(handle.start())
-        await until(opened.is_set)
-        task.cancel()
-        try:
-            await asyncio.sleep(0.03)
-            assert not task.done() and not events
-        finally:
-            release.set()
-            with pytest.raises(asyncio.CancelledError):
-                await task
-        assert events == ["close"]
-
-    asyncio.run(run())
-
-
-def test_cancelled_successful_open_does_not_start_worker_and_closes_once(tmp_path, monkeypatch):
-    entered, release, _, opened, events = fake_resources(tmp_path, monkeypatch, block_open=True)
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        starting = asyncio.create_task(handle.start())
-        await until(opened.is_set)
-        starting.cancel()
-        await asyncio.sleep(0.02)
-        starting.cancel()
-        try:
-            await asyncio.sleep(0.02)
-            assert not starting.done() and not events and not entered.is_set()
-        finally:
-            release.set()
-            with pytest.raises(asyncio.CancelledError):
-                await starting
-        assert events == ["close"] and not entered.is_set()
-        assert handle.snapshot().state == "stopped"
-        await handle.stop()
-        assert events == ["close"]
-
-    asyncio.run(run())
-
-
-def test_old_identity_one_recovers_without_generation_upgrade(tmp_path, monkeypatch):
-    entered, release, _, _, events = fake_resources(tmp_path, monkeypatch, version=1)
-    path = tmp_path / "one-memory.sqlite3"
-    with closing(sqlite3.connect(path)) as db:
-        before = db.execute("SELECT * FROM retrieval_index_generations").fetchall()
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        assert await handle.start()
-        await until(entered.is_set)
-        release.set()
-        await handle.stop()
-
-    asyncio.run(run())
-    assert events == ["cancel", "exit", "close"]
-    with closing(sqlite3.connect(path)) as db:
-        assert db.execute("SELECT * FROM retrieval_index_generations").fetchall() == before
-
-
-def test_missing_identity_fields_fail_even_when_dataclass_defaults_match(tmp_path):
-    store, _, _ = configured(tmp_path, version=1, collection=False)
-    with closing(sqlite3.connect(store.database_path)) as db, db:
-        raw = json.loads(
-            db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]
-        )
-        raw.pop("normalizer_version")
-        db.execute("UPDATE retrieval_index_generations SET identity_json=?", (json.dumps(raw),))
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().error_code == "retrieval_runtime_invalid_index"
-
-
-def test_valid_hash_but_foreign_encoder_never_opens_vectors(tmp_path, monkeypatch):
-    from dataclasses import asdict
-
-    from one_memory.retrieval.index_store import EncoderIdentity, digest
-
-    store, _, _ = configured(tmp_path)
-    with closing(sqlite3.connect(store.database_path)) as db, db:
-        raw = json.loads(
-            db.execute("SELECT identity_json FROM retrieval_index_generations").fetchone()[0]
-        )
-        raw["encoder_digest"] = "0" * 64
-        identity = EncoderIdentity(**raw)
-        store_id = db.execute("SELECT data_store_id FROM retrieval_index_state").fetchone()[0]
-        db.execute(
-            "UPDATE retrieval_index_generations SET generation_id=?,identity_json=?",
-            (digest([store_id, asdict(identity)]), json.dumps(raw)),
-        )
-    monkeypatch.setattr(
-        runtime.RawVectorStore,
-        "open_writer_existing",
-        lambda *a, **k: pytest.fail("foreign encoder opened"),
-    )
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().error_code == "retrieval_runtime_encoder_mismatch"
-
-
-def test_concurrent_stop_waits_start_and_runs_cleanup_once(tmp_path, monkeypatch):
-    entered, release, _, opened, events = fake_resources(tmp_path, monkeypatch, block_open=True)
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        starting = asyncio.create_task(handle.start())
-        await until(opened.is_set)
-        stops = [asyncio.create_task(handle.stop()), asyncio.create_task(handle.stop())]
-        try:
-            await asyncio.sleep(0.03)
-            assert all(not task.done() for task in stops)
-        finally:
-            release.set()
-            await asyncio.gather(*stops)
-            assert not await starting
-        assert events == ["close"] and not entered.is_set()
-
-    asyncio.run(run())
-
-
-def test_stop_before_start_never_opens_resources(tmp_path):
-    handle = ExistingRetrievalRuntime(tmp_path)
-    asyncio.run(handle.stop())
-    with pytest.raises(RuntimeError, match="already_started"):
-        asyncio.run(handle.start())
-    assert not list(tmp_path.iterdir())
-
-
-def test_missing_collection_in_existing_chroma_does_not_create_collection(tmp_path):
-    store, _, gen = configured(tmp_path)
-    with closing(sqlite3.connect(tmp_path / "retrieval-vectors" / "chroma.sqlite3")) as db, db:
-        db.execute("UPDATE collections SET name='foreign_collection'")
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().state == "not_initialized"
-    with closing(sqlite3.connect(tmp_path / "retrieval-vectors" / "chroma.sqlite3")) as db:
-        assert db.execute("SELECT name FROM collections").fetchall() == [("foreign_collection",)]
-
-
-@pytest.mark.parametrize("target", ["root", "database", "vectors", "vector_database"])
-def test_group_writable_paths_rejected_without_open(tmp_path, monkeypatch, target):
-    configured(tmp_path)
-    paths = {
-        "root": tmp_path,
-        "database": tmp_path / "one-memory.sqlite3",
-        "vectors": tmp_path / "retrieval-vectors",
-        "vector_database": tmp_path / "retrieval-vectors/chroma.sqlite3",
-    }
-    path = paths[target]
-    original_mode = path.stat().st_mode & 0o777
-    path.chmod(original_mode | 0o020)
-    monkeypatch.setattr(
-        runtime.RawVectorStore,
-        "open_writer_existing",
-        lambda *a, **k: pytest.fail("unsafe resource opened"),
-    )
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    try:
-        assert not asyncio.run(handle.start())
-        assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"
-    finally:
-        path.chmod(original_mode)
-
-
-def test_foreign_uid_and_windows_are_failclosed(tmp_path, monkeypatch):
-    configured(tmp_path)
-    uid = os.getuid()
-    monkeypatch.setattr(runtime.os, "getuid", lambda: uid + 1)
-    handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().error_code == "retrieval_runtime_unsafe_path"
-    monkeypatch.setattr(runtime.os, "getuid", lambda: uid)
-    monkeypatch.setattr(runtime.sys, "platform", "win32")
-    windows = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-    assert not asyncio.run(windows.start())
-    assert windows.snapshot().error_code == "retrieval_runtime_platform_unsupported"
-
-
-@pytest.mark.parametrize("target", ["root", "database"])
-def test_identical_store_id_copy_replacement_during_open_is_rejected(tmp_path, monkeypatch, target):
-    root = tmp_path / "profile"
-    root.mkdir()
-    configured(root)
-    closed = []
-
-    class Vectors:
-        def close(self):
-            closed.append(True)
-
-    def replace_path(*args, **kwargs):
-        if target == "database":
-            path = root / "one-memory.sqlite3"
-            copy = root / "replacement.sqlite3"
-            shutil.copy2(path, copy)
-            os.replace(copy, path)
-        else:
-            old = tmp_path / "old-profile"
-            root.rename(old)
-            shutil.copytree(old, root)
-        return Vectors()
-
-    monkeypatch.setattr(runtime.RawVectorStore, "open_writer_existing", replace_path)
-    handle = ExistingRetrievalRuntime(root, minimum_free_bytes=0)
-    assert not asyncio.run(handle.start())
-    assert handle.snapshot().error_code == "retrieval_runtime_index_changed"
-    assert closed == [True]
-
-
-def test_failed_drain_keeps_live_worker_vectors_open_and_raises_fixed_code(tmp_path, monkeypatch):
-    entered, release, _, _, events = fake_resources(tmp_path, monkeypatch)
-    actual = runtime.RetrievalIndexSupervisor
-    instances = []
-
-    class BadStop(actual):
-        def __init__(self, worker, **kwargs):
-            super().__init__(worker, **kwargs)
-            instances.append(self)
-
-        async def stop(self):
-            raise RuntimeError("SECRET_STOP_ERROR")
-
-    monkeypatch.setattr(runtime, "RetrievalIndexSupervisor", BadStop)
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        assert await handle.start()
-        await until(entered.is_set)
-        try:
-            with pytest.raises(RuntimeError, match="^retrieval_runtime_stop_failed$"):
-                await handle.stop()
-            assert "close" not in events
-            assert handle.snapshot().supervisor.thread_alive
-            assert handle.snapshot().state == "failed"
-            assert "SECRET" not in repr(handle.snapshot())
-        finally:
-            release.set()
-            await actual.stop(instances[0])
-            # Test alone owns cleanup of the intentionally broken supervisor.
-            handle._vectors.close()
-
-    asyncio.run(run())
-
-
-def test_stop_error_after_confirmed_exit_still_closes_vectors(tmp_path, monkeypatch):
-    entered, release, _, _, events = fake_resources(tmp_path, monkeypatch)
-    actual = runtime.RetrievalIndexSupervisor
-
-    class BadStop(actual):
-        async def stop(self):
-            await super().stop()
-            raise RuntimeError("SECRET_AFTER_EXIT")
-
-    monkeypatch.setattr(runtime, "RetrievalIndexSupervisor", BadStop)
-
-    async def run():
-        handle = ExistingRetrievalRuntime(tmp_path, minimum_free_bytes=0)
-        assert await handle.start()
-        await until(entered.is_set)
-        release.set()
-        with pytest.raises(RuntimeError, match="^retrieval_runtime_stop_failed$"):
-            await handle.stop()
-        assert events == ["cancel", "exit", "close"]
-        assert not handle.snapshot().supervisor.thread_alive
-
-    asyncio.run(run())

memhub/apps/core/tests/test_retrieval_vector_store.py

@@ -1,528 +0,0 @@
-import math
-import socket
-from time import monotonic
-
-import numpy as np
-import pytest
-
-from one_memory.retrieval.vector_store import (
-    RawVectorRecord,
-    RawVectorStore,
-    VectorStoreError,
-    _unit_vector,
-)
-
-
-@pytest.fixture(autouse=True)
-def no_network(monkeypatch):
-    def refused(*args, **kwargs):
-        pytest.fail("vector adapter attempted network")
-
-    monkeypatch.setattr(socket.socket, "connect", refused)
-    monkeypatch.setattr(socket, "create_connection", refused)
-
-
-@pytest.fixture
-def store(tmp_path):
-    value = RawVectorStore.create(
-        tmp_path / "retrieval-vectors", generation_id="generation-a", dimensions=3
-    )
-    try:
-        yield value
-    finally:
-        value.close()
-
-
-def record(chunk="chunk-a", *, build="build-a", generation="generation-a", vector=(1.0, 0.0, 0.0)):
-    return RawVectorRecord(chunk, build, generation, vector)
-
-
-def test_constructor_does_no_io_and_unopened_calls_do_not_create(tmp_path):
-    directory = tmp_path / "never-created"
-    value = RawVectorStore(directory, generation_id="generation-a", dimensions=3)
-    assert not directory.exists()
-    with pytest.raises(VectorStoreError, match="not_open"):
-        value.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 1)
-    assert not directory.exists()
-    value.close()
-
-
-def test_open_existing_missing_store_or_collection_never_creates(tmp_path, store):
-    missing = tmp_path / "missing"
-    with pytest.raises(VectorStoreError, match="not_found"):
-        RawVectorStore.open_existing(missing, generation_id="generation-a", dimensions=3)
-    assert not missing.exists()
-    before = store._client.list_collections()
-    with pytest.raises(VectorStoreError, match="not_found"):
-        RawVectorStore.open_existing(store.directory, generation_id="other", dimensions=3)
-    assert store._client.list_collections() == before
-
-
-def test_real_chroma_collection_has_no_embedding_function_or_documents(store):
-    store.upsert([record()])
-    collection = store._collection
-    assert collection._embedding_function is None
-    result = collection.get(ids=["chunk-a"], include=["embeddings", "metadatas", "documents"])
-    assert result["documents"] == [None]
-    assert result["metadatas"][0]["chunk_id"] == "chunk-a"
-    assert result["metadatas"][0]["build_id"] == "build-a"
-    assert result["metadatas"][0]["generation_id"] == "generation-a"
-    assert len(result["embeddings"][0]) == 3
-    store.verify([record()])
-
-
-def test_existing_read_handle_cannot_write_or_repair_and_releases_shared_client(store):
-    store.upsert([record()])
-    opened = RawVectorStore.open_existing(
-        store.directory, generation_id="generation-a", dimensions=3
-    )
-    try:
-        result = opened.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-        assert result.complete and result.hits[0].chunk_id == "chunk-a"
-        with pytest.raises(VectorStoreError, match="read_only"):
-            opened.upsert([record("new")])
-        opened.verify([record()])
-    finally:
-        opened.close()
-    store.verify([record()])
-    with pytest.raises(VectorStoreError, match="closed"):
-        opened.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 1)
-
-
-@pytest.mark.parametrize("changed", [record(build="different"), record(vector=(0.0, 1.0, 0.0))])
-def test_immutable_id_rejects_different_metadata_or_vector_without_overwriting(store, changed):
-    store.upsert([record()])
-    store.upsert([record()])
-    with pytest.raises(VectorStoreError, match="immutable_conflict"):
-        store.upsert([record("new"), changed])
-    store.verify([record()])
-    assert store._collection.get(ids=["new"])["ids"] == []
-
-
-def test_verify_detects_missing_and_tampered_records(store):
-    with pytest.raises(VectorStoreError, match="missing"):
-        store.verify([record()])
-    store.upsert([record()])
-    store._collection.update(ids=["chunk-a"], embeddings=[[0.0, 1.0, 0.0]])
-    with pytest.raises(VectorStoreError):
-        store.verify([record()])
-    with pytest.raises(VectorStoreError):
-        store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-
-
-@pytest.mark.parametrize(
-    "vector",
-    [
-        (1.0, 0.0),
-        (1.0, 0.0, math.nan),
-        (math.inf, 0.0, 1.0),
-        (0.0, 0.0, 0.0),
-        (True, 0.0, 1.0),
-        (1e100, 0.0, 1.0),
-    ],
-)
-def test_invalid_vectors_refused_before_writes_or_search(store, vector):
-    with pytest.raises(VectorStoreError):
-        store.upsert([record(vector=vector)])
-    with pytest.raises(VectorStoreError):
-        store.search(vector, [], 1, monotonic() + 1)
-    assert store._collection.count() == 0
-
-
-def test_wrong_generation_and_duplicate_ids_rejected(store):
-    with pytest.raises(VectorStoreError):
-        store.upsert([record(generation="other")])
-    with pytest.raises(VectorStoreError):
-        store.upsert([record(), record()])
-    assert store._collection.count() == 0
-
-
-def test_empty_eligible_ids_never_queries_all_collection(store, monkeypatch):
-    store.upsert([record()])
-    monkeypatch.setattr(
-        store._collection, "query", lambda **kwargs: pytest.fail("unfiltered query")
-    )
-    result = store.search((1.0, 0.0, 0.0), [], 3, monotonic() - 1)
-    assert result.complete and result.hits == ()
-    assert result.eligible_count == result.searched_count == 0
-
-
-def test_real_in_filter_excludes_closer_unapproved_id_and_matches_small_cosine_oracle(store):
-    rows = [
-        record("denied", vector=(1.0, 0.0, 0.0)),
-        record("b", vector=(0.9, 0.1, 0.0)),
-        record("c", vector=(0.0, 1.0, 0.0)),
-        record("d", vector=(-1.0, 0.0, 0.0)),
-    ]
-    store.upsert(rows)
-    result = store.search((1.0, 0.0, 0.0), ["b", "c", "d"], 3, monotonic() + 10)
-    assert result.complete and result.searched_count == 3
-    assert [hit.chunk_id for hit in result.hits] == ["b", "c", "d"]
-    for hit, row in zip(result.hits, rows[1:], strict=True):
-        vector = np.array(row.vector, dtype=np.float32)
-        expected = 1 - float(vector[0] / np.linalg.norm(vector))
-        assert hit.distance == pytest.approx(expected, abs=1e-5)
-
-
-def test_eligible_batches_cover_more_than_2048_and_merge_topk(store, monkeypatch):
-    ids = [f"chunk-{i:04d}" for i in range(2053)]
-    rows = [record(chunk, vector=(float(i + 1), 1.0, 0.0)) for i, chunk in enumerate(ids)]
-    store.upsert(rows)
-    calls = []
-    get = store._collection.get
-
-    def tracked(**kwargs):
-        calls.append(kwargs)
-        return get(**kwargs)
-
-    monkeypatch.setattr(store._collection, "get", tracked)
-    result = store.search((1.0, 0.0, 0.0), ids, 3, monotonic() + 30)
-    assert result.complete and result.searched_count == 2053
-    assert len(calls) == 2
-    assert [len(call["ids"]) for call in calls] == [2048, 5]
-    assert set().union(*(set(call["ids"]) for call in calls)) == set(ids)
-    assert all(call["include"] == ["embeddings", "metadatas"] for call in calls)
-    assert len(result.hits) == 3
-    # The exact ranking over every eligible chunk, not one batch's local top-K:
-    # a per-batch top-K would have kept the second batch's steepest vectors out.
-    # Distances come from the stored unit vectors, so float32 rounding decides
-    # the top of this ranking just as it does for the adapter.
-    stored = {row.chunk_id: np.array(_unit_vector(row.vector), dtype=np.float32) for row in rows}
-    expected = {
-        chunk_id: float(1 - np.dot(unit, np.array([1.0, 0.0, 0.0], dtype=np.float32)))
-        for chunk_id, unit in stored.items()
-    }
-    ranked = sorted(expected.items(), key=lambda pair: (pair[1], pair[0]))[:3]
-    assert [hit.chunk_id for hit in result.hits] == [chunk for chunk, _ in ranked]
-    for hit in result.hits:
-        assert hit.distance == pytest.approx(expected[hit.chunk_id], abs=1e-6)
-
-
-def test_deadline_before_query_and_late_batch_return_are_partial(store, monkeypatch):
-    store.upsert([record()])
-    calls = []
-    now = [0.0]
-    store._clock = lambda: now[0]
-    get = store._collection.get
-
-    def late(**kwargs):
-        calls.append(kwargs)
-        result = get(**kwargs)
-        now[0] = 2.0
-        return result
-
-    monkeypatch.setattr(store._collection, "get", late)
-    first = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, 0.0)
-    assert not first.complete and first.reason == "deadline" and first.hits == ()
-    assert calls == []
-    second = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, 1.0)
-    assert not second.complete and second.searched_count == 0 and second.hits == ()
-    assert len(calls) == 1
-
-
-def test_search_never_scores_a_vector_the_caller_did_not_declare_eligible(store, monkeypatch):
-    """Chroma's own filtered query is no longer used at all.
-
-    It does not score every ID in its `where $in` allowlist, so a query could
-    come back short while claiming completion, and a global query filtered
-    afterwards would let unauthorized vectors take the eligible ones' places.
-    """
-    store.upsert([record(), record("denied", vector=(1.0, 0.0, 0.0))])
-    monkeypatch.setattr(
-        store._collection,
-        "query",
-        lambda **kwargs: pytest.fail("global or filtered ANN query is not used"),
-    )
-    result = store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-    assert result.complete and [hit.chunk_id for hit in result.hits] == ["chunk-a"]
-
-
-def test_an_eligible_vector_missing_from_the_backend_is_unavailable_not_a_miss(store):
-    """The caller published this chunk, so an absent vector is a broken build."""
-    store.upsert([record("present")])
-    with pytest.raises(VectorStoreError, match="^vector_query_unavailable$"):
-        store.search((1.0, 0.0, 0.0), ["present", "absent"], 2, monotonic() + 10)
-
-
-def test_open_existing_validates_generation_dimensions_and_collection_config(store):
-    with pytest.raises(VectorStoreError):
-        RawVectorStore.open_existing(store.directory, generation_id="generation-a", dimensions=4)
-    store._collection.modify(metadata={"generation_id": "other"})
-    with pytest.raises(VectorStoreError):
-        RawVectorStore.open_existing(store.directory, generation_id="generation-a", dimensions=3)
-
-
-def test_open_existing_refuses_symlink_database_before_chroma(tmp_path):
-    directory = tmp_path / "retrieval-vectors"
-    directory.mkdir()
-    target = tmp_path / "not-a-chroma-db"
-    target.write_bytes(b"do not open")
-    (directory / "chroma.sqlite3").symlink_to(target)
-    with pytest.raises(VectorStoreError):
-        RawVectorStore.open_existing(directory, generation_id="generation-a", dimensions=3)
-    assert target.read_bytes() == b"do not open"
-
-
-def test_writer_reopen_resumes_idempotently_but_query_handle_still_cannot_write(tmp_path):
-    directory = tmp_path / "retrieval-vectors"
-    original = RawVectorStore.create(directory, generation_id="generation-a", dimensions=3)
-    try:
-        original.upsert([record(vector=(1.0, 1.0, 0.0))])
-    finally:
-        original.close()
-    writer = RawVectorStore.open_writer_existing(
-        directory, generation_id="generation-a", dimensions=3
-    )
-    try:
-        writer.upsert([record(vector=(1.0, 1.0, 0.0))])
-        writer.verify([record(vector=(1.0, 1.0, 0.0))])
-        with pytest.raises(VectorStoreError, match="immutable_conflict"):
-            writer.upsert([record(build="other", vector=(1.0, 1.0, 0.0))])
-        with pytest.raises(VectorStoreError, match="immutable_conflict"):
-            writer.upsert([record(vector=(2.0, 2.0, 0.0))])
-        assert writer._collection.count() == 1
-    finally:
-        writer.close()
-    reader = RawVectorStore.open_existing(directory, generation_id="generation-a", dimensions=3)
-    try:
-        with pytest.raises(VectorStoreError, match="read_only"):
-            reader.upsert([record("new")])
-        result = reader.search((1.0, 1.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-        assert result.complete and result.hits[0].distance == pytest.approx(0, abs=1e-5)
-    finally:
-        reader.close()
-
-
-def test_writer_existing_does_not_create_missing_store_or_collection(tmp_path, store):
-    missing = tmp_path / "writer-missing"
-    with pytest.raises(VectorStoreError, match="not_found"):
-        RawVectorStore.open_writer_existing(missing, generation_id="generation-a", dimensions=3)
-    assert not missing.exists()
-    with pytest.raises(VectorStoreError, match="not_found"):
-        RawVectorStore.open_writer_existing(store.directory, generation_id="other", dimensions=3)
-
-
-def test_partial_keeps_only_completed_eligible_batches(store, monkeypatch):
-    ids = [f"present-{i}" for i in range(2048)] + ["last"]
-    store.upsert([*(record(chunk, vector=(1.0, float(i + 1), 0.0)) for i, chunk in enumerate(ids))])
-    now = [0.0]
-    store._clock = lambda: now[0]
-    get = store._collection.get
-    calls = []
-
-    def timed(**kwargs):
-        calls.append(kwargs)
-        result = get(**kwargs)
-        if len(calls) == 2:
-            now[0] = 2.0
-        return result
-
-    monkeypatch.setattr(store._collection, "get", timed)
-    result = store.search((1.0, 0.0, 0.0), ids, 2, 1.0)
-    assert not result.complete and result.reason == "deadline"
-    assert result.searched_count == 2048 and result.eligible_count == 2049
-    assert [hit.chunk_id for hit in result.hits] == ["present-0", "present-1"]
-
-
-@pytest.mark.parametrize("mutation", ["metadata", "dimension", "hash", "extra_id"])
-def test_malformed_search_response_is_rejected_not_partial_success(store, monkeypatch, mutation):
-    store.upsert([record()])
-    get = store._collection.get
-
-    def malformed(**kwargs):
-        result = get(**kwargs)
-        if mutation == "metadata":
-            result["metadatas"][0][0]["generation_id"] = "wrong"
-        elif mutation == "dimension":
-            result["embeddings"] = [[1.0, 0.0]]
-        elif mutation == "hash":
-            result["metadatas"][0]["vector_sha256"] = "0" * 64
-        else:
-            result["ids"] = result["ids"] * 2
-            result["metadatas"] = result["metadatas"] * 2
-            result["embeddings"] = [result["embeddings"][0], result["embeddings"][0]]
-        return result
-
-    monkeypatch.setattr(store._collection, "get", malformed)
-    with pytest.raises(VectorStoreError):
-        store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-
-
-@pytest.mark.parametrize(
-    "ids,limit,deadline",
-    [
-        (["chunk-a", "chunk-a"], 1, 100.0),
-        ("chunk-a", 1, 100.0),
-        (["chunk-a"], 0, 100.0),
-        (["chunk-a"], True, 100.0),
-        (["chunk-a"], 1, math.inf),
-        (["chunk-a"], 1, math.nan),
-    ],
-)
-def test_invalid_query_parameters_do_not_reach_chroma(store, monkeypatch, ids, limit, deadline):
-    monkeypatch.setattr(store._collection, "query", lambda **kwargs: pytest.fail("invalid query"))
-    with pytest.raises(VectorStoreError):
-        store.search((1.0, 0.0, 0.0), ids, limit, deadline)
-
-
-def test_query_backend_error_is_unavailable_not_complete_empty(store, monkeypatch):
-    def broken(**kwargs):
-        raise RuntimeError("backend request payload")
-
-    monkeypatch.setattr(store._collection, "query", broken)
-    with pytest.raises(VectorStoreError, match="^vector_query_unavailable$"):
-        store.search((1.0, 0.0, 0.0), ["chunk-a"], 1, monotonic() + 10)
-
-
-def test_last_owned_client_close_releases_chroma_system(tmp_path):
-    from chromadb.api.shared_system_client import SharedSystemClient
-
-    directory = tmp_path / "retrieval-vectors"
-    writer = RawVectorStore.create(directory, generation_id="close-test", dimensions=3)
-    reader = RawVectorStore.open_existing(directory, generation_id="close-test", dimensions=3)
-    identifier = writer._client._identifier
-    try:
-        assert identifier in SharedSystemClient._identifier_to_system
-        writer.close()
-        assert identifier in SharedSystemClient._identifier_to_system
-        reader.close()
-        assert identifier not in SharedSystemClient._identifier_to_system
-        reader.close()
-    finally:
-        reader.close()
-        writer.close()
-
-
-def test_open_existing_missing_collection_does_not_construct_a_client(store, monkeypatch):
-    import chromadb
-
-    monkeypatch.setattr(
-        chromadb, "PersistentClient", lambda *args, **kwargs: pytest.fail("must preflight first")
-    )
-    with pytest.raises(VectorStoreError, match="collection_not_found"):
-        RawVectorStore.open_existing(store.directory, generation_id="absent", dimensions=3)
-
-
-def test_existing_dimension_mismatch_does_not_modify_collection(store):
-    before = store._collection.metadata.copy()
-    with pytest.raises(VectorStoreError):
-        RawVectorStore.open_writer_existing(
-            store.directory, generation_id="generation-a", dimensions=4
-        )
-    assert store._collection.metadata == before
-    store.upsert([record()])
-    store.verify([record()])
-
-
-def test_verify_candidates_reader_checks_every_batch_without_writing(store, monkeypatch):
-    rows = [record(f"candidate-{i}") for i in range(2050)]
-    store.upsert(rows)
-    reader = RawVectorStore.open_existing(
-        store.directory, generation_id="generation-a", dimensions=3
-    )
-    try:
-        calls = []
-        get = reader._collection.get
-
-        def tracked(**kwargs):
-            calls.append(kwargs)
-            return get(**kwargs)
-
-        monkeypatch.setattr(reader._collection, "get", tracked)
-        for name in ("add", "upsert", "update", "delete", "query"):
-            monkeypatch.setattr(
-                reader._collection, name, lambda **kwargs: pytest.fail("unexpected backend call")
-            )
-        assert (
-            reader.verify_candidates(
-                [(row.chunk_id, row.build_id) for row in rows], deadline=monotonic() + 30
-            )
-            is True
-        )
-        assert [len(call["ids"]) for call in calls] == [2048, 2]
-        assert all(call["include"] == ["embeddings", "metadatas"] for call in calls)
-    finally:
-        reader.close()
-
-
-def test_verify_candidates_empty_and_expired_do_not_read(store, monkeypatch):
-    monkeypatch.setattr(store._collection, "get", lambda **kwargs: pytest.fail("unexpected read"))
-    store._clock = lambda: 10.0
-    assert store.verify_candidates([], deadline=0.0) is True
-    assert store.verify_candidates([("chunk-a", "build-a")], deadline=10.0) is False
-
-
-@pytest.mark.parametrize("expires", ["after_read", "between_batches"])
-def test_verify_candidates_deadline_stops_before_another_batch(store, monkeypatch, expires):
-    rows = [record(f"candidate-{i}") for i in range(2049)]
-    store.upsert(rows)
-    calls = []
-    get = store._collection.get
-
-    def tracked(**kwargs):
-        calls.append(kwargs)
-        return get(**kwargs)
-
-    ticks = iter([0.0, 2.0] if expires == "after_read" else [0.0, 0.0, 2.0])
-    store._clock = lambda: next(ticks)
-    monkeypatch.setattr(store._collection, "get", tracked)
-    assert (
-        store.verify_candidates([(row.chunk_id, row.build_id) for row in rows], deadline=1.0)
-        is False
-    )
-    assert len(calls) == 1 and len(calls[0]["ids"]) == 2048
-
-
-@pytest.mark.parametrize("pair", [("missing", "build-a"), ("chunk-a", "other-build")])
-def test_verify_candidates_missing_or_wrong_build_refuses(store, pair):
-    store.upsert([record()])
-    with pytest.raises(VectorStoreError, match="missing|identity_mismatch"):
-        store.verify_candidates([pair], deadline=monotonic() + 10)
-
-
-@pytest.mark.parametrize("mutation", ["generation", "dimension", "hash", "extra_id"])
-def test_verify_candidates_corrupt_backend_record_refuses(store, monkeypatch, mutation):
-    store.upsert([record()])
-    get = store._collection.get
-
-    def corrupt(**kwargs):
-        data = get(**kwargs)
-        if mutation == "generation":
-            data["metadatas"][0]["generation_id"] = "other-generation"
-        elif mutation == "dimension":
-            data["embeddings"] = [[1.0, 0.0]]
-        elif mutation == "hash":
-            data["metadatas"][0]["vector_sha256"] = "0" * 64
-        else:
-            data["ids"][0] = "not-eligible"
-        return data
-
-    monkeypatch.setattr(store._collection, "get", corrupt)
-    with pytest.raises(VectorStoreError):
-        store.verify_candidates([("chunk-a", "build-a")], deadline=monotonic() + 10)
-
-
-@pytest.mark.parametrize(
-    "candidates",
-    [
-        [("chunk-a", "build-a"), ("chunk-a", "build-a")],
-        [("chunk-a", "build-a"), ("chunk-a", "other")],
-        [("../escape", "build-a")],
-        [("chunk-a", "")],
-        [("chunk-a", 123)],
-        [("chunk-a",)],
-        ["ab"],
-        "ab",
-        None,
-    ],
-)
-def test_verify_candidates_invalid_input_refuses_before_read(store, monkeypatch, candidates):
-    monkeypatch.setattr(store._collection, "get", lambda **kwargs: pytest.fail("unexpected read"))
-    with pytest.raises(VectorStoreError):
-        store.verify_candidates(candidates, deadline=monotonic() + 10)
-
-
-@pytest.mark.parametrize("deadline", [math.nan, math.inf, True, "soon"])
-def test_verify_candidates_invalid_deadline_refuses_even_empty(store, deadline):
-    with pytest.raises(VectorStoreError, match="invalid_deadline"):
-        store.verify_candidates([], deadline=deadline)

memhub/apps/desktop/src/diagnostics-ipc.test.ts

@@ -94,12 +94,14 @@ it("returns only diagnostic events written during this desktop launch", async ()
   await mkdir(logs, { recursive: true });
   const target = path.join(logs, "diagnostic-core.jsonl");
   const old = {
+    schema_version: 1,
     timestamp: "2026-10-14T09:00:00.000Z",
     level: "info",
     service: "core",
     component: "core.startup",
     event: "core.old",
     outcome: "succeeded",
+    session_id: "11111111-1111-4111-8111-111111111111",
   };
   await writeFile(target, `${JSON.stringify(old)}\n`);
   const cursor = await captureDebugLogCursor(dataDir);
@@ -109,6 +111,11 @@ it("returns only diagnostic events written during this desktop launch", async ()
     timestamp: "2026-10-14T10:01:00.000Z",
     event: "core.current",
     level: "warn",
+    details: {
+      count: 1,
+      "custom.private": "/Users/private/query.txt",
+      credential_hint: "ghp_private_token",
+    },
   };
   await appendFile(target, `${JSON.stringify(current)}\nnot-json\n`);
 
@@ -118,8 +125,11 @@ it("returns only diagnostic events written during this desktop launch", async ()
   expect(snapshot.entries[0]).toMatchObject({
     event: "core.current",
     level: "warn",
+    details: { count: 1 },
     source: "diagnostic",
   });
+  expect(JSON.stringify(snapshot)).not.toContain("/Users/private");
+  expect(JSON.stringify(snapshot)).not.toContain("ghp_private_token");
 });
 
 it("tracks diagnostic JSONL across rotation without rereading the old prefix", async () => {
@@ -130,12 +140,14 @@ it("tracks diagnostic JSONL across rotation without rereading the old prefix", a
   const target = path.join(logs, "diagnostic-core.jsonl");
   const event = (name: string, timestamp: string) =>
     JSON.stringify({
+      schema_version: 1,
       timestamp,
       level: "info",
       service: "core",
       component: "core.test",
       event: name,
       outcome: "succeeded",
+      session_id: "22222222-2222-4222-8222-222222222222",
     });
   await writeFile(target, `${event("before", "2026-10-14T09:00:00.000Z")}\n`);
   const cursor = await captureDebugLogCursor(dataDir);

memhub/apps/desktop/src/diagnostics-ipc.ts

@@ -1,10 +1,13 @@
 import { lstat, open, readdir } from "node:fs/promises";
 import path from "node:path";
 import {
+  type DiagnosticEventV1,
   type DiagnosticLogger,
+  diagnosticEvent,
   diagnosticZipFilename,
   exportDiagnosticZip,
   inspectBackgroundHealth,
+  projectDiagnosticEventForSafeOutput,
   recordDiagnosticMarker,
 } from "@one-memory/runtime-control";
 import type { BrowserWindow, IpcMainInvokeEvent } from "electron";
@@ -370,42 +373,58 @@ function debugLogEntry(
   const timestamp = typeof value.timestamp === "string" ? value.timestamp : "";
   const timestampMs = Date.parse(timestamp);
   if (!Number.isFinite(timestampMs) || timestampMs < startedAt) return null;
-  const level = value.level;
-  const service = value.service;
-  const component = value.component;
-  const event = value.event;
-  const outcome = value.outcome;
-  if (
-    !["info", "warn", "error"].includes(String(level)) ||
-    !["runtime", "core", "host-bridge", "desktop", "cli"].includes(
-      String(service),
-    ) ||
-    typeof component !== "string" ||
-    typeof event !== "string" ||
-    typeof outcome !== "string"
-  ) {
+  let projected: DiagnosticEventV1;
+  try {
+    projected = projectDiagnosticEventForSafeOutput(
+      diagnosticEvent({
+        timestamp,
+        level: value.level as never,
+        service: value.service as never,
+        component: value.component as never,
+        event: value.event as never,
+        outcome: value.outcome as never,
+        session_id: value.session_id as never,
+        ...(typeof value.trace_id === "string"
+          ? { trace_id: value.trace_id }
+          : {}),
+        ...(typeof value.duration_ms === "number"
+          ? { duration_ms: value.duration_ms }
+          : {}),
+        ...(typeof value.error_code === "string"
+          ? { error_code: value.error_code }
+          : {}),
+        ...(typeof value.error_type === "string"
+          ? { error_type: value.error_type }
+          : {}),
+        ...(value.details && typeof value.details === "object"
+          ? { details: value.details as never }
+          : {}),
+      }),
+    );
+  } catch {
     return null;
   }
-  const details = safeDebugDetails(value.details);
   return {
     id: `${filename}:${lineNumber}:${timestamp}`,
-    timestamp,
-    level: level as DebugLogEntry["level"],
-    service: service as DebugLogEntry["service"],
-    component,
-    event,
-    outcome,
-    ...(typeof value.duration_ms === "number"
-      ? { duration_ms: value.duration_ms }
-      : {}),
-    ...(typeof value.error_code === "string"
-      ? { error_code: value.error_code }
-      : {}),
-    ...(typeof value.error_type === "string"
-      ? { error_type: value.error_type }
-      : {}),
-    ...(typeof value.trace_id === "string" ? { trace_id: value.trace_id } : {}),
-    ...(details === undefined ? {} : { details }),
+    timestamp: projected.timestamp,
+    level: projected.level,
+    service: projected.service,
+    component: projected.component,
+    event: projected.event,
+    outcome: projected.outcome,
+    ...(projected.duration_ms === undefined
+      ? {}
+      : { duration_ms: projected.duration_ms }),
+    ...(projected.error_code === undefined
+      ? {}
+      : { error_code: projected.error_code }),
+    ...(projected.error_type === undefined
+      ? {}
+      : { error_type: projected.error_type }),
+    ...(projected.trace_id === undefined
+      ? {}
+      : { trace_id: projected.trace_id }),
+    ...(projected.details === undefined ? {} : { details: projected.details }),
     source: "diagnostic",
   };
 }
@@ -523,25 +542,6 @@ function logLineTimestamp(
   return new Date(Math.max(startedAt, fallback)).toISOString();
 }
 
-function safeDebugDetails(
-  value: unknown,
-): Record<string, string | number | boolean | null> | undefined {
-  if (!value || typeof value !== "object" || Array.isArray(value))
-    return undefined;
-  const details: Record<string, string | number | boolean | null> = {};
-  for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
-    if (
-      item === null ||
-      typeof item === "string" ||
-      typeof item === "number" ||
-      typeof item === "boolean"
-    ) {
-      details[key] = item;
-    }
-  }
-  return Object.keys(details).length ? details : undefined;
-}
-
 async function pipelineLogFiles(root: string): Promise<PipelineLogCursor[]> {
   const rootInfo = await lstat(root).catch(() => null);
   if (!rootInfo?.isDirectory() || rootInfo.isSymbolicLink()) return [];

memhub/apps/desktop/src/main.ts

@@ -112,8 +112,11 @@ import {
   installBundledCliIfNeeded,
   installServiceWithDisabledFallback,
   isInstalledApplicationsApp,
+  requireCurrentRuntimeAtLeast,
   stableRuntimeErrorCode,
+  stableRuntimeFailureDetail,
   startStableRuntimeRepair,
+  startupFailureCause,
 } from "./stable-runtime";
 import { STARTUP_SCREEN_URL, startupScreenHtml } from "./startup-screen";
 import {
@@ -1092,13 +1095,13 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
       component: "desktop.sdk_migration",
       event: "sdk_migration.startup",
       outcome:
-        migration.state === "cutover"
+        migration.state === "cutover" || migration.state === "complete"
           ? "succeeded"
           : migration.state === "deferred"
             ? "blocked"
             : "skipped",
       ...(migration.code === undefined ? {} : { error_code: migration.code }),
-      details: { state: migration.state },
+      details: { runtime_state: migration.state },
     });
     requireCompletedStartupSdkMigration(migration);
   };
@@ -1157,30 +1160,52 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
       const previousOwner = owner;
       let runtimeChanged = false;
       let transientRuntime = false;
-      if (owner?.owner !== "cli") {
-        const installed = await installPackagedRuntime({
-          resourcesPath: process.resourcesPath,
-          executablePath: process.execPath,
-          installRoot,
-          platform: "darwin",
-          arch: process.arch,
-          ...(packagedBuildNumber === undefined
-            ? {}
-            : { buildNumber: packagedBuildNumber }),
-          ...(packagedCommit === undefined
-            ? {}
-            : { gitCommit: packagedCommit }),
-          // Only the packaged Desktop path opts into a valid same-build,
-          // different-commit takeover. CLI installs remain strict.
-          replaceSameBuild: true,
-          serviceEnableMarker: runtimeServiceEnableMarkerPath(dataDir),
-        });
-        runtimeChanged = !installed.reused_current;
-        await installBundledCliIfNeeded({
-          home: app.getPath("home"),
-          installRoot,
-          platform: "darwin",
-        });
+      // Install whoever owns the service. Owner decides who starts and stops
+      // it, not which build runs: skipping the install for a CLI owner (also
+      // the App's own guess when its receipt is missing) left an upgraded App
+      // on the previous Runtime and, with `current` gone, on none at all.
+      {
+        let installed: Awaited<
+          ReturnType<typeof installPackagedRuntime>
+        > | null = null;
+        try {
+          installed = await installPackagedRuntime({
+            resourcesPath: process.resourcesPath,
+            executablePath: process.execPath,
+            installRoot,
+            platform: "darwin",
+            arch: process.arch,
+            ...(packagedBuildNumber === undefined
+              ? {}
+              : { buildNumber: packagedBuildNumber }),
+            ...(packagedCommit === undefined
+              ? {}
+              : { gitCommit: packagedCommit }),
+            // Only the packaged Desktop path opts into a valid same-build,
+            // different-commit takeover. CLI installs remain strict.
+            replaceSameBuild: true,
+            serviceEnableMarker: runtimeServiceEnableMarkerPath(dataDir),
+          });
+        } catch (error) {
+          // `memhub upgrade` may already have installed a newer build.
+          if (
+            !(
+              owner?.owner === "cli" &&
+              error instanceof Error &&
+              error.message === "distribution_downgrade_refused"
+            )
+          )
+            throw error;
+        }
+        if (installed !== null) {
+          runtimeChanged = !installed.reused_current;
+          await installBundledCliIfNeeded({
+            home: app.getPath("home"),
+            installRoot,
+            platform: "darwin",
+          });
+        }
+        await requireCurrentRuntimeAtLeast(installRoot, packagedBuildNumber);
       }
       if (
         !desktopBackgroundLaunch &&
@@ -1484,6 +1509,9 @@ async function initialize(isQuitting: () => boolean): Promise<void> {
       outcome: "failed",
       error_code: stableRuntimeErrorCode(error),
       ...diagnosticError(error),
+      // The code alone is often `desktop_start_failed`; the cause is what
+      // support needs. Home is shortened and the text bounded.
+      details: { cause: startupFailureCause(error, app.getPath("home")) },
     });
     throw error;
   }
@@ -1671,7 +1699,7 @@ if (desktopVersion !== null) {
           languagePreferences?.read().locale ?? "zh-CN",
           "nativeErrorTitle",
         ),
-        error instanceof Error ? error.message : "desktop_unavailable",
+        stableRuntimeFailureDetail(error),
       );
     },
   });

memhub/apps/desktop/src/stable-runtime.test.ts

@@ -15,9 +15,12 @@ import {
   installServiceWithDisabledFallback,
   isDisabledServiceInstallError,
   isInstalledApplicationsApp,
+  requireCurrentRuntimeAtLeast,
   stableRuntimeErrorCode,
+  stableRuntimeFailureDetail,
   stableRuntimeUserMessage,
   startStableRuntimeRepair,
+  startupFailureCause,
 } from "./stable-runtime";
 
 it("only treats a packaged macOS executable under Applications as installed", () => {
@@ -274,4 +277,80 @@ describe("startup install errors", () => {
       "MemHub 启动失败",
     );
   });
+
+  it("says why a data upgrade failed, and what the person can do", () => {
+    const noSpace = stableRuntimeUserMessage(
+      new Error("sdk_migration_disk_space_insufficient"),
+    );
+    // The only failure here the person fixes themselves: name the action.
+    expect(noSpace).toContain("磁盘空间不足");
+    expect(noSpace).toContain("重新启动");
+    expect(noSpace).toContain("不会丢失");
+    expect(noSpace).not.toContain("sdk_migration_disk_space_insufficient");
+
+    const unreadable = stableRuntimeUserMessage(
+      new Error("sdk_migration_memory_unreadable"),
+    );
+    expect(unreadable).toContain("没有切换数据源");
+    expect(unreadable).toContain("不会丢失");
+  });
+});
+
+describe("startup dialog body", () => {
+  it("keeps a code we have no copy for, so support can see which failure fired", () => {
+    expect(stableRuntimeFailureDetail(new Error("page_load_failed"))).toBe(
+      "page_load_failed",
+    );
+    expect(
+      stableRuntimeFailureDetail(new Error("runtime_broker_unavailable")),
+    ).toBe("runtime_broker_unavailable");
+  });
+
+  it("shows the sentence we wrote for a migration failure", () => {
+    const detail = stableRuntimeFailureDetail(
+      new Error("sdk_migration_disk_space_insufficient"),
+    );
+    expect(detail).toContain("磁盘空间不足");
+    expect(detail).not.toContain("sdk_migration_disk_space_insufficient");
+  });
+});
+
+describe("packaged runtime build after an App upgrade", () => {
+  it("refuses to run on a Runtime older than the App, or none", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "memhub-current-"));
+    await expect(requireCurrentRuntimeAtLeast(root, 21)).rejects.toThrow(
+      "packaged_runtime_not_current",
+    );
+    await mkdir(path.join(root, "current"));
+    const manifest = path.join(root, "current", "manifest.json");
+    await writeFile(manifest, JSON.stringify({ build_number: 19 }));
+    await expect(requireCurrentRuntimeAtLeast(root, 21)).rejects.toThrow(
+      "packaged_runtime_not_current",
+    );
+    await writeFile(manifest, JSON.stringify({ build_number: 21 }));
+    await expect(
+      requireCurrentRuntimeAtLeast(root, 21),
+    ).resolves.toBeUndefined();
+    await writeFile(manifest, JSON.stringify({ build_number: 22 }));
+    await expect(
+      requireCurrentRuntimeAtLeast(root, 21),
+    ).resolves.toBeUndefined();
+    await expect(
+      requireCurrentRuntimeAtLeast(root, undefined),
+    ).resolves.toBeUndefined();
+    expect(
+      stableRuntimeUserMessage(new Error("packaged_runtime_not_current")),
+    ).toContain("重新安装");
+  });
+
+  it("keeps the cause of a startup failure, without the home path", () => {
+    const cause = startupFailureCause(
+      new Error("ENOENT: no such file /Users/someone/Library/x\nmore"),
+      "/Users/someone",
+    );
+    expect(cause).toBe("Error: ENOENT: no such file ~/Library/x more");
+    expect(startupFailureCause(new Error("x".repeat(500)), "/h").length).toBe(
+      300,
+    );
+  });
 });

memhub/apps/desktop/src/stable-runtime.ts

@@ -126,6 +126,21 @@ const SAFE_STABLE_RUNTIME_ERROR_CODES = new Set([
   "installed_version_conflict",
   "distribution_downgrade_refused",
   "install_current_unmanaged",
+  // Migration runs before the app window exists, so these are the only names
+  // a first launch can be given. They stay codes here and become sentences in
+  // STABLE_RUNTIME_USER_MESSAGES: a person reading the dialog never sees them.
+  "packaged_runtime_not_current",
+  "sdk_migration_preflight_failed",
+  "sdk_migration_memory_unreadable",
+  "sdk_migration_model_profile_required",
+  "sdk_migration_model_secret_unavailable",
+  "sdk_migration_secure_storage_unavailable",
+  "sdk_migration_disk_space_insufficient",
+  "sdk_migration_cutover_failed",
+  "sdk_migration_bundle_copy_failed",
+  "sdk_migration_response_invalid",
+  "sdk_migration_status_unavailable",
+  "sdk_migration_incomplete",
 ]);
 
 export function stableRuntimeErrorCode(error: unknown): string {
@@ -191,6 +206,30 @@ export const STABLE_RUNTIME_USER_MESSAGES: Record<string, string> = {
     "已安装的运行环境比这个安装包更新，无法用旧安装包启动。请安装最新版本的 MemHub；你的记忆数据不会受到影响。",
   install_current_unmanaged:
     "运行环境的安装目录被改动过，无法安全接管。请重新安装 MemHub，你的记忆数据不会受到影响。",
+  packaged_runtime_not_current:
+    "新版本的运行环境没有装上，MemHub 不会用旧版本继续运行。请重新安装 MemHub；如果仍然失败，请把这个提示反馈给我们。你的记忆数据不会受到影响。",
+  sdk_migration_memory_unreadable:
+    "升级前核对发现，新版本还读不到你的全部记忆，所以没有切换数据源，旧数据保持原样可用。请把这个提示反馈给我们；原有记忆数据不会丢失。",
+  sdk_migration_model_profile_required:
+    "升级记忆数据格式需要先有可用的模型配置，用来核对新版本能否读回你的记忆。请在旧版本中完成模型配置后重启；你的记忆数据不会受到影响。",
+  sdk_migration_model_secret_unavailable:
+    "模型密钥读不到（可能被系统钥匙串拒绝），无法核对新版本能否读回你的记忆。请允许 MemHub 访问钥匙串后重新启动；你的记忆数据不会受到影响。",
+  sdk_migration_secure_storage_unavailable:
+    "系统钥匙串暂时不可用，无法核对新版本能否读回你的记忆。请解锁钥匙串后重新启动；你的记忆数据不会受到影响。",
+  sdk_migration_disk_space_insufficient:
+    "磁盘空间不足，升级记忆数据格式时中断了。请至少腾出与记忆数据同等大小的可用空间（可在「关于本机 → 储存空间」查看），然后重新启动 MemHub；升级会自动继续，原有记忆数据不会丢失。",
+  sdk_migration_bundle_copy_failed:
+    "升级前备份没有写完，通常是磁盘空间不足。请清理磁盘空间后重新启动；原有记忆数据不会丢失。",
+  sdk_migration_cutover_failed:
+    "切换数据源时失败，原有记忆数据仍然完整可用。请重新启动重试；如果仍然失败，请把这个提示反馈给我们。",
+  sdk_migration_preflight_failed:
+    "升级前的检查没有通过，升级已停止。请重新启动重试；如果仍然失败，请把这个提示反馈给我们，原有记忆数据不会丢失。",
+  sdk_migration_status_unavailable:
+    "读不到升级状态，无法判断是否需要升级。请重新启动 MemHub；如果仍然失败，请重新安装，你的记忆数据不会受到影响。",
+  sdk_migration_response_invalid:
+    "升级过程返回了无法识别的内容，已停止。原有记忆数据不会丢失；请重新启动重试，如果仍然失败，请把这个提示反馈给我们。",
+  sdk_migration_incomplete:
+    "记忆数据格式升级没有完成，MemHub 暂时无法打开。请重新启动重试；如果仍然失败，请把这个提示反馈给我们，原有记忆数据不会丢失。",
   desktop_start_failed:
     "MemHub 启动失败。请重新启动；如果仍然打不开，请重新安装，你的记忆数据不会受到影响。",
 };
@@ -203,3 +242,44 @@ export function stableRuntimeUserMessage(error: unknown): string {
     ""
   );
 }
+
+/** What the startup dialog body says.
+ *
+ * A code we wrote copy for becomes a sentence. Every other code is shown as it
+ * is: this dialog is the last thing a person sees before the app closes, and an
+ * unexpected failure is more useful as the code support can look up than as a
+ * reassuring sentence that erases which failure actually fired. */
+export function stableRuntimeFailureDetail(error: unknown): string {
+  if (!(error instanceof Error)) {
+    return STABLE_RUNTIME_USER_MESSAGES.desktop_start_failed ?? "";
+  }
+  return STABLE_RUNTIME_USER_MESSAGES[error.message] ?? error.message;
+}
+
+/** The installed Runtime must be at least the build this App carries. */
+export async function requireCurrentRuntimeAtLeast(
+  installRoot: string,
+  buildNumber: number | undefined,
+): Promise<void> {
+  if (buildNumber === undefined) return;
+  const manifest = await readFile(
+    path.join(installRoot, "current", "manifest.json"),
+    "utf8",
+  )
+    .then((text) => JSON.parse(text) as { build_number?: unknown })
+    .catch(() => null);
+  const installed = manifest?.build_number;
+  if (typeof installed !== "number" || installed < buildNumber) {
+    throw new Error("packaged_runtime_not_current");
+  }
+}
+
+export function startupFailureCause(error: unknown, home: string): string {
+  const text =
+    error instanceof Error ? `${error.name}: ${error.message}` : String(error);
+  return text
+    .split(home)
+    .join("~")
+    .replace(/[\r\n]+/g, " ")
+    .slice(0, 300);
+}

memhub/apps/desktop/src/startup-screen.test.ts

@@ -21,6 +21,21 @@ describe("startup screen", () => {
     );
     expect(spanish).not.toMatch(/[一-鿿]/);
   });
+  it("says an upgrade may take a while instead of leaving a silent wait", () => {
+    expect(startupScreenHtml()).toContain(
+      "首次启动或升级记忆数据可能需要几分钟，完成后会自动进入应用。",
+    );
+    expect(startupScreenHtml("en")).toContain(
+      "The first launch or a memory-data upgrade can take a few minutes.",
+    );
+  });
+  it("keeps a draggable strip so the waiting window can be moved", () => {
+    const html = startupScreenHtml();
+    // CSS alone does not move a frameless window: the region has to be a real
+    // element in the document, and it has to be the top strip.
+    expect(html).toContain('<div class="window-drag-region"');
+    expect(html).toContain("-webkit-app-region: drag");
+  });
   it("is served from the app origin, not from the web bundle", () => {
     expect(STARTUP_SCREEN_URL).toBe("one-memory://app/__startup");
   });

memhub/apps/desktop/src/startup-screen.ts

@@ -37,6 +37,19 @@ export function startupScreenHtml(locale: GuiLocale = "zh-CN"): string {
           "Helvetica Neue", Arial, sans-serif;
         -webkit-font-smoothing: antialiased;
       }
+      /* A frameless window still needs somewhere to grab; without this the
+         whole wait is undraggable. */
+      .window-drag-region {
+        position: absolute;
+        z-index: 1;
+        top: 0;
+        right: 0;
+        left: 0;
+        height: 58px;
+        -webkit-app-region: drag;
+        -webkit-user-select: none;
+        user-select: none;
+      }
       @media (prefers-color-scheme: dark) {
         body {
           background: #15181d;
@@ -61,6 +74,13 @@ export function startupScreenHtml(locale: GuiLocale = "zh-CN"): string {
         font-size: 13px;
         opacity: 0.62;
       }
+      p.hint {
+        max-width: 320px;
+        font-size: 12px;
+        line-height: 1.55;
+        text-align: center;
+        opacity: 0.45;
+      }
       .bar {
         position: relative;
         width: 168px;
@@ -101,9 +121,11 @@ export function startupScreenHtml(locale: GuiLocale = "zh-CN"): string {
     </style>
   </head>
   <body>
+    <div class="window-drag-region" aria-hidden="true"></div>
     <main>
       <h1>MemHub</h1>
       <p>${translate(locale, "startupPreparing")}</p>
+      <p class="hint">${translate(locale, "startupUpgradeHint")}</p>
       <div class="bar"><span></span></div>
     </main>
   </body>

memhub/apps/desktop/src/startup-sdk-migration.test.ts

@@ -1,155 +1,17 @@
-import { expect, it, vi } from "vitest";
-import {
-  requireCompletedStartupSdkMigration,
-  runStartupSdkMigration,
-} from "./startup-sdk-migration";
+import { expect, it } from "vitest";
+import { requireCompletedStartupSdkMigration } from "./startup-sdk-migration";
 
-const core = { command: "/core/one-memory", args: [] };
-
-it("runs bundle backfill and cutover inside the startup maintenance fence", async () => {
-  const calls: string[][] = [];
-  const execute = vi.fn(async (_core, args: string[]) => {
-    calls.push(args);
-    const command = args[0];
-    return {
-      exitCode: 0,
-      document:
-        command === "sdk-migration-status"
-          ? {
-              authority: "legacy",
-              state: "required",
-              generation: "0123456789abcdef",
-            }
-          : command === "sdk-migrate"
-            ? { state: "ready" }
-            : command === "sdk-cutover"
-              ? { state: "cutover" }
-              : { state: "created" },
-    };
-  });
-  const fenceCalls: string[] = [];
-  const fence = async <T>(
-    dataDir: string,
-    action: () => Promise<T>,
-  ): Promise<T> => {
-    fenceCalls.push(dataDir);
-    return action();
-  };
-
-  const result = await runStartupSdkMigration({
-    dataDir: "/data",
-    core,
-    runtimeBuild: "build-13",
-    execute,
-    fence,
-  });
-
-  expect(result).toEqual({ state: "cutover" });
-  expect(fenceCalls).toEqual(["/data"]);
-  expect(calls.map((args) => args[0])).toEqual([
-    "sdk-migration-status",
-    "migration-bundle",
-    "sdk-migrate",
-    "sdk-cutover",
-  ]);
-});
-
-it("does not stop Runtime when authority is already SDK or the home is empty", async () => {
-  for (const document of [
-    { authority: "sdk", state: "complete" },
-    { authority: "legacy", state: "empty" },
-  ]) {
-    let fenced = false;
-    const fence = async <T>(
-      _dataDir: string,
-      action: () => Promise<T>,
-    ): Promise<T> => {
-      fenced = true;
-      return action();
-    };
-    const result = await runStartupSdkMigration({
-      dataDir: "/data",
-      core,
-      runtimeBuild: "build-13",
-      execute: vi.fn(async () => ({ exitCode: 0, document })),
-      fence,
-    });
-    expect(result).toEqual({ state: "not_required" });
-    expect(fenced).toBe(false);
-  }
-});
-
-it("reports an unreadable migration status as deferred", async () => {
-  const result = await runStartupSdkMigration({
-    dataDir: "/data",
-    core,
-    runtimeBuild: "build-13",
-    execute: vi.fn(async () => ({ exitCode: 1, document: {} })),
-    fence: async (_dataDir, action) => action(),
-  });
-
-  expect(result).toEqual({
-    state: "deferred",
-    code: "sdk_migration_status_unavailable",
-  });
-  expect(() => requireCompletedStartupSdkMigration(result)).toThrow(
-    "sdk_migration_status_unavailable",
-  );
-});
-
-it("resumes legacy table cleanup under the maintenance fence", async () => {
-  const calls: string[] = [];
-  const result = await runStartupSdkMigration({
-    dataDir: "/data",
-    core,
-    runtimeBuild: "build-13",
-    execute: vi.fn(async (_core, args: string[]) => {
-      calls.push(args[0]);
-      return args[0] === "sdk-migration-status"
-        ? {
-            exitCode: 0,
-            document: {
-              authority: "sdk",
-              state: "cleanup_required",
-            },
-          }
-        : { exitCode: 0, document: { state: "clean" } };
+it("refuses to open the app on a deferred migration, naming its code", () => {
+  expect(() =>
+    requireCompletedStartupSdkMigration({
+      state: "deferred",
+      code: "sdk_migration_disk_space_insufficient",
     }),
-    fence: async (_dataDir, action) => action(),
-  });
-
-  expect(result).toEqual({ state: "cutover" });
-  expect(calls).toEqual(["sdk-migration-status", "sdk-cleanup"]);
-});
-
-it("keeps legacy authority when backfill is blocked", async () => {
-  const execute = vi.fn(async (_core, args: string[]) => {
-    const command = args[0];
-    return {
-      exitCode: command === "sdk-migrate" ? 4 : 0,
-      document:
-        command === "sdk-migration-status"
-          ? {
-              authority: "legacy",
-              state: "required",
-              generation: "0123456789abcdef",
-            }
-          : command === "sdk-migrate"
-            ? { state: "blocked", code: "quarantine" }
-            : { state: "created" },
-    };
-  });
-
-  const result = await runStartupSdkMigration({
-    dataDir: "/data",
-    core,
-    runtimeBuild: "build-13",
-    execute,
-    fence: async (_dataDir, action) => action(),
-  });
-
-  expect(result).toEqual({ state: "deferred", code: "quarantine" });
-  expect(execute.mock.calls.map((call) => call[1][0])).not.toContain(
-    "sdk-cutover",
-  );
+  ).toThrow("sdk_migration_disk_space_insufficient");
+  expect(() =>
+    requireCompletedStartupSdkMigration({ state: "deferred" }),
+  ).toThrow("sdk_migration_incomplete");
+  expect(() =>
+    requireCompletedStartupSdkMigration({ state: "cutover" }),
+  ).not.toThrow();
 });

memhub/apps/desktop/src/startup-sdk-migration.ts

@@ -1,13 +1,10 @@
-import { spawn } from "node:child_process";
-import { withRuntimeMaintenanceFence } from "@one-memory/runtime-control";
+import {
+  runSdkMigration,
+  type SdkMigrationResult,
+} from "@one-memory/runtime-control";
 
-const MIGRATION_PREFIX = "sdk-authority-v1";
-const MAX_OUTPUT_BYTES = 1024 * 1024;
-
-export interface StartupMigrationResult {
-  state: "not_required" | "cutover" | "deferred";
-  code?: string;
-}
+export type StartupMigrationResult = SdkMigrationResult;
+export const runStartupSdkMigration = runSdkMigration;
 
 export function requireCompletedStartupSdkMigration(
   result: StartupMigrationResult,
@@ -16,179 +13,3 @@ export function requireCompletedStartupSdkMigration(
     throw new Error(result.code ?? "sdk_migration_incomplete");
   }
 }
-
-interface CommandResult {
-  exitCode: number;
-  document: Record<string, unknown>;
-}
-
-interface CoreCommand {
-  command: string;
-  args?: string[];
-  cwd?: string;
-  env?: NodeJS.ProcessEnv;
-}
-
-interface StartupMigrationOptions {
-  dataDir: string;
-  core: CoreCommand;
-  runtimeBuild: string;
-  execute?: (
-    launch: CoreCommand,
-    args: string[],
-    dataDir: string,
-  ) => Promise<CommandResult>;
-  fence?: <T>(dataDir: string, action: () => Promise<T>) => Promise<T>;
-}
-
-export async function runStartupSdkMigration(
-  options: StartupMigrationOptions,
-): Promise<StartupMigrationResult> {
-  const execute = options.execute ?? executeCoreCommand;
-  const fence = options.fence ?? withRuntimeMaintenanceFence;
-  const status = await execute(
-    options.core,
-    ["sdk-migration-status", "--json"],
-    options.dataDir,
-  );
-  if (status.exitCode !== 0) {
-    return { state: "deferred", code: "sdk_migration_status_unavailable" };
-  }
-  if (
-    status.document.authority === "sdk" &&
-    status.document.state === "cleanup_required"
-  ) {
-    return fence(options.dataDir, async () => {
-      const cleanup = await execute(
-        options.core,
-        ["sdk-cleanup", "--yes", "--json"],
-        options.dataDir,
-      );
-      return cleanup.exitCode === 0 && cleanup.document.state === "clean"
-        ? { state: "cutover" }
-        : deferred(cleanup);
-    });
-  }
-  if (
-    status.document.authority === "sdk" ||
-    status.document.state !== "required"
-  ) {
-    return { state: "not_required" };
-  }
-  const generation = status.document.generation;
-  if (typeof generation !== "string" || !/^[a-f0-9]{16}$/.test(generation)) {
-    return { state: "deferred", code: "sdk_migration_generation_invalid" };
-  }
-  const migrationId = `${MIGRATION_PREFIX}-${generation}`;
-  const bundleId = `${migrationId}-pre-cutover`;
-  return fence(options.dataDir, async () => {
-    const bundle = await execute(
-      options.core,
-      [
-        "migration-bundle",
-        "--bundle-id",
-        bundleId,
-        "--runtime-build",
-        options.runtimeBuild,
-        "--yes",
-        "--json",
-      ],
-      options.dataDir,
-    );
-    if (bundle.exitCode !== 0) return deferred(bundle);
-    const backfill = await execute(
-      options.core,
-      [
-        "sdk-migrate",
-        "--migration-id",
-        migrationId,
-        "--bundle-id",
-        bundleId,
-        "--yes",
-        "--json",
-      ],
-      options.dataDir,
-    );
-    if (backfill.exitCode !== 0 || backfill.document.state !== "ready") {
-      return deferred(backfill);
-    }
-    const cutover = await execute(
-      options.core,
-      [
-        "sdk-cutover",
-        "--migration-id",
-        migrationId,
-        "--bundle-id",
-        bundleId,
-        "--yes",
-        "--json",
-      ],
-      options.dataDir,
-    );
-    if (cutover.exitCode !== 0 || cutover.document.state !== "cutover") {
-      return deferred(cutover);
-    }
-    return { state: "cutover" };
-  });
-}
-
-function deferred(result: CommandResult): StartupMigrationResult {
-  return {
-    state: "deferred",
-    code:
-      typeof result.document.code === "string"
-        ? result.document.code
-        : "sdk_migration_incomplete",
-  };
-}
-
-async function executeCoreCommand(
-  launch: CoreCommand,
-  args: string[],
-  dataDir: string,
-): Promise<CommandResult> {
-  return new Promise((resolve, reject) => {
-    const child = spawn(launch.command, [...(launch.args ?? []), ...args], {
-      cwd: launch.cwd,
-      env: { ...process.env, ...launch.env, ONE_MEMORY_DATA_DIR: dataDir },
-      stdio: ["ignore", "pipe", "pipe"],
-    });
-    let stdout = "";
-    let stderrBytes = 0;
-    child.stdout.setEncoding("utf8");
-    child.stdout.on("data", (chunk: string) => {
-      stdout += chunk;
-      if (Buffer.byteLength(stdout) > MAX_OUTPUT_BYTES) {
-        child.kill("SIGKILL");
-        reject(new Error("sdk_migration_output_too_large"));
-      }
-    });
-    child.stderr.on("data", (chunk: Buffer) => {
-      stderrBytes += chunk.length;
-      if (stderrBytes > MAX_OUTPUT_BYTES) {
-        child.kill("SIGKILL");
-        reject(new Error("sdk_migration_error_output_too_large"));
-      }
-    });
-    child.once("error", reject);
-    child.once("close", (code) => {
-      try {
-        const line = stdout.trim().split("\n").at(-1) ?? "{}";
-        const document = JSON.parse(line) as unknown;
-        if (
-          !document ||
-          typeof document !== "object" ||
-          Array.isArray(document)
-        ) {
-          throw new Error("sdk_migration_response_invalid");
-        }
-        resolve({
-          exitCode: code ?? 1,
-          document: document as Record<string, unknown>,
-        });
-      } catch {
-        reject(new Error("sdk_migration_response_invalid"));
-      }
-    });
-  });
-}

memhub/apps/web/src/MemorySpace.sdk-authority.test.tsx

@@ -0,0 +1,252 @@
+// @vitest-environment jsdom
+import type {
+  GovernedMemory,
+  MemoryGraphResponse,
+  SdkMemoryRecord,
+} from "@one-memory/contracts";
+import {
+  cleanup,
+  fireEvent,
+  render,
+  screen,
+  waitFor,
+} from "@testing-library/react";
+import { afterEach, expect, it, vi } from "vitest";
+import { MemorySpace } from "./MemorySpace";
+import { sdkMapMemory } from "./sdk-memory-map";
+
+const api = vi.hoisted(() => ({
+  authority: vi.fn(),
+  legacyLibrary: vi.fn(),
+  legacyGraph: vi.fn(),
+  sdkLibrary: vi.fn(),
+  sdkGraph: vi.fn(),
+}));
+vi.mock("./api", () => ({
+  getMemoryAuthority: api.authority,
+  getMemoryLibrary: api.legacyLibrary,
+  getMemories: async () => [],
+  getMemoryGraph: api.legacyGraph,
+  getSdkMemoryLibrary: api.sdkLibrary,
+  getSdkMemoryGraph: api.sdkGraph,
+  getMemoryProcessing: async () => null,
+  getModelProfiles: async () => [],
+  apiErrorCode: () => null,
+}));
+vi.mock("./GraphWidthPrompt", () => ({ GraphWidthPrompt: () => null }));
+vi.mock("./MemoryNoteComposer", () => ({ MemoryNoteComposer: () => null }));
+vi.mock("./MemoryMap", () => ({
+  MemoryMap: ({
+    hidden,
+    memories,
+    graph,
+    loaded,
+    onOpenMemory,
+  }: {
+    hidden: boolean;
+    memories: GovernedMemory[];
+    graph: MemoryGraphResponse | null;
+    loaded: boolean;
+    onOpenMemory(id: string): void;
+  }) => (
+    <div hidden={hidden} data-testid="map">
+      <span data-testid="map-memories">
+        {memories.map((m) => `${m.memory_id}:${m.status}`).join(",")}
+      </span>
+      <span data-testid="map-nodes">{graph?.nodes.length ?? "none"}</span>
+      <span data-testid="map-loaded">{String(loaded)}</span>
+      <button type="button" onClick={() => onOpenMemory("a")}>
+        open a
+      </button>
+    </div>
+  ),
+}));
+vi.mock("./SdkMemoryLibrary", () => ({
+  SdkMemoryLibrary: ({
+    focusedMemoryId,
+  }: {
+    focusedMemoryId: string | null;
+  }) => <div data-testid="sdk-library">{focusedMemoryId ?? ""}</div>,
+}));
+vi.mock("./MemoryRecallScope", () => ({
+  MemoryRecallScope: () => null,
+  memoryProvenanceLabel: () => "原话",
+}));
+
+const record = (memory_id: string): SdkMemoryRecord => ({
+  memory_id,
+  content: `memory ${memory_id}`,
+  layer: "l2_fact",
+  status: "active",
+  is_latest: true,
+  supersedes: [],
+  superseded_by: [],
+  tags: ["工作"],
+  owner: null,
+  source_raw_ids: [],
+  user_id: "local",
+  agent_id: "all-apps",
+  session_id: "s1",
+  source: "cursor",
+  memory_at: null,
+  gmt_created: 1_761_000_000,
+});
+
+afterEach(() => {
+  cleanup();
+  vi.clearAllMocks();
+});
+
+function sdkAuthority() {
+  api.authority.mockResolvedValue({ authority: "sdk", migration_id: "m" });
+  api.sdkLibrary.mockResolvedValue({
+    memories: [record("a"), record("b")],
+    total: 3018,
+    limit: 300,
+    offset: 0,
+  });
+  api.sdkGraph.mockResolvedValue({
+    total_memories: 3018,
+    projected_memories: 2,
+    nodes: [
+      { node_id: "memory:a", node_type: "memory", label: "a", memory_id: "a" },
+      { node_id: "memory:b", node_type: "memory", label: "b", memory_id: "b" },
+      { node_id: "source:cursor", node_type: "source", label: "Cursor" },
+    ],
+    edges: [],
+    hy_graph_available: true,
+  });
+}
+
+it("draws the map from SDK memories on SDK authority, without legacy reads", async () => {
+  sdkAuthority();
+
+  render(<MemorySpace simplified active />);
+
+  await waitFor(() =>
+    expect(screen.getByTestId("map-loaded").textContent).toBe("true"),
+  );
+  expect(screen.getByTestId("map").hidden).toBe(false);
+  expect(screen.getByTestId("map-memories").textContent).toBe(
+    "a:confirmed,b:confirmed",
+  );
+  expect(screen.getByTestId("map-nodes").textContent).toBe("3");
+  expect(await screen.findByText(/3,018/)).toBeTruthy();
+  expect(api.sdkLibrary).toHaveBeenCalledWith(
+    { statuses: ["active"], offset: 0, limit: 300 },
+    expect.any(AbortSignal),
+  );
+  expect(api.legacyLibrary).not.toHaveBeenCalled();
+  expect(api.legacyGraph).not.toHaveBeenCalled();
+});
+
+it("opens a memory from the SDK map in the SDK library", async () => {
+  sdkAuthority();
+
+  render(<MemorySpace simplified active />);
+  await waitFor(() =>
+    expect(screen.getByTestId("map-loaded").textContent).toBe("true"),
+  );
+  fireEvent.click(screen.getByText("open a"));
+
+  const library = await screen.findByTestId("sdk-library");
+  expect(library.textContent).toBe("a");
+  expect(library.closest("[hidden]")).toBeNull();
+  expect(screen.getByTestId("map").hidden).toBe(true);
+});
+
+it("keeps asking for the authority until Core answers", async () => {
+  vi.useFakeTimers({ shouldAdvanceTime: true });
+  try {
+    sdkAuthority();
+    api.authority
+      .mockRejectedValueOnce(new Error("core_not_ready"))
+      .mockRejectedValueOnce(new Error("core_not_ready"));
+
+    render(<MemorySpace simplified active />);
+    await vi.advanceTimersByTimeAsync(2_000);
+
+    await waitFor(() => expect(api.authority).toHaveBeenCalledTimes(3));
+    await waitFor(() =>
+      expect(screen.getByTestId("map-loaded").textContent).toBe("true"),
+    );
+  } finally {
+    vi.useRealTimers();
+  }
+});
+
+it("projects SDK records into the fields the map groups by", () => {
+  const coding = sdkMapMemory({ ...record("c"), layer: "coding_experience" });
+  expect(coding).toMatchObject({
+    memory_id: "c",
+    kind: "coding",
+    status: "confirmed",
+    tags: ["工作"],
+    deleted_at: null,
+  });
+  expect(sdkMapMemory({ ...record("s"), layer: "l6_schema" }).kind).toBe(
+    "schema",
+  );
+  expect(sdkMapMemory({ ...record("x"), status: "superseded" }).status).toBe(
+    "superseded",
+  );
+});
+
+const node = (
+  node_id: string,
+  node_type: MemoryGraphResponse["nodes"][number]["node_type"],
+  extra: Partial<MemoryGraphResponse["nodes"][number]> = {},
+): MemoryGraphResponse["nodes"][number] => ({
+  node_id,
+  node_type,
+  label: node_id,
+  memory_id: null,
+  source_id: null,
+  kind: null,
+  status: null,
+  confidence: null,
+  ...extra,
+});
+const provenance = (
+  from: string,
+  to: string,
+): MemoryGraphResponse["edges"][number] => ({
+  edge_id: `${from}->${to}`,
+  source_node_id: from,
+  target_node_id: to,
+  relation: "evidence_source",
+  weight: null,
+  origin: "provenance",
+});
+
+it("groups and draws SDK memories by source with the real map model", async () => {
+  const { mapGroups, projectMap } = await import("./memory-map-model");
+  const memories = [record("a"), record("b")].map(sdkMapMemory);
+  const graph: MemoryGraphResponse = {
+    total_memories: 2,
+    projected_memories: 2,
+    nodes: [
+      node("memory:a", "memory", { memory_id: "a" }),
+      node("memory:b", "memory", { memory_id: "b" }),
+      node("source:cursor", "source", { source_id: "cursor", label: "Cursor" }),
+      node("hy:t1", "topic"),
+    ],
+    edges: [
+      provenance("source:cursor", "memory:a"),
+      provenance("source:cursor", "memory:b"),
+    ],
+    hy_graph_available: true,
+  };
+
+  const groups = mapGroups(memories, graph, "source");
+  expect(groups.map((group) => [group.key, group.memoryIds])).toEqual([
+    ["source:cursor", ["a", "b"]],
+  ]);
+  const drawn = projectMap(graph, memories, "source", ["a", "b"]);
+  expect(
+    drawn.nodes
+      .filter((node) => node.node_type === "memory")
+      .map((n) => n.memory_id),
+  ).toEqual(["a", "b"]);
+  expect(drawn.edges.length).toBeGreaterThan(0);
+});

memhub/apps/web/src/MemorySpace.tsx

@@ -39,6 +39,8 @@ import {
   getMemoryGraph,
   getMemoryProcessing,
   getModelProfiles,
+  getSdkMemoryGraph,
+  getSdkMemoryLibrary,
   manualExtract,
   mutateMemory,
   retryUnknownMemoryProcessing,
@@ -56,6 +58,7 @@ import { subscribePoll } from "./poll-scheduler";
 import { R6Button } from "./R6Controls";
 import { SdkMemoryLibrary } from "./SdkMemoryLibrary";
 import { SdkRawHistory } from "./SdkRawHistory";
+import { sdkMapMemory } from "./sdk-memory-map";
 import { sourceNameOr } from "./source-label";
 import { traceUiAction } from "./ui-trace";
 import { useMemoryLibrary } from "./use-memory-library";
@@ -168,13 +171,23 @@ export function MemorySpace({
     if ((!active && !preload) || typeof getMemoryAuthority !== "function")
       return;
     let current = true;
-    void getMemoryAuthority()
-      .then((result) => {
-        if (current) setMemoryAuthority(result.authority);
-      })
-      .catch(() => {});
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    // The app opens before Core is ready, so the first ask usually fails.
+    // Without a retry the authority stayed unknown and nothing ever loaded.
+    const ask = (delay: number) => {
+      void getMemoryAuthority()
+        .then((result) => {
+          if (current) setMemoryAuthority(result.authority);
+        })
+        .catch(() => {
+          if (!current) return;
+          timer = setTimeout(() => ask(Math.min(delay * 2, 10_000)), delay);
+        });
+    };
+    ask(500);
     return () => {
       current = false;
+      if (timer !== undefined) clearTimeout(timer);
     };
   }, [active, preload]);
   useEffect(() => {
@@ -286,6 +299,49 @@ export function MemorySpace({
   const [sdkFocusedMemoryId, setSdkFocusedMemoryId] = useState<string | null>(
     null,
   );
+  // The map on SDK authority. Legacy reads (library page, governed graph)
+  // are refused there, so it draws SDK memories and the v2 graph instead.
+  const [sdkMap, setSdkMap] = useState<{
+    memories: GovernedMemory[];
+    graph: MemoryGraphResponse;
+  } | null>(null);
+  const [sdkMapFailed, setSdkMapFailed] = useState(false);
+  const sdkMapVisible =
+    memoryAuthority === "sdk" &&
+    simplified &&
+    active &&
+    displayedMode === "map";
+  useEffect(() => {
+    if (!sdkMapVisible) return;
+    let cancelled = false;
+    let controller: AbortController | null = null;
+    const read = async () => {
+      controller?.abort();
+      const request = new AbortController();
+      controller = request;
+      try {
+        const [page, graph] = await Promise.all([
+          getSdkMemoryLibrary(
+            { statuses: ["active"], offset: 0, limit: 300 },
+            request.signal,
+          ),
+          getSdkMemoryGraph(request.signal),
+        ]);
+        if (cancelled) return;
+        setSdkMap({ memories: page.memories.map(sdkMapMemory), graph });
+        setSdkMapFailed(false);
+      } catch {
+        if (!cancelled && !request.signal.aborted) setSdkMapFailed(true);
+      }
+    };
+    void read();
+    const timer = setInterval(() => void read(), 30_000);
+    return () => {
+      cancelled = true;
+      clearInterval(timer);
+      controller?.abort();
+    };
+  }, [sdkMapVisible]);
   const [draft, setDraft] = useState("");
   const [selection, setSelection] = useState<Set<string>>(new Set());
   const bulkInFlight = useRef(false);
@@ -1444,33 +1500,50 @@ export function MemorySpace({
             {simplified ? t("memory:记忆") : t("memory:记忆空间")}
           </h1>
           <span className="memory-loaded-count">
-            {simplified && !management
-              ? mapLibrary.failed
+            {memoryAuthority === "sdk" && simplified && !management
+              ? sdkMapFailed && !sdkMap
                 ? t("memory:记忆读取失败")
-                : mapLibrary.stale
-                  ? t("memory:正在导入历史会话，记忆将陆续出现")
-                  : mapLibrary.page
+                : sdkMap
+                  ? tn(
+                      "memory:已沉淀 {count} 条{filter} · 当前展示 {shown} 条",
+                      sdkMap.graph.total_memories,
+                      {
+                        filter: "",
+                        shown: new Intl.NumberFormat(locale).format(
+                          sdkMap.memories.length,
+                        ),
+                      },
+                    )
+                  : t("memory:正在读取…")
+              : memoryAuthority === "sdk"
+                ? null
+                : simplified && !management
+                  ? mapLibrary.failed
+                    ? t("memory:记忆读取失败")
+                    : mapLibrary.stale
+                      ? t("memory:正在导入历史会话，记忆将陆续出现")
+                      : mapLibrary.page
+                        ? tn(
+                            "memory:已沉淀 {count} 条{filter} · 当前展示 {shown} 条",
+                            mapLibrary.page.total,
+                            {
+                              filter: settledFilterLabel
+                                ? t("memory:（{filter}）", {
+                                    filter: settledFilterLabel,
+                                  })
+                                : "",
+                              shown: new Intl.NumberFormat(locale).format(
+                                mapLibrary.page.items.length,
+                              ),
+                            },
+                          )
+                        : t("memory:正在读取…")
+                  : loaded
                     ? tn(
-                        "memory:已沉淀 {count} 条{filter} · 当前展示 {shown} 条",
-                        mapLibrary.page.total,
-                        {
-                          filter: settledFilterLabel
-                            ? t("memory:（{filter}）", {
-                                filter: settledFilterLabel,
-                              })
-                            : "",
-                          shown: new Intl.NumberFormat(locale).format(
-                            mapLibrary.page.items.length,
-                          ),
-                        },
+                        "memory:已加载 {count} 条记忆",
+                        memories.filter((m) => m.status !== "deleted").length,
                       )
-                    : t("memory:正在读取…")
-              : loaded
-                ? tn(
-                    "memory:已加载 {count} 条记忆",
-                    memories.filter((m) => m.status !== "deleted").length,
-                  )
-                : t("memory:正在读取…")}
+                    : t("memory:正在读取…")}
           </span>
         </div>
         <MemoryNoteComposer
@@ -1814,7 +1887,7 @@ export function MemorySpace({
         focusDialogue={focusDialogue}
         onFocusDialogueConsumed={onFocusDialogueConsumed}
         libraryView={
-          simplified
+          simplified && memoryAuthority !== "sdk"
             ? {
                 query: mapQuery,
                 source: mapSource,
@@ -1838,19 +1911,39 @@ export function MemorySpace({
             .catch((reason) => setError(message(reason, t)));
         }}
         hidden={!active || displayedMode !== "map"}
-        memories={simplified ? (mapLibrary.page?.items ?? []) : memories}
-        graph={simplified && sourceSnapshotStale ? null : graph}
+        memories={
+          memoryAuthority === "sdk"
+            ? (sdkMap?.memories ?? [])
+            : simplified
+              ? (mapLibrary.page?.items ?? [])
+              : memories
+        }
+        graph={
+          memoryAuthority === "sdk"
+            ? (sdkMap?.graph ?? null)
+            : simplified && sourceSnapshotStale
+              ? null
+              : graph
+        }
         sources={sources}
         processing={processing}
         arrival={memoryArrival}
         onViewHistory={simplified ? onConnect : () => setMode("history")}
-        loaded={simplified ? mapLibrary.page !== null : loaded}
+        loaded={
+          memoryAuthority === "sdk"
+            ? sdkMap !== null
+            : simplified
+              ? mapLibrary.page !== null
+              : loaded
+        }
         detail={detail}
         detailLoad={detailLoad}
         overlayBlocked={
           !active ||
           deleteCount > 0 ||
-          (simplified && (!mapLibrary.page || sourceSnapshotStale))
+          (memoryAuthority === "sdk"
+            ? sdkMap === null
+            : simplified && (!mapLibrary.page || sourceSnapshotStale))
         }
         draft={draft}
         busy={busy || detailReadBlocked}
@@ -1858,6 +1951,14 @@ export function MemorySpace({
         streaming={streaming}
         onDraft={setDraft}
         onOpenMemory={(memoryId) => {
+          if (memoryAuthority === "sdk") {
+            // Governed detail is refused on SDK authority; the SDK library
+            // opens, edits and deletes the same memory through v2.
+            setSdkFocusedMemoryId(memoryId);
+            if (simplified) setManagementOpen(true);
+            else setMode("library");
+            return;
+          }
           const recalled = answerMemories.current.find(
             (memory) => memory.memory_id === memoryId,
           );

memhub/apps/web/src/api.ts

@@ -1162,6 +1162,17 @@ export async function getMemory(memoryId: string): Promise<MemoryDetail> {
  */
 let pendingMemoryGraph: Promise<MemoryGraphResponse> | null = null;
 
+/** The memory map on SDK authority: SDK memories plus the graph overlay. */
+export async function getSdkMemoryGraph(
+  signal?: AbortSignal,
+): Promise<MemoryGraphResponse> {
+  const response = await client.get<MemoryGraphResponse>(
+    "/api/v2/memories/graph",
+    { params: { limit: 300 }, signal },
+  );
+  return response.data;
+}
+
 export async function getMemoryGraphWith(
   transport: { get: typeof client.get } = client,
 ): Promise<MemoryGraphResponse> {

memhub/apps/web/src/sdk-memory-map.ts

@@ -0,0 +1,51 @@
+import type {
+  GovernedMemory,
+  MemoryKind,
+  SdkMemoryRecord,
+} from "@one-memory/contracts";
+
+function kindOf(layer: string): MemoryKind {
+  if (layer === "l6_schema") return "schema";
+  if (layer === "l7_intention") return "intention";
+  if (layer.startsWith("coding")) return "coding";
+  if (layer.startsWith("working")) return "working";
+  return "chat";
+}
+
+function stamp(seconds: number | null | undefined): string {
+  return typeof seconds === "number" && Number.isFinite(seconds)
+    ? new Date(seconds * 1_000).toISOString()
+    : "";
+}
+
+/** An SDK memory in the shape the memory map draws.
+ *
+ * The map groups and labels by id, content, kind, status and tags only. It
+ * never writes through this shape: opening a memory goes to the SDK library,
+ * so the governed-only fields are inert placeholders here.
+ */
+export function sdkMapMemory(record: SdkMemoryRecord): GovernedMemory {
+  const created = stamp(record.gmt_created ?? record.memory_at);
+  return {
+    memory_id: record.memory_id,
+    user_id: record.user_id,
+    kind: kindOf(record.layer),
+    status:
+      record.status === "active"
+        ? "confirmed"
+        : record.status === "superseded"
+          ? "superseded"
+          : "deleted",
+    content: record.content,
+    scopes: [],
+    evidence_ids: [],
+    origin: "extracted",
+    tags: [...record.tags],
+    supersedes: [...record.supersedes],
+    superseded_by: [...record.superseded_by],
+    revision: 1,
+    created_at: created,
+    updated_at: created,
+    deleted_at: null,
+  };
+}

memhub/docs/ARCHITECTURE.md

@@ -1,6 +1,6 @@
 # MemHub Architecture
 
-Updated: 2026-10-21.
+Updated: 2026-10-24.
 
 Hy Memory 是 SDK / 引擎；MemHub 是产品与发布名。`one-memory` 仍可能出现在
 兼容路径、模块名和服务标识中。
@@ -129,15 +129,15 @@ directly for low-latency Cypher projection and store-width administration.
 This exception does not extend to VDB, Coding, prompt, reconcile, or lifecycle.
 
 - Storage backends：仅 SDK config/env（Chroma/FAISS/Qdrant、Kuzu/Neo4j、cache/history sqlite|mysql、coding sqlite 等）。
-- Queue：pending `memory_write_jobs` 只表示尚未调用 SDK 的待处理请求，因此没有 SDK raw/memory ID。worker 单次调用 SDK `add()`；SDK 内部完成 raw 写入与 extraction/reconciliation，成功后 MemHub 仅将返回的 raw ID / memory IDs 记作 job outcome。旧用户启动迁移在 Runtime writers 停止期间直接操作 SDK 存储，不向公开 `HyMemoryClient` 增加迁移专用方法，也不拆分正常写路径。
+- Queue：pending `memory_write_jobs` 只表示尚未调用 SDK 的待处理请求，因此没有 SDK raw/memory ID。worker 单次调用 SDK `add()`；SDK 内部完成 raw 写入与 extraction/reconciliation，成功后 MemHub 仅将返回的 raw ID / memory IDs 记作 job outcome。
+- Raw retention：SDK L1 raw 只保留最近三天，由 Runtime 的 `prune_expired_raw` 执行；每个完整 turn 只保存真人 user 与 assistant，tool 不进入 raw。高层 Chat/Coding/Working memory 不受该窗口影响。
 - Recall：Host Bridge / Core 先做宿主安装凭据与 tool grant，再调 **SDK search/get**。当前产品没有逐 memory 的 A-harness→B-harness 可见性开关，默认允许跨 harness 搜索；未来若增加来源限制，直接透传 SDK `source` / `include_sources` / `exclude_sources`，不在 MemHub 建 memory ACL。
 - Context wire：cutover marker 提交前只使用 legacy Context v1；提交后 Host Bridge 只使用 SDK-native Context v2。两种 schema 原样校验、原样返回，不把 SDK item 补造成 v1 的 `revision`、`evidence_ids`、`paths` 或 `adoption`，已选择的 v2 路径失败时也不回退 legacy 权威。
 - Ask wire：marker 前保留 Governance Ask v1；marker 后 Web / Host Bridge 只调用 SDK Ask v2。v2 的引用只含 SDK 原生 `memory_id/content/layer/status/source/gmt_created`，生成过程中重复 `get` 校验同一 SDK 快照，不补造 revision、Evidence 或 scopes，SDK 失败不回退 v1。
 - Matter：未形成已发布产品合同，不迁移占位数据；所有 `matter*` 表在升级清理时物理删除，fresh install 不创建，Runtime 不注册 Matter API。
-- 旧用户迁移：由 App 在可见的启动 loading 阶段统一执行；此时 Core 尚未对外提供产品读写。启动流程创建/校验完整 bundle、续跑 ledger、完成 SDK backfill 与 reconciliation，满足全部门禁后才提交 marker 并进入主界面。普通用户不需要另跑 CLI，也不按旧 App 内 SDK 版本分叉迁移。
-- SDK backfill 只写同文件系统的私有 staging 副本；验证后先原子切换 SDK
-  root，再提交 authority marker。切换或 marker 提交失败必须恢复原 live root；
-  App 不得在 migration `deferred` 状态继续启动 Runtime。
+- 旧用户迁移：由 App 在可见的启动 loading 阶段统一执行；此时 Core 尚未对外提供产品读写。所有已发布版本都先经 SDK 写入，Governance/Evidence 只是镜像，所以迁移**不复制任何 memory 或 raw**：创建并校验完整 bundle、让写入队列脱离旧表、把旧版命名的 SDK 向量集合接回当前名字、用 Runtime 同样配置的公开 SDK client 按 ID 回读核对，通过后才提交 marker、删除旧表。由一个 Core 命令 `sdk-migrate` 完成；Desktop 启动与 CLI `memhub install`/`upgrade`（macOS、Linux）在启动新 Runtime 前调用它，普通用户不需要另跑命令。App 不得在 migration `deferred` 状态继续启动 Runtime。
+- 迁移期间宿主 capture hook 由 Desktop 活跃 PID marker 静默挡住，不再追加
+  Evidence；cutover 后先从宿主 History checkpoint 补齐遗漏 turn，再恢复 live sync。
 - Matter 若未来重新开发，需以新的 SDK-native 合同重新设计，不继承当前占位 schema。
 
 Pinned SDK 版本见发布锁文件 / `MEMORY.md`（随发布更新）。
@@ -303,9 +303,9 @@ Test entry points: [`../CONTRIBUTING.md`](../CONTRIBUTING.md).
 
 ## Upgrade Compatibility
 
-Released installations require the offline, resumable compatibility protocol in
+Released installations require the offline compatibility protocol in
 [`contracts/sdk-data-plane-migration-v1.md`](contracts/sdk-data-plane-migration-v1.md).
 The App runs it during the startup loading phase before Core exposes product
-reads or writes. A cutover requires a complete bundle, resumable backfill,
-reconciliation, and verified cleanup; a SQLite-only backup or replaying legacy
-rows through extraction is not sufficient.
+reads or writes. A cutover requires a verified full bundle and a write queue
+that no longer depends on legacy tables; it copies no memory, because the SDK
+stores already hold every memory and raw turn.

memhub/docs/INSTALLATION.md

@@ -100,7 +100,7 @@ Downloader 使用标准 `HTTPS_PROXY`/`https_proxy` 与 `NO_PROXY`/`no_proxy`
 - `npm uninstall -g memhub-cli`：只删除 npm CLI；不停止或删除受管 Runtime，不删除 Memory。
   完整 fake Release 验收会在 Runtime 运行中执行真实 npm uninstall，并再次验证
   systemd、`current`、受管 `memhub`、doctor 和 SQLite 均仍可用。
-- `memhub upgrade`：下载并校验新 Runtime，健康检查失败时恢复旧版本。
+- `memhub upgrade`：下载并校验新 Runtime；旧版本写过的数据会先迁移到 SDK 数据面（先做完整备份，终端显示进度），再启动新 Runtime。迁移失败时恢复旧版本；迁移完成后旧版本无法打开数据，健康检查失败时保留新版本并提示 `memhub restart` / `memhub doctor`。
 - `memhub uninstall`：停止并移除受管服务、Runtime versions/current 和受管 CLI 链接；
   默认保留数据。
 - `memhub delete-data`：唯一删除 Memory 的命令；Runtime 必须已停止，并要求 `--yes` 与

memhub/docs/PRODUCT_LOG_TROUBLESHOOTING.md

@@ -6,10 +6,13 @@
 2. 记忆沉淀有没有持续运行、产出了多少条；
 3. 如果停住，停在导入、模型调用还是最终写入。
 
-默认 `info` 下的结构化诊断日志不包含会话正文、模型输入输出或 API Key。`debug`
-会额外启用 HY pipeline trace，其中可能包含完整 prompt/response 和获授权的对话正文。
-排查时优先通过 App 的“导出日志”提供文件，不要直接发送整个 `~/.one-memory` 数据目录；
-debug 期间产生的 pipeline 日志在分享前必须由开发人员确认并脱敏。
+默认 `info` 下的官方结构化事件不应写入会话正文、模型输入输出或 API Key；但本地原始日志允许
+新增受限标量标签，仍应按私人数据处理，不能直接对外发送。`debug` 会额外启用 HY pipeline
+trace，其中可能包含完整 prompt/response 和获授权的对话正文。
+
+排查时只通过 App 的“导出日志”提供文件：导出过程会按安全白名单重新生成诊断事件，不会打包
+原始日志。不要直接发送 `diagnostic-*.jsonl`、整个 `~/.one-memory` 数据目录或 debug pipeline
+日志；后者如确需分享，必须由开发人员另行确认并脱敏。
 
 ## 1. Log level 设什么
 
@@ -39,7 +42,8 @@ debug 可能把完整模型 prompt/response 写入本机调试日志，不得长
 ## 2. 日志在哪里
 
 App 内推荐直接使用“运行诊断 → 导出日志”。开发人员远程指导时，优先使用
-`memhub logs`，它会再次过滤不安全字段。需要现场查看原始结构化日志时，Core 日志默认位于：
+`memhub logs`，它会再次过滤不安全字段。原始结构化日志只用于本机现场查看，不得作为附件发送。
+Core 日志默认位于：
 
 ```text
 ~/.one-memory/logs/diagnostic-core.jsonl

memhub/docs/contracts/diagnostic-logging.md

@@ -15,9 +15,17 @@ Desktop -> Runtime -> Core -> HY -> Host 的实际执行链。日志不是用户
 可选关联字段为 `trace_id`、`incident_id`、`instance_id`、`operation_id`、`request_id`、
 `job_id`、`task_id`、`duration_ms`、状态、稳定错误码、错误类型和清理后的仓库相对栈帧。
 
-`details` 只接受代码声明的标量白名单。禁止记录对话原文、查询、回答、模型输入输出、
-请求体、响应体、token、API Key、secret ref、完整用户路径或异常 message/cause。未知字段和
-不合规标识拒绝写入。日志写入失败不能阻断产品主链。
+`details` 的本地写入合同只约束结构，不维护业务字段白名单：最多 32 个 key；key 必须是
+1–64 字符的字母、数字、点、下划线或连字符并以字母开头；value 只接受 `null`、布尔值、
+跨 Python/JavaScript 可无损表示的有限数字（整数限于 ±(2^53-1)）或不超过 256 字符且不含
+控制换行的短字符串，不接受对象和数组。这样新增诊断标签
+不需要先修改中央白名单。诊断日志的校验、序列化、排队或落盘发生任何异常时都只能丢弃该条
+日志，不能阻断产品主链。
+
+本地写入放宽不代表允许官方代码记录用户内容。对话原文、查询、回答、模型输入输出、请求体、
+响应体、token、API Key、secret ref、完整用户路径和异常 message/cause 仍禁止进入诊断事件，
+由类型、静态检查和隐私测试共同防止误用；确需保存完整用户输入的调试能力必须使用独立的本地
+trace/session 文件，不得混入可导出的诊断日志。
 
 ## 关联规则
 
@@ -34,9 +42,16 @@ Desktop -> Runtime -> Core -> HY -> Host 的实际执行链。日志不是用户
 host-bridge、desktop 和 cli。单文件 4 MiB，保留 3 份轮转；额外清理 14 天以前的文件并将
 总量限制在 128 MiB。POSIX 目录为 0700、文件为 0600。
 
-读取器只枚举上述固定文件名的 regular file，不跟随符号链接，不接受任意路径。损坏行跳过
-并计数，不让一行损坏遮蔽其余证据。诊断导出是单个脱敏 JSON，包含版本、平台、架构、
-非秘密状态和最多 5000 条已验证事件，不包含数据库或配置文件。
+读取器只枚举上述固定文件名的 regular file，不跟随符号链接，不接受任意路径。日常本地查询
+遇到损坏行时跳过并计数，不让一行损坏遮蔽其余证据。
+
+导出是独立的安全边界：它重新读取并校验全部候选事件，只把明确列入导出白名单的字段投影为
+新的 `diagnostics/events.jsonl`；details 不只检查 key，还按字段约束 value 类型，并拒绝自由文本、
+路径和凭证形态的字符串。绝不复制原始 `diagnostic-*.jsonl`、普通运行日志或崩溃报告。
+导出包还可以包含固定 schema 的系统摘要和问题时间点，但不接受调用方传入的任意对象展开。
+任何事件、marker、安全投影或日志目录读取失败时，导出必须 fail-closed：删除临时文件、不发布
+诊断包，更不能回退为打包原始日志。发布使用已设好权限的临时文件原子创建最终目标，已存在的
+目标不得覆盖，发布后也不存在再 chmod 失败留下伪成功文件的窗口。
 
 ## 覆盖与节流
 
@@ -56,7 +71,8 @@ host-bridge、desktop 和 cli。单文件 4 MiB，保留 3 份轮转；额外清
 
 ## 验收
 
-隐私测试必须先植入一条可检测的秘密作为正向对照，再证明真实链路日志和导出均不含该秘密。
-至少用隔离数据制造 Core 启动失败、HY 失败和宿主连接失败，并仅凭日志还原失败组件、时间、
+隐私测试必须先向本地日志植入带未知 key 的可检测秘密作为正向对照，再证明本地读取仍保留该
+证据、导出安全投影必定剔除它；另需证明损坏行会阻止导出且不会留下伪成功文件。至少用隔离
+数据制造 Core 启动失败、HY 失败和宿主连接失败，并仅凭允许导出的字段还原失败组件、时间、
 关联 ID、稳定错误码和代码位置。自动测试通过只能标记 verified，正式安装 App 的真实体验仍需
 单独验收。

memhub/docs/contracts/sdk-data-plane-migration-v1.md

@@ -1,296 +1,192 @@
 # SDK Data Plane Migration v1
 
-Updated: 2026-10-21.
+Updated: 2026-10-24 (vector reattach, readback).
 
-This contract migrates released MemHub installations from the historical
+This contract moves released MemHub installations from the historical
 Evidence/Governance authority to the Hy Memory SDK data plane defined by
 [`../ARCHITECTURE.md`](../ARCHITECTURE.md).
 
-It is an offline, resumable migration executed inside the App's visible startup
-loading phase, before Core accepts product traffic. It is not a permanent
-dual-read or dual-write design and is not a separate manual upgrade flow for
-ordinary users.
+It is an offline migration executed inside the App's visible startup loading
+phase, before Core accepts product traffic. It is not a dual-read or dual-write
+design and not a separate manual upgrade flow for ordinary users.
 
-## 1. Authority before and after cutover
+## 1. Why no memory is copied
 
-Before the cutover marker is committed:
+Every released build (`v1.0.0` through build 19) wrote memory through SDK
+`add()` / `add_extracted()` first. The SDK stored the raw turn (L1 raw) and the
+extracted Chat / Coding Experience / Working Fact memories in its own stores.
+The Governance and Evidence tables were a mirror that MemHub read back from the
+SDK afterwards.
 
-- `evidence*` and `governed_*` remain the product read authority;
-- SDK stores are inspected and backfilled, but are not merged into live search;
-- new imports, writes, queue drains, corrections, deletes and session delivery
-  are paused.
+The SDK stores are therefore already complete. The migration copies no memory
+and no raw turn, and does not compare the mirror with the SDK:
+
+- the mirror's layer labels did not record which Coding store held a row, and
+  the SDK physically deletes experiences its reconciler retires, so the mirror
+  cannot be trusted to decide what the SDK should contain;
+- the product has no user editing of memories in practice, so the mirror holds
+  no user intent the SDK lacks;
+- SDK L1 raw is retained for three days by the Runtime's own
+  `prune_expired_raw`, not by the migration.
+
+"Already complete" holds only if the new Runtime opens the same stores. Builds
+before #145 named a profile's vector collections `one_memory_<12 hex>…`; #145
+renamed them `memhub_<16 hex>…` without carrying the old ones over, so later
+Runtimes read an empty collection (build 20 cut over showing 37 of 3317
+memories). The migration therefore reattaches those collections (§3, step 3)
+and proves the new Runtime reads the memories back before it cuts over
+(step 4).
+
+## 2. Authority before and after cutover
+
+Before the cutover marker is committed, legacy tables remain the product read
+authority; a Runtime that was never upgraded through Desktop or `memhub
+install`/`upgrade` still serves them. Desktop does not start Runtime while
+migration is incomplete.
 
 After the cutover marker is committed:
 
-- SDK memory/raw/vector/graph/coding stores are the only memory data authority;
+- SDK stores are the only memory data authority;
 - MemHub stores management state and `memory_id` / raw references only;
-- legacy Evidence/Governance/retrieval/session-shadow tables are physically
+- legacy Evidence/Governance/retrieval/session-shadow and Matter tables are
   dropped before the normal App window is revealed;
-- startup must not fall back to legacy authority because an SDK read failed.
+- startup never falls back to legacy authority.
 
-The surviving `memory_authority_state` singleton, not legacy table existence,
-selects the authority.
+The surviving `memory_authority_state` singleton, not table existence, selects
+the authority.
 
-### Fresh install
+A fresh home bootstraps the latest management schema with SDK authority
+directly and never creates legacy tables.
 
-A fresh home does not execute the historical application schema chain. It
-bootstraps the latest management-plane schema directly with SDK authority and
-never creates Evidence, Governance, legacy retrieval, any Matter, or
-session-shadow tables. Historical schema migrations are an input format reader
-for old homes only.
+## 3. Who runs it, and how
 
-## 2. Supported source state
+One Core command does the whole migration:
 
-The migration preflight records and validates:
+```text
+one-memory sdk-migration-status --json      # read-only: is there work?
+one-memory sdk-migrate --runtime-build <build> --yes --json
+```
 
-- source MemHub Runtime tag/build and management schema version;
-- migration Runtime SDK capabilities and the actual on-disk store shape;
-- SDK namespace, vector/graph/cache/history/coding providers and locations;
-- embedding model identity and dimensions;
-- source grants, model profiles, config, queue leases, and pending operations.
+`sdk-migrate` requires a stopped Runtime. It takes the host capture fence
+(hooks return silently instead of writing Evidence while its PID is alive),
+then:
 
-The migration fails closed when the installed SDK lacks:
+1. **bundle**: copies `one-memory.sqlite3`, the SDK data plane and non-secret
+   configuration into a versioned bundle and verifies it by an isolated
+   restore; an existing verified bundle of the same run is reused;
+2. **prepare**: applies the management schema migrations, discards SDK staging
+   left by builds through 19 (§5) and makes legacy queue work self-contained
+   (§4);
+3. **vectors**: for the active embedding profile, renames every
+   `one_memory_<old hash>…` collection to the current `memhub_<hash>…` name
+   through Chroma's public client. A current collection that already holds
+   writes is merged in first (newer record wins on a shared id); counts are
+   checked after each step;
+4. **verify**: opens the public SDK client exactly as the Runtime will and reads
+   the legacy confirmed memory ids back (`list_memories`, then `get`). Every
+   chat memory must read back. Coding/Working may be fewer, because the SDK's
+   Coding reconciler deletes experiences it retires, but not zero when the
+   legacy mirror has some. Otherwise the run stops with
+   `sdk_migration_memory_unreadable` and the counts, before anything
+   irreversible; legacy authority and tables stay intact;
+5. **cutover**: re-verifies the bundle, repeats §4, commits SDK authority;
+6. **cleanup**: drops the legacy tables (including the `evidence_search`
+   index) and runs `VACUUM` (skipped if the disk is too full).
 
-- complete lifecycle/raw inventory;
-- exact raw and exact memory import;
-- durable request outcome lookup;
-- consistent snapshot/export and restore.
+A home with legacy memories but no active model profile is not migrated
+(`sdk_migration_model_profile_required`): its collections can be neither named
+nor read back.
 
-MemHub tags `v1.0.0` through `v1.0.4` all use the `governed-v2` namespace and
-the same SDK SQL table families (`memory_operations_v2`, `pipeline_logs_v2`,
-`system_metrics`, `episode_state`, `memory_history`, `coding_memory_meta`,
-`working_fact`, and `coding_trace`). They therefore use one migration selected
-by inspected schema/store capabilities, not one branch per historical SDK pin.
-The old SDK version is informational only and is not a migration discriminator.
+A home already on SDK authority still reports `required` while legacy-named
+vector collections exist for its profile (build 20 homes). `sdk-migrate` then
+reattaches them and reads back against the memory ids in the pre-cutover
+bundle; that report is recorded but does not block, as nothing irreversible is
+left. Runtime code never renames or reads Chroma directly.
 
-The migration Runtime itself must expose the required APIs. Its exact SDK
-version is defined by the release lockfiles; old data is not rejected merely
-because an earlier App wrote it.
+With `--json` each step prints `{"state":"running","step":…}` and the last line
+is the result: `cutover`, `complete` (only cleanup was left), `not_required`, or
+`failed` with a stable `code`. Every step is safe to repeat; a retry resumes the
+newest unfinished `sdk-authority-v2-*` run and its bundle, including a run that
+build 19 left in `review_required/blocked`.
 
-## 3. Legacy-state classification
+Callers only keep the Runtime stopped, and its service from restarting, around
+that command (`runSdkMigration` in `packages/runtime-control`):
 
-While Core and Runtime writers are stopped, build four inventories:
+- **Desktop** runs it in the startup loading phase before attaching the UI; a
+  failure keeps the App closed with the code's sentence.
+- **CLI `memhub install` / `memhub upgrade`** (macOS and Linux) run it with the
+  new Runtime's Core after switching `current` and before starting the service.
+  A failure before cutover rolls back to the previous version. After cutover an
+  older Runtime refuses the home, so a later readiness failure keeps the new
+  version and reports `runtime_not_ready_after_sdk_migration`.
 
-- `E`: Evidence rows and revisions;
-- `G`: current Governance rows, statuses, supersession links, and latest content;
-- `M`: task-output mappings from legacy `memory_id` to SDK `hy_memory_id`;
-- `S`: complete SDK inventory, including raw and non-active lifecycle records.
+After cutover, History catch-up scans forward from each source checkpoint and
+recovers turns skipped while capture was fenced.
 
-Every item must end in exactly one class:
+## 4. Queue work
 
-| Class | Meaning | Required action |
-| --- | --- | --- |
-| matched | Governance/mapping and SDK identity, content, owner and lifecycle agree | verify only |
-| mapped mismatch | legacy ID maps to a different existing SDK ID | keep SDK ID; record alias |
-| Governance-only | current legacy memory has no SDK entity | exact import without extraction |
-| SDK-only | SDK entity has no legacy mirror | retain and adopt |
-| divergent | both exist but content, owner or lifecycle differs | reconcile explicit user intent or quarantine |
-| raw-only | Evidence has no SDK raw receipt | exact raw import |
-| unknown outcome | a task may have reached SDK without a confirmed result | SDK outcome inquiry; never blind retry |
+Queue rows remain MemHub management state. Before the legacy tables disappear:
 
-An unclassified item blocks cutover.
+- each `memory_write_jobs` row that only names Evidence IDs gets those records
+  frozen into `payload_json`;
+- unstarted legacy `memory_tasks` become
+  `memory_write_jobs(idempotency_key="legacy-task:<task_id>")`;
+- Evidence waiting in `evidence_pending_memory_dispatch` (its source cursor has
+  already passed it) becomes one write job per conversation.
 
-## 4. User edits, deletion and supersession
+Legacy work that cannot continue does not block cutover; its counts are
+reported in the `sdk-migrate` / cutover receipt as `abandoned`:
 
-The current product has no MemHub revision history or Undo.
+- a legacy task that may already have reached the SDK (`unknown`, or `running`
+  with an SDK invocation) is not replayed: the SDK already holds whatever it
+  committed;
+- pending legacy user deletions are not carried over.
 
-- The latest non-deleted user correction is imported as the current SDK body.
-- Historical `governed_memory_versions` remain in the compatibility backup, but
-  are not recreated as product-visible SDK versions.
-- A legacy user deletion is applied as an SDK physical delete. Deleted
-  Governance tombstones are not imported and no deleted-list UI survives.
-- SDK supersession remains an SDK lifecycle relationship. Existing SDK chains
-  are retained. Governance links are used only to repair a missing SDK chain
-  during exact import.
-- Pending correction/delete records are settled before ordinary backfill.
+Surviving `memory_write_jobs` with `hy_invoked=1` keep their normal
+post-cutover handling through SDK `get_write_outcome(request_id)`.
 
-`update(memory_id, content)` modifies the same SDK ID. There is no product CAS,
-revision number, or restore operation.
+## 5. Staging left by earlier builds
 
-## 5. Evidence to SDK raw
+Builds through 19 copied the SDK root to `<home>/.sdk-migration/` and swapped
+directories at cutover. Before SDK authority, a leftover
+`hy-memory/.<namespace>.pre-migration-*` directory is the original store and is
+restored; after SDK authority it is stale and removed. More than one such
+directory before authority is ambiguous and fails closed
+(`sdk_migration_swap_state_ambiguous`).
 
-Evidence must not be replayed through ordinary `add()`: doing so would extract
-and reconcile again, changing semantic memories.
+## 6. Failure and rollback
 
-The SDK exact-raw import preserves:
+Any failure leaves the marker absent and reports a stable code to the startup
+dialog (for example `sdk_migration_disk_space_insufficient`,
+`sdk_migration_bundle_copy_failed`, `sdk_migration_preflight_failed`,
+`sdk_migration_cutover_failed`). The next startup retries.
 
-- source, conversation/session, workspace, role and author;
-- original event time and stable source ordering;
-- verbatim content and source identity;
-- a deterministic migration key.
+After cutover but before any SDK-only write, rollback restores the bundle
+together with the matching old Runtime. After an SDK-only write, automatic
+binary-only rollback is forbidden; restoring the pre-cutover bundle loses the
+post-cutover writes and must be explicit. An old Runtime must fail closed on a
+post-cutover home. Changing only the Runtime `current` symlink is not data
+rollback.
 
-The operation is idempotent and returns an SDK raw ID. Evidence revisions and
-repairs are preserved in the migration bundle for audit; only the current
-accepted raw record becomes the normal product read.
-
-## 6. Full migration bundle
-
-A successful preflight creates one versioned bundle containing:
-
-- an online backup of `one-memory.sqlite3`;
-- an SDK-consistent export/snapshot of raw, vector, graph, cache, history and
-  coding stores;
-- `config.json` and non-secret configuration needed to locate those stores;
-- Runtime, SDK, schema, namespace, backend and embedding fingerprints;
-- object/file hashes, row counts, object counts and creation time;
-- migration ledger version and bundle ID.
-
-Secrets are not copied into a plaintext manifest. The bundle records only
-secret references and the secure-store identity required for restore.
-
-The bundle is published only after an isolated restore verifies hashes, counts
-and SDK reads. A remote backend without a supported snapshot/export mechanism
-blocks migration; SQLite-only backup is not reported as success.
-
-## 7. Authority state and migration ledger
-
-The MemHub management database owns:
-
-- `memory_authority_state`: the surviving singleton authority and migration ID;
-- `sdk_data_plane_migrations`: one run, source/target fingerprints, bundle,
-  phase, cutover state and reconciliation digest;
-- `sdk_data_plane_migration_items`: source identity/revision/hash, target SDK
-  identity/hash, deterministic operation key, state, attempts and error code;
-- `memory_id_aliases`: temporary historical ID to canonical SDK ID mapping for
-  call-events, write-job receipts, and links;
-- `raw_id_aliases`: legacy Evidence ID/item index to canonical SDK raw ID.
-- `sdk_data_plane_cutover`: a temporary detailed cutover receipt containing
-  SDK/backend/bundle/reconciliation fingerprints.
-
-Progress is committed only after SDK verification. Retrying an item uses the
-same deterministic SDK operation key and must not create another memory.
-Before cleanup, every surviving reference is rewritten to its canonical SDK
-ID. Alias, item-ledger, and detailed cutover tables are then dropped; the
-authority singleton and verified bundle remain.
-
-Backfill never mutates the live SDK root. Startup first copies the complete
-local SDK data plane, including SDK-only records, to a same-filesystem staging
-home. Exact legacy writes and reconciliation run there. Cutover renames the
-live root to a protective path, renames staging into place, and only then
-commits SDK authority. A failed commit restores the previous root; restart
-recovery resolves any interrupted rename before opening SDK stores.
-
-## 8. Queue and unknown outcomes
-
-Queue rows and leases remain MemHub management state.
-
-- A pending job has not called the SDK and therefore has no SDK raw or memory
-  ID yet.
-- Its worker calls SDK `add()` once. The SDK owns raw persistence plus
-  extraction/reconciliation; a successful job stores only the returned raw ID,
-  canonical memory IDs and terminal outcome.
-- Legacy raw and semantic backfill is an App-owned privileged offline
-  operation while Runtime writers are stopped. It writes SDK stores directly;
-  no migration-only import/get/list method is added to public
-  `HyMemoryClient`, and this is not the first half of a two-stage normal write
-  path.
-
-- `hy_invoked=0` may be safely retried after normal lease recovery.
-- `memory_write_jobs.hy_invoked=1`, `state=unknown`, or any non-terminal SDK
-  outcome must use `get_write_outcome(request_id)`.
-- `running`/`unknown` is not absence and must not be replayed.
-- confirmed SDK IDs become write-job outcomes/receipts; Governance publication is not part of
-  completion after cutover.
-
-## 9. Call events and session delivery
-
-- Matter was an unpublished placeholder. Its rows are not migrated and every
-  `matter*` table is deleted during cleanup.
-- Before cleanup, historical IDs resolve through `memory_id_aliases`; surviving
-  records are rewritten to canonical SDK IDs before that table is dropped.
-- `call_events` remains metadata-only; reference titles/content resolve through
-  SDK APIs.
-- Evidence-based session inbox/outbox is disabled at cutover unless replaced by
-  a management-event or SDK-event contract. It must not send shadow memory
-  bodies.
-
-## 10. Cutover
-
-The packaged App owns orchestration. While the startup placeholder is visible,
-it starts no product writer, creates or verifies the full bundle, resumes the
-durable migration ledger, reconciles SDK state, and only then commits cutover.
-The normal App window is revealed after this phase. A retry resumes the same
-migration and deterministic operation keys; it does not create a second run.
-
-If migration cannot safely complete, the marker remains absent and startup must
-report the durable blocked/deferred reason. It must not silently continue into
-an SDK/legacy dual-write mode. A missing model profile may defer migration while
-legacy authority is still intact so the user can configure the App; it never
-authorizes a partial cutover.
-
-Cutover requires:
-
-- no unclassified or quarantined item;
-- complete raw coverage;
-- no unsafe unknown task;
-- pending corrections and deletions settled;
-- call-event references resolvable;
-- exact counts and hashes reconciled;
-- deterministic full-ID get verification;
-- a verified full migration bundle.
-
-The final transaction writes the detailed cutover receipt and changes
-`memory_authority_state` to SDK authority. Cleanup may remove the detailed
-receipt after bundle verification; the authority singleton remains.
-
-After that transaction, legacy memory/raw writes are rejected.
-
-## 11. Rollback and version skew
-
-Before cutover, a failed run resumes from the ledger or restores the full
-bundle.
-
-After cutover but before any SDK-only write, rollback restores the full bundle
-and matching old Runtime together.
-
-After an SDK-only write:
-
-- automatic binary-only rollback is forbidden;
-- operators either forward-fix, or explicitly restore the pre-cutover bundle
-  while acknowledging loss of post-cutover writes;
-- an old Runtime must fail closed on the post-cutover marker/schema.
-
-Changing only the Runtime `current` symlink is not data rollback.
-
-## 12. Legacy cleanup
-
-Legacy tables are not a compatibility runtime. After every memory/raw
-reference resolves, the verified bundle has passed restore validation, and the
-SDK cutover marker is durable, the offline startup transaction drops:
-
-- every `evidence*` and `governed_*` table;
-- legacy retrieval/index tables;
-- every `matter*` placeholder table;
-- legacy deletion/index-sync/task-output and session-shadow tables;
-- migration-only aliases and ledgers after their canonical references have
-  been materialized into the surviving management tables.
-
-Before that DROP, every surviving `memory_write_jobs` row is made
-self-contained by freezing its source records into `payload_json`. Unstarted
-legacy `memory_tasks` rows are converted to deterministic
-`memory_write_jobs(idempotency_key="legacy-task:<task_id>")`; possibly-sent
-task rows remain a cutover blocker until SDK request-ID reconciliation proves
-their outcome.
-
-The verified bundle is the audit and rollback artifact; keeping duplicate
-plaintext tables in the live database is not. A cleanup failure keeps startup
-blocked and never exposes a half-cleaned schema.
-
-## 13. Acceptance matrix
+## 7. Acceptance
 
 The release gate covers:
 
-- a clean install with no legacy rows;
-- a clean install whose schema contains no legacy table names;
-- Governance-only, SDK-only, mapped-ID and divergent fixtures;
-- Evidence/raw-only content;
-- pending correction and physical deletion;
-- unknown SDK outcome and crash after SDK commit;
-- interruption before/after each ledger transition;
-- full bundle restore and hash/count mismatch;
-- call-event alias preservation;
-- old Runtime against a post-cutover home;
-- successful migration followed by physical absence of every legacy table;
-- local and supported remote backend snapshot behavior.
-
-No narrow unit test may be used as evidence that the full migration is safe.
+- a fresh install with no legacy table names;
+- a legacy home reaching cutover with its SDK stores byte-for-byte unchanged;
+- a home left blocked by build 19 (quarantined ledger and staging copy)
+  completing on the next startup;
+- CLI upgrade migrating before restart, rolling back on a failed migration,
+  and keeping the new version when a migrated home is slow to come up;
+- host capture fenced for the whole command;
+- legacy-named vector collections reattached (with merge) before verifying,
+  and on a home that already cut over;
+- readback refusing a cutover the new Runtime cannot read, and tolerating
+  retired Coding experiences;
+- `/api/v2/memory-authority` agreeing with the API gates on fresh, migrated
+  and legacy homes;
+- pending job freezing, legacy task and pending dispatch conversion;
+- invoked/unknown legacy tasks and pending deletions not blocking cutover;
+- interrupted directory swap recovery before and after authority;
+- bundle creation, reuse and restore;
+- physical absence of every legacy table after cutover.

memhub/package.json

@@ -2,7 +2,7 @@
   "name": "one-memory",
   "version": "1.0.5",
   "release": {
-    "build_number": 14,
+    "build_number": 17,
     "macos_min_version": "13.0",
     "app_update": "manual",
     "runtime_manifest_schema": 1,

memhub/packages/host-bridge/src/recall-broker.test.ts

@@ -192,16 +192,12 @@ describe("RecallBroker", () => {
       level: "error",
       component: "host.broker",
       outcome: "failed",
-      details: expect.objectContaining({
-        action: "host",
-        code: "connection_control_unavailable",
-        error_type: "Error",
-      }),
+      error_code: "connection_control_unavailable",
+      error_type: "Error",
+      details: { action: "host" },
     });
-    expect(typeof exception?.details).toBe("object");
-    const details = exception?.details as Record<string, unknown>;
-    expect(typeof details.stack_origin).toBe("string");
-    expect(details.stack_origin).toContain("recall-broker");
+    expect(exception?.details).not.toHaveProperty("message");
+    expect(exception?.details).not.toHaveProperty("stack_origin");
   });
   it.each(["recall", "ask"])(
     "answers %s without waiting for a slow diagnostic write",

memhub/packages/host-bridge/src/recall-broker.ts

@@ -702,20 +702,10 @@ export class RecallBroker {
           event: "connection_control.exception",
           outcome: "failed",
           instance_id: this.options.instanceId,
-          details: {
-            action: action ?? "invalid",
-            code,
-            error_type:
-              error instanceof Error ? error.constructor.name : typeof error,
-            message:
-              error instanceof Error
-                ? error.message.slice(0, 256)
-                : String(error).slice(0, 256),
-            stack_origin:
-              error instanceof Error && error.stack
-                ? (error.stack.split("\n").at(1)?.trim() ?? "").slice(0, 256)
-                : null,
-          },
+          error_code: code,
+          error_type:
+            error instanceof Error ? error.constructor.name : typeof error,
+          details: { action: action ?? "invalid" },
         });
       }
       const fingerprint = `error:${code}`;

memhub/packages/i18n/src/gui-i18n-english.ts

@@ -1933,6 +1933,8 @@ const baseEnglishMessages = {
   languageLoadError: "Could not load the language setting. Please retry.",
   nativeErrorTitle: "MemHub could not start",
   startupPreparing: "Starting and preparing the memory service…",
+  startupUpgradeHint:
+    "The first launch or a memory-data upgrade can take a few minutes. MemHub opens automatically when it is done.",
 } as const;
 
 export const englishMessages = {

memhub/packages/i18n/src/gui-i18n-messages.ts

@@ -52,6 +52,8 @@ const languageChineseMessages = {
   languageLoadError: "无法加载语言设置，请重试。",
   nativeErrorTitle: "MemHub 无法启动",
   startupPreparing: "正在启动，正在准备记忆服务…",
+  startupUpgradeHint:
+    "首次启动或升级记忆数据可能需要几分钟，完成后会自动进入应用。",
 } as const;
 
 const languageSpanishMessages: Record<
@@ -71,6 +73,8 @@ const languageSpanishMessages: Record<
   languageLoadError: "No se pudo cargar el idioma. Inténtalo de nuevo.",
   nativeErrorTitle: "MemHub no pudo iniciarse",
   startupPreparing: "Iniciando y preparando el servicio de memorias…",
+  startupUpgradeHint:
+    "El primer inicio o la actualización de tus memorias puede tardar unos minutos. La aplicación se abrirá automáticamente al terminar.",
 };
 
 type Catalog = {

memhub/packages/runtime-control/src/core-supervisor.ts

@@ -680,7 +680,7 @@ export class CoreSupervisor {
           outcome: "failed",
           instance_id: this.activeInstanceId,
           error_code: "core_restart_circuit_open",
-          details: { streak: this.restartStreak },
+          details: { restart_streak: this.restartStreak },
         });
         return;
       }
@@ -765,7 +765,7 @@ export class CoreSupervisor {
             outcome: "failed",
             instance_id: this.activeInstanceId,
             error_code: "core_restart_exhausted",
-            details: { attempts: budget, streak: this.restartStreak },
+            details: { attempts: budget, restart_streak: this.restartStreak },
           });
           await this.options.diagnostics?.log({
             level: "error",
@@ -774,7 +774,7 @@ export class CoreSupervisor {
             outcome: "failed",
             instance_id: this.activeInstanceId,
             error_code: "core_restart_circuit_open",
-            details: { streak: this.restartStreak },
+            details: { restart_streak: this.restartStreak },
           });
         }
       } finally {

memhub/packages/runtime-control/src/diagnostic-log.test.ts

@@ -1,4 +1,5 @@
 import {
+  chmod,
   lstat,
   mkdir,
   mkdtemp,
@@ -21,8 +22,10 @@ import {
   diagnosticLogFiles,
   exportDiagnosticBundle,
   exportDiagnosticZip,
+  projectDiagnosticEventForSafeOutput,
   pruneDiagnosticLogs,
   readDiagnosticEvents,
+  recordDiagnosticMarker,
   writeDiagnosticEventSync,
 } from "./diagnostic-log.js";
 
@@ -66,6 +69,30 @@ describe("diagnostic log", () => {
     });
   });
 
+  it("keeps the synchronous writer fail-open for invalid input", async () => {
+    const dataDir = await root();
+    expect(() =>
+      writeDiagnosticEventSync(
+        {
+          dataDir,
+          service: "desktop",
+          sessionId: "55555555-5555-4555-8555-555555555555",
+        },
+        {
+          level: "error",
+          component: "desktop.lifecycle",
+          event: "desktop.early_start_failed",
+          outcome: "failed",
+          details: { nested: ["private"] } as never,
+        },
+      ),
+    ).not.toThrow();
+    expect(await readDiagnosticEvents(dataDir)).toEqual({
+      events: [],
+      invalid_lines: 0,
+    });
+  });
+
   it("writes structured, private events and reads them by correlation identity", async () => {
     const dataDir = await root();
     const session = "11111111-1111-4111-8111-111111111111";
@@ -89,6 +116,8 @@ describe("diagnostic log", () => {
         open_file_count: 7,
         memory_bytes: 4096,
         thread_count: 4,
+        "cache.hit-rate": 0.75,
+        custom_label: "warm",
       },
     });
     const result = await readDiagnosticEvents(dataDir, { traceId: trace });
@@ -101,6 +130,8 @@ describe("diagnostic log", () => {
       event: "runtime.ready",
       details: expect.objectContaining({
         open_file_count: 7,
+        "cache.hit-rate": 0.75,
+        custom_label: "warm",
       }),
     });
     const file = (await diagnosticLogFiles(dataDir))[0];
@@ -184,7 +215,7 @@ describe("diagnostic log", () => {
     }
   });
 
-  it("rejects arbitrary detail fields and absolute stack paths", () => {
+  it("strictly rejects complex detail values, dangerous keys and absolute stacks", () => {
     const base = {
       timestamp: new Date().toISOString(),
       level: "error" as const,
@@ -195,11 +226,142 @@ describe("diagnostic log", () => {
       session_id: "11111111-1111-4111-8111-111111111111",
     };
     expect(() =>
-      diagnosticEvent({ ...base, details: { prompt: "private" } }),
+      diagnosticEvent({
+        ...base,
+        details: { nested: { secret: "private" } } as never,
+      }),
+    ).toThrow("diagnostic_detail_value_invalid");
+    expect(() =>
+      diagnosticEvent({ ...base, details: { constructor: "private" } }),
     ).toThrow("diagnostic_detail_key_invalid");
+    expect(() =>
+      diagnosticEvent({ ...base, details: { "9invalid": "private" } }),
+    ).toThrow("diagnostic_detail_key_invalid");
+    expect(() =>
+      diagnosticEvent({
+        ...base,
+        details: Object.fromEntries(
+          Array.from({ length: 33 }, (_, index) => [`key_${index}`, index]),
+        ),
+      }),
+    ).toThrow("diagnostic_detail_count_invalid");
+    expect(() =>
+      diagnosticEvent({ ...base, details: { message: "line1\nline2" } }),
+    ).toThrow("diagnostic_text_invalid");
+    expect(() =>
+      diagnosticEvent({ ...base, details: { metric: Number.NaN } }),
+    ).toThrow("diagnostic_detail_value_invalid");
+    expect(() =>
+      diagnosticEvent({
+        ...base,
+        details: { count: Number.MAX_SAFE_INTEGER + 1 },
+      }),
+    ).toThrow("diagnostic_detail_value_invalid");
+    expect(() =>
+      diagnosticEvent({ ...base, details: { list: ["private"] } as never }),
+    ).toThrow("diagnostic_detail_value_invalid");
     expect(() =>
       diagnosticEvent({ ...base, safe_stack: ["/Users/me/private.py:1"] }),
     ).toThrow("diagnostic_stack_invalid");
+    expect(() =>
+      diagnosticEvent({
+        ...base,
+        safe_stack: ["packages/runtime/sk-live-secret-token.ts:1:1"],
+      }),
+    ).toThrow("diagnostic_stack_invalid");
+    for (const line of [
+      "arbitrary stack text",
+      "thread=worker|threading.py:12:wait",
+      "FILE:///Users/me/private.py:1:call",
+      "one_memory/worker.py:9:call /Users/private",
+      "one_memory/GOCSPX-privateCredential.py:9:call",
+      "packages/ASIA1234567890ABCDEF/file.ts:1:1",
+      "packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
+    ]) {
+      expect(() => diagnosticEvent({ ...base, safe_stack: [line] })).toThrow(
+        "diagnostic_stack_invalid",
+      );
+    }
+    expect(() =>
+      diagnosticEvent({
+        ...base,
+        safe_stack: Array.from(
+          { length: 17 },
+          (_, index) => `one_memory/worker.py:${index + 1}:call`,
+        ),
+      }),
+    ).toThrow("diagnostic_stack_invalid");
+
+    const accepted = diagnosticEvent({
+      ...base,
+      safe_stack: [
+        "apps/desktop/src/main.ts:80:1",
+        "one_memory/diagnostic_log.py:895:_safe_stack_line",
+        "threading.py:1045:_bootstrap_inner",
+        "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+      ],
+    });
+    expect(accepted.safe_stack).toHaveLength(4);
+
+    const projected = projectDiagnosticEventForSafeOutput({
+      ...accepted,
+      safe_stack: [
+        "free text",
+        "thread=worker|one_memory/worker.py:9:call",
+        "packages/runtime-control/src/diagnostic-log.ts:42:7",
+        "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+      ],
+    });
+    expect(projected.safe_stack).toEqual([
+      "packages/runtime-control/src/diagnostic-log.ts:42:7",
+      "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+    ]);
+    const conservative = projectDiagnosticEventForSafeOutput(
+      diagnosticEvent({
+        ...base,
+        details: {
+          source_id: "cursor",
+          reason: "abcdef1234567890abcdef1234567890abcdef12",
+          status: "11111111-1111-4111-8111-111111111111",
+        },
+      }),
+    );
+    expect(conservative.details).toEqual({
+      source_id: "cursor",
+      reason: "abcdef1234567890abcdef1234567890abcdef12",
+      status: "11111111-1111-4111-8111-111111111111",
+    });
+  });
+
+  it("drops malformed logger events without interrupting later writes", async () => {
+    const dataDir = await root();
+    const logger = new DiagnosticLogger({
+      dataDir,
+      service: "desktop",
+      sessionId: "11111111-1111-4111-8111-111111111111",
+    });
+
+    await expect(
+      logger.log({
+        level: "info",
+        component: "desktop.sdk_migration",
+        event: "sdk_migration.startup",
+        outcome: "succeeded",
+        details: { nested: { secret: "private" } } as never,
+      }),
+    ).resolves.toBeUndefined();
+    await logger.log({
+      level: "info",
+      component: "desktop.sdk_migration",
+      event: "sdk_migration.startup",
+      outcome: "succeeded",
+      details: { arbitrary_state: "cutover" },
+    });
+
+    const result = await readDiagnosticEvents(dataDir);
+    expect(result.invalid_lines).toBe(0);
+    expect(result.events).toHaveLength(1);
+    expect(result.events[0].details).toEqual({ arbitrary_state: "cutover" });
   });
 
   it("keeps code locations but strips exception messages and absolute prefixes", () => {
@@ -215,29 +377,162 @@ describe("diagnostic log", () => {
     expect(JSON.stringify(safe)).not.toContain("/Users/me");
   });
 
-  it("marks incidents and exports only validated events", async () => {
+  it("exports events through an independent detail whitelist", async () => {
     const dataDir = await root();
     const logger = new DiagnosticLogger({ dataDir, service: "desktop" });
     const incident = await logger.markIncident({
       component: "desktop.diagnostics",
       status: "degraded",
     });
-    await writeFile(
-      path.join(dataDir, "logs", "diagnostic-core.jsonl"),
-      "not-json\n",
-      { mode: 0o600 },
-    );
+    await logger.log({
+      level: "info",
+      component: "desktop.diagnostics",
+      event: "diagnostics.checked",
+      outcome: "succeeded",
+      details: {
+        count: 1,
+        reason: "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signaturevalue",
+        status: "ASIA1234567890ABCDEF",
+        source_id: "GOCSPX-clientSecret_A1",
+        mode: "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+        action: "AIzaSyExampleCredential1234567890",
+        platform: "x_ASIA1234567890ABCDEF",
+        phase: "x_GOCSPX-clientSecret_A1",
+        local_only_secret: "must-not-export",
+      },
+    });
+    await logger.log({
+      level: "error",
+      component: "credential.private",
+      event: "token.failed",
+      outcome: "ghp_private_token",
+      trace_id: "ghp_private_token",
+      instance_id: "ASIA1234567890ABCDEF",
+      request_id: "GOCSPX-clientSecret_A1",
+      task_id: "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+      error_code: "secret.token",
+    });
     const target = path.join(await root(), "bundle.json");
     const result = await exportDiagnosticBundle({
       dataDir,
       target,
       productVersion: "1.2.3",
-      status: { status: "degraded" },
+      status: { status: "degraded", local_only_secret: "must-not-export" },
     });
-    expect(result).toEqual({ event_count: 1, invalid_lines: 1 });
+    expect(result).toEqual({ event_count: 3, invalid_lines: 0 });
     const payload = JSON.parse(await readFile(target, "utf8"));
     expect(payload.events[0].incident_id).toBe(incident.incident_id);
-    expect(JSON.stringify(payload)).not.toContain(dataDir);
+    expect(payload.events[1].details).toEqual({ count: 1 });
+    expect(payload.events[2]).toMatchObject({
+      component: "diagnostic.redacted",
+      event: "diagnostic.redacted",
+      outcome: "redacted",
+    });
+    expect(payload.events[2].trace_id).toBeUndefined();
+    expect(payload.events[2].instance_id).toBeUndefined();
+    expect(payload.events[2].request_id).toBeUndefined();
+    expect(payload.events[2].task_id).toBeUndefined();
+    expect(payload.events[2].error_code).toBeUndefined();
+    expect(JSON.stringify(payload)).not.toContain("local_only_secret");
+    expect(JSON.stringify(payload)).not.toContain("must-not-export");
+    expect(JSON.stringify(payload)).not.toContain("eyJhbGciOiJIUzI1NiJ9");
+    expect(JSON.stringify(payload)).not.toContain("ASIA1234567890ABCDEF");
+    expect(JSON.stringify(payload)).not.toContain("GOCSPX-clientSecret_A1");
+    expect(JSON.stringify(payload)).not.toContain("x_ASIA1234567890ABCDEF");
+    expect(JSON.stringify(payload)).not.toContain("x_GOCSPX-clientSecret_A1");
+    expect(JSON.stringify(payload)).not.toContain("Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8");
+    expect(JSON.stringify(payload)).not.toContain(
+      "AIzaSyExampleCredential1234567890",
+    );
+    expect(JSON.stringify(payload)).not.toContain("ghp_private_token");
+  });
+
+  it("fails closed when exported build identity has a credential shape", async () => {
+    const dataDir = await root();
+    const outputDir = await root();
+    await expect(
+      exportDiagnosticBundle({
+        dataDir,
+        target: path.join(outputDir, "bundle.json"),
+        productVersion: "xoxb-123456-abcdef",
+      }),
+    ).rejects.toThrow("diagnostic_product_version_invalid");
+    await expect(
+      exportDiagnosticZip({
+        dataDir,
+        target: path.join(outputDir, "commit.zip"),
+        productVersion: "1.2.3",
+        commit: "ghp_private_token",
+      }),
+    ).rejects.toThrow("diagnostic_commit_invalid");
+    await expect(
+      exportDiagnosticZip({
+        dataDir,
+        target: path.join(outputDir, "sha.zip"),
+        productVersion: "1.2.3",
+        coreSha256: "AKIDExampleSecretId1234567890",
+      }),
+    ).rejects.toThrow("diagnostic_sha256_invalid");
+    expect(await readdir(outputDir)).toEqual([]);
+  });
+
+  it("fails closed without publishing a JSON bundle when any line is invalid", async () => {
+    const dataDir = await root();
+    await mkdir(path.join(dataDir, "logs"), { recursive: true });
+    await writeFile(
+      path.join(dataDir, "logs", "diagnostic-core.jsonl"),
+      "not-json\n",
+      { mode: 0o600 },
+    );
+    const outputDir = await root();
+    const target = path.join(outputDir, "bundle.json");
+    await expect(
+      exportDiagnosticBundle({
+        dataDir,
+        target,
+        productVersion: "1.2.3",
+      }),
+    ).rejects.toThrow("diagnostic_export_invalid_lines");
+    expect(await readdir(outputDir)).toEqual([]);
+  });
+
+  it("fails closed when the diagnostic directory cannot be read", async () => {
+    if (process.platform === "win32") return;
+    const dataDir = await root();
+    const logs = path.join(dataDir, "logs");
+    await mkdir(logs, { recursive: true, mode: 0o700 });
+    await writeFile(path.join(logs, "diagnostic-core.jsonl"), "{}\n");
+    const outputDir = await root();
+    const target = path.join(outputDir, "bundle.json");
+    await chmod(logs, 0o000);
+    try {
+      await expect(
+        exportDiagnosticBundle({
+          dataDir,
+          target,
+          productVersion: "1.2.3",
+        }),
+      ).rejects.toMatchObject({ code: "EACCES" });
+    } finally {
+      await chmod(logs, 0o700);
+    }
+    expect(await readdir(outputDir)).toEqual([]);
+  });
+
+  it("never overwrites an existing export target or leaves temporary files", async () => {
+    const dataDir = await root();
+    const outputDir = await root();
+    const target = path.join(outputDir, "bundle.json");
+    await writeFile(target, "existing-artifact", { mode: 0o600 });
+    await expect(
+      exportDiagnosticBundle({
+        dataDir,
+        target,
+        productVersion: "1.2.3",
+      }),
+    ).rejects.toMatchObject({ code: "EEXIST" });
+    expect(await readFile(target, "utf8")).toBe("existing-artifact");
+    expect(await readdir(outputDir)).toEqual(["bundle.json"]);
   });
 
   it("preserves microsecond ordering across service log files", async () => {
@@ -280,7 +575,7 @@ describe("diagnostic log", () => {
     expect(result.events[1].timestamp).toBe("2026-10-14T00:00:00.123999Z");
   });
 
-  it("exports a private seven-day zip with summary, logs, crashes and markers", async () => {
+  it("exports only reprojected events, a strict summary and README", async () => {
     const dataDir = await root();
     const outputDir = await root();
     const crashDir = await root();
@@ -326,39 +621,178 @@ describe("diagnostic log", () => {
       markers: [
         {
           timestamp: "2026-10-05T01:30:00.000Z",
-          description: "蓝莓没出来",
+          description:
+            "蓝莓没出来 /Users/private/query.txt ASIA1234567890ABCDEF GOCSPX-clientSecret_A1 Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8 token=ghp_private_token xoxb-123456-abcdef AKIDExampleSecretId1234567890 AIzaSyExampleCredential1234567890 eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signaturevalue",
         },
       ],
       summary: {
         sources: [{ source_id: "workbuddy", imported_messages: 12 }],
-        memory_tasks: { unknown: 1, last_error_codes: { model_failed: 1 } },
+        summary_secret: conversation,
+        background: {
+          status: "exported",
+          runtime_state: "ready",
+          core_state: "healthy",
+          broker_state: "ready",
+          instance_id: "instance-1",
+          secret: conversation,
+        },
       },
     });
 
-    expect(result.filename).toBe("MemHub.zip");
+    expect(result).toEqual({ filename: "MemHub.zip", file_count: 3 });
     const files = unzipSync(await readFile(target));
-    expect(Object.keys(files)).toEqual(
-      expect.arrayContaining([
-        "README.txt",
-        "summary.json",
-        "logs/diagnostic-core.jsonl",
-        "logs/core.log",
-        "crashes/MemHub-2026-10-05.ips",
-      ]),
-    );
-    const summary = files["summary.json"];
+    expect(Object.keys(files).sort()).toEqual([
+      "README.txt",
+      "diagnostics/events.jsonl",
+      "summary.json",
+    ]);
+    const eventFile = files["diagnostics/events.jsonl"];
+    const summaryFile = files["summary.json"];
     const readme = files["README.txt"];
-    if (!summary || !readme) throw new Error("missing support bundle metadata");
-    expect(strFromU8(summary)).toContain("model_failed");
-    expect(strFromU8(readme)).toContain("蓝莓没出来");
+    if (!eventFile || !summaryFile || !readme)
+      throw new Error("missing support bundle metadata");
+    const exportedEvent = JSON.parse(strFromU8(eventFile).trim());
+    expect(exportedEvent.details).toEqual({ count: 1 });
+    const summary = JSON.parse(strFromU8(summaryFile));
+    expect(summary.background).toEqual({
+      status: "exported",
+      runtime_state: "ready",
+      core_state: "healthy",
+      broker_state: "ready",
+      instance_id: "instance-1",
+    });
+    expect(summary.sources).toBeUndefined();
+    expect(summary.summary_secret).toBeUndefined();
+    expect(summary.problem_markers).toEqual([
+      {
+        timestamp: "2026-10-05T01:30:00.000Z",
+        description: "user_marked_incident",
+      },
+    ]);
+    expect(strFromU8(readme)).toContain("user_marked_incident");
+    expect(strFromU8(readme)).not.toContain("蓝莓没出来");
     const all = Object.values(files)
       .map((value) => strFromU8(value))
       .join("\n");
     expect(all).not.toContain(secret);
+    expect(all).not.toContain("ghp_private_token");
+    expect(all).not.toContain("xoxb-123456-abcdef");
+    expect(all).not.toContain("AKIDExampleSecretId1234567890");
+    expect(all).not.toContain("ASIA1234567890ABCDEF");
+    expect(all).not.toContain("GOCSPX-clientSecret_A1");
+    expect(all).not.toContain("Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8");
+    expect(all).not.toContain("AIzaSyExampleCredential1234567890");
+    expect(all).not.toContain("eyJhbGciOiJIUzI1NiJ9");
     expect(all).not.toContain(conversation);
     expect(all).not.toContain(home);
     expect(all).not.toContain("/Users/");
-    expect(all).toContain("~/MemHub");
+  });
+
+  it("fails closed without publishing a ZIP when a diagnostic line is invalid", async () => {
+    const dataDir = await root();
+    await mkdir(path.join(dataDir, "logs"), { recursive: true });
+    await writeFile(
+      path.join(dataDir, "logs", "diagnostic-core.jsonl"),
+      "not-json\n",
+    );
+    const outputDir = await root();
+    const target = path.join(outputDir, "MemHub.zip");
+    await expect(
+      exportDiagnosticZip({
+        dataDir,
+        target,
+        productVersion: "3800",
+      }),
+    ).rejects.toThrow("diagnostic_export_invalid_lines");
+    expect(await readdir(outputDir)).toEqual([]);
+  });
+
+  it("stores and exports only the fixed marker description", async () => {
+    const dataDir = await root();
+    const markerFile = path.join(dataDir, "logs", "problem-markers.jsonl");
+    await recordDiagnosticMarker(
+      dataDir,
+      {
+        timestamp: "2026-10-05T01:00:00.000Z",
+        description:
+          "蓝莓 /Users/private/query.txt ASIA1234567890ABCDEF GOCSPX-clientSecret_A1 Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+      },
+      "/Users/private",
+    );
+    expect(await readFile(markerFile, "utf8")).toBe(
+      `${JSON.stringify({ timestamp: "2026-10-05T01:00:00.000Z", description: "user_marked_incident" })}\n`,
+    );
+    await writeFile(
+      markerFile,
+      `${await readFile(markerFile, "utf8")}${JSON.stringify({ timestamp: "2026-10-05T01:30:00.000Z", description: "legacy arbitrary text" })}\n`,
+      { mode: 0o600 },
+    );
+
+    const target = path.join(await root(), "markers.zip");
+    await exportDiagnosticZip({
+      dataDir,
+      target,
+      productVersion: "3800",
+      now: new Date("2026-10-05T02:00:00.000Z"),
+    });
+    const files = unzipSync(await readFile(target));
+    const summaryFile = files["summary.json"];
+    if (!summaryFile) throw new Error("missing summary.json");
+    const payload = JSON.stringify(JSON.parse(strFromU8(summaryFile)));
+    expect(payload.match(/user_marked_incident/g)).toHaveLength(2);
+    expect(payload).not.toContain("legacy arbitrary text");
+    expect(payload).not.toContain("蓝莓");
+    expect(payload).not.toContain("/Users/");
+  });
+
+  it("never follows a problem marker symlink for reads or writes", async () => {
+    const dataDir = await root();
+    const logs = path.join(dataDir, "logs");
+    await mkdir(logs, { recursive: true });
+    const outside = path.join(await root(), "outside-marker");
+    await writeFile(outside, "private");
+    await symlink(outside, path.join(logs, "problem-markers.jsonl"));
+
+    await expect(
+      recordDiagnosticMarker(dataDir, {
+        timestamp: "2026-10-05T01:00:00.000Z",
+        description: "anything",
+      }),
+    ).rejects.toThrow("diagnostic_marker_file_unsafe");
+    await expect(
+      exportDiagnosticZip({
+        dataDir,
+        target: path.join(await root(), "markers.zip"),
+        productVersion: "3800",
+      }),
+    ).rejects.toThrow("diagnostic_marker_file_unsafe");
+    expect(await readFile(outside, "utf8")).toBe("private");
+  });
+
+  it("rejects invalid explicit and persisted markers without publishing a ZIP", async () => {
+    const dataDir = await root();
+    const logs = path.join(dataDir, "logs");
+    await mkdir(logs, { recursive: true });
+    const outputDir = await root();
+    const explicitTarget = path.join(outputDir, "explicit.zip");
+    await expect(
+      exportDiagnosticZip({
+        dataDir,
+        target: explicitTarget,
+        productVersion: "3800",
+        markers: [{ timestamp: "invalid", description: "marker" }],
+      }),
+    ).rejects.toThrow("diagnostic_marker_invalid");
+    await writeFile(path.join(logs, "problem-markers.jsonl"), "not-json\n");
+    const persistedTarget = path.join(outputDir, "persisted.zip");
+    await expect(
+      exportDiagnosticZip({
+        dataDir,
+        target: persistedTarget,
+        productVersion: "3800",
+      }),
+    ).rejects.toThrow("diagnostic_marker_invalid");
+    expect(await readdir(outputDir)).toEqual([]);
   });
 
   it("an incident retrieves its surrounding session rather than only its marker", async () => {