Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e2907ed4ea | ||
|
|
48b2540eb5 | ||
|
|
0625b8f414 | ||
|
|
d6a95760e2 | ||
|
|
65171074b2 | ||
|
|
35620812df | ||
|
|
b562a75dd0 | ||
|
|
74608fa9d6 | ||
|
|
5e05af59e0 | ||
|
|
23572c31f9 | ||
|
|
fefd28de5a | ||
|
|
24f6fb6e28 | ||
|
|
872e3710c5 | ||
|
|
db94e24b3f | ||
|
|
04f13cfeed | ||
|
|
e974e656ef | ||
|
|
415db57eb1 | ||
|
|
8c60e5c90a | ||
|
|
cbe4397d14 | ||
|
|
04321a2425 | ||
|
|
2a2a8047ef | ||
|
|
c7fc92c9c7 | ||
|
|
eb98f3c992 | ||
|
|
146413992d | ||
|
|
56f98b1889 | ||
|
|
6f81f3f22f | ||
|
|
3a0752b5b3 | ||
|
|
a218d3c52c | ||
|
|
d07ae91c93 | ||
|
|
1e2c8baa78 | ||
|
|
2163a013a4 | ||
|
|
00d7b41eae | ||
|
|
28d373d798 | ||
|
|
92ab29f95d | ||
|
|
2369bc30dc | ||
|
|
430ddc95e8 | ||
|
|
a720ecdfa8 | ||
|
|
587147e75d | ||
|
|
1630c672c6 | ||
|
|
dbfd6163d4 | ||
|
|
81a8224147 | ||
|
|
f61f3dabb1 | ||
|
|
8fa7263316 | ||
|
|
60902fbfa9 | ||
|
|
220725b254 | ||
|
|
bc07b43f8d | ||
|
|
01a0369055 | ||
|
|
9507dad424 | ||
|
|
9e4e38cb09 | ||
|
|
1ea277b0d0 | ||
|
|
38aaa5997f | ||
|
|
abfd97fdec | ||
|
|
4a6699eda8 | ||
|
|
1048959cba | ||
|
|
db11adff84 | ||
|
|
9b668e29ac | ||
|
|
7e240e769c | ||
|
|
df2aefb922 | ||
|
|
504dded830 | ||
|
|
2289d07715 | ||
|
|
bbbcbae692 | ||
|
|
76e8419e1c | ||
|
|
fbdea89396 | ||
|
|
2e1237f29f | ||
|
|
d96dd1c79e | ||
|
|
71f74e4534 |
No files matched your search
@@ -1,49 +0,0 @@
|
||||
name: Hermes provider
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "plugins/hermes/**"
|
||||
- ".github/workflows/hermes-provider.yml"
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "plugins/hermes/**"
|
||||
- ".github/workflows/hermes-provider.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
provider:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 3
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: astral-sh/setup-uv@d0d8abe699bfb85fec6de9f7adb5ae17292296ff
|
||||
with:
|
||||
python-version: "3.12"
|
||||
enable-cache: false
|
||||
- name: Download and verify native Hermes contract
|
||||
run: |
|
||||
uv run --no-project --python 3.12 python - <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import urllib.request
|
||||
|
||||
url = "https://raw.githubusercontent.com/NousResearch/hermes-agent/818c13be1dc4fd28987e1e881a9408224afd4535/agent/memory_provider.py"
|
||||
with urllib.request.urlopen(url, timeout=30) as response:
|
||||
data = response.read()
|
||||
expected = "a4e44a293013fff831ac136ab77c556541cd18817f53bd662e97090cd80bd5a6"
|
||||
if hashlib.sha256(data).hexdigest() != expected:
|
||||
raise SystemExit("Hermes contract SHA-256 mismatch")
|
||||
snapshot = Path(os.environ["RUNNER_TEMP"]) / "hermes-memory-provider.py"
|
||||
snapshot.write_bytes(data)
|
||||
with open(os.environ["GITHUB_ENV"], "a") as env:
|
||||
env.write(f"HERMES_MEMORY_PROVIDER_SNAPSHOT={snapshot}\n")
|
||||
PY
|
||||
- name: Test provider with native contract
|
||||
run: uv run --no-project --python 3.12 --with pytest python -m pytest plugins/hermes/tests -o addopts= -o log_cli=false -q
|
||||
@@ -26,9 +26,6 @@ package-lock.json
|
||||
ui_walkthrough_*/
|
||||
|
||||
# Internal release/open-source coordination must never enter the public tree.
|
||||
HANDSOFF.md
|
||||
新鲜度_todo.md
|
||||
memhub/docs/LOCAL_PRODUCT_APP_MIGRATION_TEST.md
|
||||
OPEN_SOURCE_TODO.md
|
||||
release_todo.md
|
||||
|
||||
|
||||
+40
-115
@@ -2,110 +2,6 @@
|
||||
|
||||
Hy Memory and MemHub are versioned and released independently.
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Fixed legacy-to-SDK startup migration to adopt native Coding Experience and
|
||||
Working Fact IDs instead of importing them as Chat layers. Raw migration now
|
||||
rebuilds bounded user/assistant turns from the latest three days, excludes
|
||||
tools, and enforces the same rolling retention after cutover.
|
||||
- Paused host capture writes with a live Desktop migration fence; Cursor
|
||||
History catch-up resumes from its existing checkpoint after cutover.
|
||||
- Removed MemHub's private `hy_memory_bridge` implementation. MemHub now calls
|
||||
public `HyMemoryClient` methods at runtime; prompt/reconcile/response
|
||||
monkey-patches and private VDB/Coding runtime access were deleted. Kuzu Graph
|
||||
projection/admin and stopped-writer App migration are the documented product
|
||||
exceptions.
|
||||
- Added a MemHub-managed native Hermes MemoryProvider with automatic prefetch,
|
||||
persistent per-session memory-ID deduplication, authorized Broker reads, and
|
||||
recall-context stripping to prevent memory recursion.
|
||||
- Added an extensible MemHub interface-localization layer with Simplified
|
||||
Chinese, English, and Spanish catalogs shared by Web and Electron-native
|
||||
surfaces. Interface language remains independent from memory content and
|
||||
model processing language.
|
||||
- Fixed governed-memory revisions and repeated evidence links failing when the
|
||||
visibility dirty-marker trigger encounters the same memory twice. Startup
|
||||
now replaces the affected legacy trigger without changing schema versions
|
||||
or governed and visibility data.
|
||||
- Kept post-write authority verification failures available for read-only
|
||||
reconciliation while preventing automatic or generic confirmed retries from
|
||||
invoking the model again. Automatic reconciliation skips those sticky rows
|
||||
before applying its batch limit, and linked durable jobs exclusively own
|
||||
retry scheduling for their memory tasks.
|
||||
- Removed MemHub's task-scoped HyPermission store wrappers and global
|
||||
model-content consent switch. The trusted embedded Hy Memory SDK now uses
|
||||
its native local stores directly.
|
||||
- Existing installations ignore the retired config flag, readmit previously
|
||||
parked pre-invocation work, and drop only the obsolete
|
||||
`hy_node_permissions` proof table. Evidence, governed memories, model
|
||||
profiles, SDK stores, and source/host grants are preserved.
|
||||
- Fixed the applications list going permanently blank when a source's root was
|
||||
chosen by the person rather than detected. The web transport guard listed
|
||||
the root kinds by hand and rejected the whole list over the `user_selected`
|
||||
kind Core already grants; the kinds now come from the shared contract, so the
|
||||
two cannot drift again, and a parity test pins Core's enumeration to it.
|
||||
|
||||
## Hy Memory 2.0.5 — 2026-10-22
|
||||
|
||||
- Published
|
||||
[`hy-memory==2.0.5`](https://pypi.org/project/hy-memory/2.0.5/).
|
||||
- Extended `get()` across Chat, Coding Experience, Working Fact, and raw IDs;
|
||||
the returned `layer`, `kind`, and `variant` identify the actual record.
|
||||
- Added explicit raw retrieval through the existing `search()` and
|
||||
`list_memories()` contracts, including user, agent, session, and source
|
||||
filters.
|
||||
- Added public embedding-service injection and corrected extraction language
|
||||
detection for structured OpenAI-style messages.
|
||||
- Added lifecycle status filters, durable idempotent write outcomes, and
|
||||
quiesced local data-plane snapshot/restore.
|
||||
- Kept legacy migration mechanics out of the public Client API; separate
|
||||
migration-only raw/import methods from development previews were removed.
|
||||
- Physical delete now reports partial VDB/Graph cleanup as failure.
|
||||
- Coding search no longer injects the version-evolution diff block by default;
|
||||
opt in with `MEMORY_CODING_CHAIN_BLOCK_ENABLED=true`.
|
||||
- Serialized SDK SQLite cache/history connection lifecycle and tightened
|
||||
Coding reconciliation to its agent/session/workspace authority.
|
||||
- Clarified persistent caller context through `meta`, while `metadata` remains
|
||||
legacy processing configuration.
|
||||
|
||||
## Hy Memory 2.0.5.dev4 — 2026-10-21
|
||||
|
||||
- Development preview used for SDK-authority migration validation; not the
|
||||
final `2.0.5` release.
|
||||
- Published as
|
||||
[`hy-memory==2.0.5.dev4`](https://pypi.org/project/hy-memory/2.0.5.dev4/).
|
||||
|
||||
## Hy Memory 2.0.5.dev3 — 2026-10-21
|
||||
|
||||
- Development preview; not the final `2.0.5` release.
|
||||
- Published as
|
||||
[`hy-memory==2.0.5.dev3`](https://pypi.org/project/hy-memory/2.0.5.dev3/).
|
||||
- Lets the existing `search` API explicitly query SDK raw with
|
||||
`layers=["l1_raw"]`, including existing source filters.
|
||||
|
||||
## Hy Memory 2.0.5.dev2 — 2026-10-21
|
||||
|
||||
- Development preview; not the final `2.0.5` release.
|
||||
- Published as
|
||||
[`hy-memory==2.0.5.dev2`](https://pypi.org/project/hy-memory/2.0.5.dev2/).
|
||||
- Adds exact, idempotent semantic-memory import for legacy MemHub cutover
|
||||
without extraction, reconciliation or identity changes.
|
||||
|
||||
## Hy Memory 2.0.5.dev1 — 2026-10-21
|
||||
|
||||
- Development preview for the MemHub SDK-authority migration; this is not the
|
||||
final `2.0.5` release.
|
||||
- Published as
|
||||
[`hy-memory==2.0.5.dev1`](https://pypi.org/project/hy-memory/2.0.5.dev1/).
|
||||
- Adds lifecycle status filtering, durable idempotent write outcomes, exact raw
|
||||
APIs, offline local data-plane snapshot/restore, and fail-closed physical
|
||||
delete receipts.
|
||||
|
||||
## Hy Memory 2.0.4 — 2026-10-17
|
||||
|
||||
- Published
|
||||
[`hy-memory==2.0.4`](https://pypi.org/project/hy-memory/2.0.4/) as a
|
||||
maintenance release.
|
||||
|
||||
## Hy Memory 2.0.3 — 2026-10-17
|
||||
|
||||
- Added an optional runtime HTTP transport factory so embedded products can
|
||||
@@ -116,6 +12,45 @@ Hy Memory and MemHub are versioned and released independently.
|
||||
unchanged memory extraction, reconciliation, storage, and Coding/Working
|
||||
behavior.
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Added SDK-authoritative lifecycle listing with explicit status filters,
|
||||
idempotent durable write outcomes, exact raw import/get/list APIs, and
|
||||
quiesced local data-plane snapshot/restore contracts for the MemHub
|
||||
authority migration.
|
||||
- Physical delete now reports incomplete VDB/Graph cleanup as a partial failure
|
||||
instead of claiming success after only one store was changed.
|
||||
- Coding search no longer injects the version-evolution diff block
|
||||
(`chain_block`) by default. Enable with
|
||||
`MEMORY_CODING_CHAIN_BLOCK_ENABLED=true` or
|
||||
`CodingConfig.chain_block_enabled=True` (write-side observe unchanged).
|
||||
- Fixed governed-memory revisions and repeated evidence links failing when the
|
||||
visibility dirty-marker trigger encounters the same memory twice. Startup
|
||||
now replaces the affected legacy trigger without changing schema versions
|
||||
or governed and visibility data.
|
||||
- Serialized each SDK SQLite store's shared connection lifecycle so concurrent
|
||||
history/cache operations cannot race with `close()` and crash the process.
|
||||
- Scoped Coding reconciliation targets to the current HY authority,
|
||||
conversation, and workspace, preventing near-duplicate sessions from
|
||||
mutating an experience or working fact owned by another evidence context.
|
||||
- Kept post-write authority verification failures available for read-only
|
||||
reconciliation while preventing automatic or generic confirmed retries from
|
||||
invoking the model again. Automatic reconciliation skips those sticky rows
|
||||
before applying its batch limit, and linked durable jobs exclusively own
|
||||
retry scheduling for their memory tasks.
|
||||
- Clarified that `add(metadata=...)` carries legacy SDK processing options,
|
||||
while persistent caller business context belongs in `meta`.
|
||||
- Added an opt-in `get(..., include_raw=True)` path for reading `L1_RAW`
|
||||
memories, including memories written in lite mode. The default remains
|
||||
hidden for backward compatibility.
|
||||
- Removed MemHub's task-scoped HyPermission store wrappers and global
|
||||
model-content consent switch. The trusted embedded Hy Memory SDK now uses
|
||||
its native local stores directly.
|
||||
- Existing installations ignore the retired config flag, readmit previously
|
||||
parked pre-invocation work, and drop only the obsolete
|
||||
`hy_node_permissions` proof table. Evidence, governed memories, model
|
||||
profiles, SDK stores, and source/host grants are preserved.
|
||||
|
||||
## Hy Memory 2.0.2 — 2026-10-17
|
||||
|
||||
- Published
|
||||
@@ -130,17 +65,7 @@ Hy Memory and MemHub are versioned and released independently.
|
||||
Because PyPI release metadata is immutable, the next SDK patch release must
|
||||
carry the updated license metadata; `2.0.1` retains its published metadata.
|
||||
|
||||
## MemHub 1.0.5 — Unreleased
|
||||
|
||||
- Updated the source version to `1.0.5` (build 14).
|
||||
- Pinned the public SDK to the stable `hy-memory==2.0.5` release.
|
||||
|
||||
## MemHub 1.0.4 — 2026-10-18
|
||||
|
||||
- Updated the source version to `1.0.4` (build 13).
|
||||
- Published the `v1.0.4` source and release metadata.
|
||||
|
||||
## MemHub 1.0.3 — 2026-10-17
|
||||
## MemHub 1.0.3 — Unreleased
|
||||
|
||||
- Updated the source version to `1.0.3` (build 7).
|
||||
- Pinned the public SDK to `hy-memory==2.0.3`.
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
# HANDSOFF — MemHub / Hy Memory 架构改造
|
||||
|
||||
更新:2026-10-21
|
||||
权威细则:[`memhub/docs/ARCHITECTURE.md`](memhub/docs/ARCHITECTURE.md)(~~旧删线~~ + 新并列)
|
||||
配套:[`memhub/MEMORY.md`](memhub/MEMORY.md);其余 docs 已打「架构变更」条,冲突以 ARCHITECTURE Top Rules 为准。
|
||||
旧用户迁移:[`memhub/docs/contracts/sdk-data-plane-migration-v1.md`](memhub/docs/contracts/sdk-data-plane-migration-v1.md)。
|
||||
|
||||
> 根目录原先无有效 `HANDSOFF` 文件;本文件为清空后重写,只保留本次架构改造交接。
|
||||
|
||||
---
|
||||
|
||||
## IMPORTANT:测试执行规则
|
||||
|
||||
1. 先完整运行目标测试套件一次,收集**全部失败 case**。
|
||||
2. 按根因分组并批量修复,不得用 `-x` 反复重跑已经通过的前置测试。
|
||||
3. 修复后只重跑上次失败的 case。
|
||||
4. 失败 case 全部通过后,才再完整运行一次目标套件作为最终 gate。
|
||||
|
||||
## Top Rules(现行根本)
|
||||
|
||||
1. **所有对 memory 的读写(含存储 backend)都交给 SDK**,经 SDK config / env 配置。
|
||||
2. **MemHub 内对 memory 的原子操作也一律经 SDK**;缺接口 → **先补 SDK client**,禁止 Core 直改 VDB/Graph/SDK 私库或再维护正文权威。
|
||||
3. **MemHub 不存 memory 实体 / 引擎侧关系**(正文、supersede 权威、向量图副本等)。主责是管理面:harness、LLM model、异步写入队列、config、Core/Runtime 状态。
|
||||
- **可以且应当存 `memory_id`**(SDK 给外层的唯一身份)及组织元数据。
|
||||
- Matter 占位功能未发布,删除其表与入口;未来若重做,必须另立合同。
|
||||
|
||||
**推论:**
|
||||
|
||||
| 旧 | 新 |
|
||||
|---|---|
|
||||
| MemHub `evidence*` 原文权威 | 原文 → **SDK raw**;Evidence 权威拆除 |
|
||||
| `governed_*` 列表/版本/supersede 权威 | 拆除方向;UI/纠正/删除 → **授权 + SDK API**;组织层只挂 `memory_id` |
|
||||
| SDK success 后再镜像进 Governance 才算产品写入 | SDK 合同为准;MemHub 至多记 `memory_id` / 任务回执 |
|
||||
| 拦 CRUD / parse ops 双写 supersede | 禁止作为目标架构 |
|
||||
|
||||
---
|
||||
|
||||
## 已改文档(合同先于代码)
|
||||
|
||||
- 重写:`memhub/docs/ARCHITECTURE.md`、`memhub/MEMORY.md`
|
||||
- 条幅 + 关键段新旧对照:`contracts/*`(含 runtime-unification / unified-context-read / source-adapter 等)、`INSTALLATION.md`、`DEVELOPMENT_LOOP.md`、`MEMORY_EVENT_TIME_SCHEDULING_PLAN.md`、`PRODUCT_LOG_TROUBLESHOOTING.md`、`harness-support/README.md`
|
||||
- 各 harness 专页未逐段改写 → 冲突时以 ARCHITECTURE 为准
|
||||
|
||||
**代码尚未按此拆除。** 文档是目标架构,不是已完成迁移。
|
||||
|
||||
---
|
||||
|
||||
## 建议实现顺序
|
||||
|
||||
1. 冻结新功能写入 `governed_*` / MemHub `evidence*` 权威路径。
|
||||
2. 记忆库 UI / recall:授权 + SDK list/get/search/update/delete;组织层只挂 `memory_id`。
|
||||
3. 导入/同步:写入 SDK(含 raw);退役 Evidence 权威。
|
||||
4. 队列降级为「调度 SDK 调用」,完成态以 SDK 可验证结果为准。
|
||||
5. 清空/删除镜像台账;缺的能力开 **SDK PR**,不在 Core 补影子表。
|
||||
|
||||
旧用户的数据迁移不是独立的手工升级步骤:由 App 在可见的启动 loading
|
||||
阶段、Core 开始提供产品读写之前,完成 bundle、可续跑 backfill、reconciliation
|
||||
与最终 cutover。失败或缺少前置配置时 marker 保持未提交,不允许静默进入双写。
|
||||
|
||||
Fresh install 直接创建最新管理面 schema,禁止创建 `evidence*`、`governed_*`
|
||||
及旧 retrieval/Matter/session-shadow 表。Legacy upgrade 在 bundle 与 SDK
|
||||
迁移校验完成后物理 DROP 这些旧表;备份 bundle 才是审计/回滚载体,不在 live DB
|
||||
保留一套“兼容权威”。
|
||||
|
||||
---
|
||||
|
||||
## 管理面仍属 MemHub(勿误拆)
|
||||
|
||||
- harness:`source_grants`、sync desired-state(`config.json`)、适配器扫描(产出进 SDK,不是 Evidence 权威库)
|
||||
- model profiles / 密钥边界 / 模型内容许可
|
||||
- `memory_write_jobs` 单层异步队列、FIFO / event-time 调度与租约(始终属于 MemHub;调度,不是 memory 实体)。旧 `memory_tasks` 二级队列与 task receipt 删除。
|
||||
- pending job 尚未调用 SDK,因此没有 raw/memory ID;worker 对 SDK 只调用一次
|
||||
`add()`,SDK 内部完成 raw + extraction/reconciliation,成功后 MemHub 仅保存
|
||||
返回的 raw ID / memory IDs 与 outcome。禁止“先 import_raw 再排 extraction”。
|
||||
- Core / Runtime 收据与健康
|
||||
- Matter 尚未形成已发布产品合同;删除全部 `matter*` 占位表与运行时入口。未来若重新开发,
|
||||
以新合同重新设计,不为当前占位 schema 保留兼容层。
|
||||
- call_events 类审计元数据(不存记忆正文)
|
||||
- ~~session inbox / change_outbox 围绕 Evidence 外发~~ → 需按新规则重定义或标旧
|
||||
|
||||
---
|
||||
|
||||
## 接手时注意
|
||||
|
||||
- Pin 的公开 `hy-memory` 与「先补 SDK 接口」可能不同步:合源码 ≠ 用户包已带能力;用 `Refs` / pin PR 闭环。
|
||||
- Top Rule 与旧 `ARCHITECTURE`「SQLite 含 Governance 为业务权威」冲突处,**以本 HANDSOFF + 新 ARCHITECTURE 文首为准**。
|
||||
- 本地 `pr_principle.md` / `pr_review.md` 是 PR 流程台账,不替代本架构 HANDSOFF。
|
||||
@@ -253,7 +253,6 @@ That version is usually not in the working tree or at HEAD either, so it cannot
|
||||
- [Graph memory design](docs/pro_graph_design.md) — graph projection, rerank, relation paths, and deployment limits
|
||||
- [Environment template](.env.example) — a copy-ready configuration template
|
||||
- [MemHub README](memhub/README.md) — local-first desktop & CLI product overview, runtime commands, and privacy boundaries
|
||||
- [Hermes native memory provider](plugins/hermes/README.md) — explicit owner/agent scope; separate HTTP service; automatic capture and recall opt-in.
|
||||
- [OpenClaw plugin](plugins/openclaw/README.md) — agent integrations live under `plugins/` (overview: [plugins/README.md](plugins/README.md))
|
||||
|
||||
### License
|
||||
|
||||
@@ -253,7 +253,6 @@ Coding 与 Working **复用同一套双层 pipeline 和数据 schema**,区别
|
||||
- [Graph 记忆设计](docs/pro_graph_design.md) — 图投影、重排、关系路径与部署限制
|
||||
- [环境变量模板](.env.example) — 可直接复制的配置模板
|
||||
- [MemHub 说明文档](memhub/README.md) — 本地优先桌面端与 CLI 产品总览、Runtime 命令与隐私边界
|
||||
- [Hermes 原生记忆 Provider](plugins/hermes/README.md) — 显式用户与 Agent 隔离,独立 HTTP 服务,自动记录与召回默认关闭。
|
||||
- [OpenClaw 插件](plugins/openclaw/README.md) — Agent 集成见 `plugins/`(总览:[plugins/README.md](plugins/README.md))
|
||||
|
||||
### License
|
||||
|
||||
+6
-7
@@ -273,8 +273,8 @@ catalog 也可以在没有基础 L0 时形成只读的合成画像。
|
||||
|
||||
```python
|
||||
memory = client.get("memory_id")
|
||||
# raw ID 返回 L1_RAW,fact ID 返回对应事实层。
|
||||
raw_memory = client.get("raw_id")
|
||||
# lite 模式写入的是 L1_RAW;按 ID 读取时必须显式选择返回原文。
|
||||
raw_memory = client.get("memory_id", include_raw=True)
|
||||
result = client.update("memory_id", "更新后的内容")
|
||||
result = client.delete("memory_id")
|
||||
result = client.delete_all(user_id="user_001")
|
||||
@@ -284,7 +284,7 @@ page = client.list_memories(
|
||||
limit=100,
|
||||
offset=0,
|
||||
order="desc",
|
||||
layers=["l1_raw"], # 显式列出 raw;不传仍沿用默认语义层列表
|
||||
layers=None,
|
||||
with_evolution=False,
|
||||
)
|
||||
|
||||
@@ -294,10 +294,9 @@ result = client.clone_user("user_001", "user_002", agent_id=None)
|
||||
对应异步方法为 `async_get()`、`async_update()`、`async_delete()`、
|
||||
`async_delete_all()`、`async_list_memories()`、`async_clone_user()`。
|
||||
|
||||
`get()` / `async_get()` 是统一的 ID 精确读取:返回结果的 `layer`、`kind` 和
|
||||
`variant` 表明实际类型。授权由调用 SDK 前的产品边界完成,不由 SDK 根据 layer
|
||||
猜测。`list_memories()` 默认仍不包含 `L1_RAW`;显式传
|
||||
`layers=["l1_raw"]` 时按同一列表合同返回 raw。
|
||||
`get()` / `async_get()` 默认继续隐藏 `L1_RAW`,因此不会改变现有调用方的原文访问
|
||||
边界。只有显式传入 `include_raw=True` 才返回该层;这也是 lite 模式按 ID 读取
|
||||
写入内容及其 `meta` 的方式。MemHub 的 HTTP 读取接口不开放此参数。
|
||||
|
||||
普通 `list_memories()` 返回 `vdb` 桶,并在 Graph 有结果时附带 `graph`;传入
|
||||
`workspace_id` 时返回生产力记忆的 `scene="productivity"` 和 `experiences`。
|
||||
|
||||
@@ -65,12 +65,6 @@ from .snapshot import (
|
||||
|
||||
# ====== 配置 ======
|
||||
from .config import MemoryConfig
|
||||
from .embedding import (
|
||||
EmbeddingDimensionMismatchError,
|
||||
EmbeddingService,
|
||||
EmbeddingServiceProtocol,
|
||||
)
|
||||
from .retrieval import AmbiguousMemoryIdError, MemoryKind, MemoryVariant
|
||||
|
||||
# ====== Embedding 端点覆盖(per-call,不改 model/dims,只换接入端点)======
|
||||
from .core.embed_service import embed_endpoint_override
|
||||
@@ -131,13 +125,6 @@ __all__ = [
|
||||
"snapshot_local_data_plane",
|
||||
# 配置
|
||||
"MemoryConfig",
|
||||
# 公共扩展协议 / 精确读取类型
|
||||
"EmbeddingService",
|
||||
"EmbeddingServiceProtocol",
|
||||
"EmbeddingDimensionMismatchError",
|
||||
"MemoryKind",
|
||||
"MemoryVariant",
|
||||
"AmbiguousMemoryIdError",
|
||||
# Embedding 端点覆盖
|
||||
"embed_endpoint_override",
|
||||
# 高级 API
|
||||
|
||||
+252
-346
@@ -38,8 +38,6 @@ from typing import List, Dict, Any, Optional, Tuple, Union, TYPE_CHECKING
|
||||
|
||||
from .config import MemoryConfig
|
||||
from .core.embed_service import EmbedService, embed_endpoint_override
|
||||
from .embedding import EmbeddingDimensionMismatchError, EmbeddingService
|
||||
from .retrieval import AmbiguousMemoryIdError, MemoryKind, MemoryVariant
|
||||
from .data.vector_store import create_vector_store
|
||||
from .data.history_store import HistoryStore
|
||||
from .data.cache import create_cache
|
||||
@@ -436,8 +434,6 @@ class HyMemoryClient:
|
||||
Args:
|
||||
config: 自定义 MemoryConfig(默认从环境变量加载)
|
||||
mode: 处理模式 ("lite" / "pro" / "ultra"),默认从环境变量 MEMORY_MODE 读取
|
||||
embedding_service: 可选的调用方自管 embedding service;须满足公开的
|
||||
EmbeddingService Protocol,client 不接管其生命周期。
|
||||
"""
|
||||
|
||||
# 合法的 mode 值
|
||||
@@ -449,15 +445,12 @@ class HyMemoryClient:
|
||||
mode: Optional[str] = None,
|
||||
coding_writer: Optional[str] = None,
|
||||
runtime: Optional["SharedRuntime"] = None,
|
||||
embedding_service: Optional[EmbeddingService] = None,
|
||||
):
|
||||
# 确保日志系统已配置(幂等,多次调用安全)
|
||||
setup_logging()
|
||||
|
||||
# 配置
|
||||
self._config = config or MemoryConfig.from_env()
|
||||
self._borrowed_embedding_service = embedding_service
|
||||
self._validate_embedding_service(embedding_service)
|
||||
|
||||
# coding_writer 显式传入则覆盖 config / env(默认 None → 沿用 config.coding.writer)
|
||||
# 取值: "legacy" | "agent"
|
||||
@@ -544,12 +537,8 @@ class HyMemoryClient:
|
||||
)
|
||||
logger.info(f"[init] Default ThreadPoolExecutor max_workers={_max_workers}")
|
||||
|
||||
# 1. EmbedService(注入发生在 ComponentFactory / pipeline 预热之前)
|
||||
self._embed_service = (
|
||||
self._borrowed_embedding_service
|
||||
if self._borrowed_embedding_service is not None
|
||||
else EmbedService(self._config)
|
||||
)
|
||||
# 1. EmbedService(纯配置,首次调用时真正连接 API)
|
||||
self._embed_service = EmbedService(self._config)
|
||||
logger.debug(
|
||||
f"[init] EmbedService created: model={self._config.embedder.model}"
|
||||
)
|
||||
@@ -579,10 +568,8 @@ class HyMemoryClient:
|
||||
)
|
||||
|
||||
# 4. ComponentFactory(注入已初始化的共享组件)
|
||||
self._registry = ComponentFactory(
|
||||
config=self._config,
|
||||
embedding_service=self._embed_service,
|
||||
)
|
||||
self._registry = ComponentFactory(config=self._config)
|
||||
self._registry._shared_embed_service = self._embed_service
|
||||
self._registry._shared_vector_store = self._vector_store
|
||||
self._registry._shared_vector_store_initialized = True
|
||||
if self._graph_store is not None:
|
||||
@@ -716,7 +703,6 @@ class HyMemoryClient:
|
||||
cls,
|
||||
config_dict: Dict[str, Any],
|
||||
mode: Optional[str] = None,
|
||||
embedding_service: Optional[EmbeddingService] = None,
|
||||
) -> "HyMemoryClient":
|
||||
"""
|
||||
从字典创建客户端(类似 mem0 风格)。
|
||||
@@ -748,49 +734,7 @@ class HyMemoryClient:
|
||||
"""
|
||||
config = MemoryConfig.from_dict(config_dict)
|
||||
resolved_mode = mode or config_dict.get("mode")
|
||||
return cls(
|
||||
config=config,
|
||||
mode=resolved_mode,
|
||||
embedding_service=embedding_service,
|
||||
)
|
||||
|
||||
def _validate_embedding_service(
|
||||
self, embedding_service: Optional[EmbeddingService]
|
||||
) -> None:
|
||||
"""Validate a borrowed service before any store or pipeline is created."""
|
||||
if embedding_service is None:
|
||||
return
|
||||
required = (
|
||||
"embed",
|
||||
"embed_with_meta",
|
||||
"embed_batch",
|
||||
"embed_queued",
|
||||
"embed_queued_with_meta",
|
||||
"get_embedding_dims",
|
||||
)
|
||||
missing = [
|
||||
name for name in required
|
||||
if not callable(getattr(embedding_service, name, None))
|
||||
]
|
||||
if missing:
|
||||
raise TypeError(
|
||||
"embedding_service does not implement EmbeddingService; "
|
||||
f"missing: {', '.join(missing)}"
|
||||
)
|
||||
try:
|
||||
actual_dims = int(embedding_service.get_embedding_dims())
|
||||
except Exception as exc:
|
||||
raise TypeError(
|
||||
"embedding_service.get_embedding_dims() must return an integer"
|
||||
) from exc
|
||||
expected_dims = getattr(self._config.vector_store, "embedding_dims", None)
|
||||
if expected_dims is None:
|
||||
expected_dims = getattr(self._config.embedder, "embedding_dims", None)
|
||||
if expected_dims is not None and actual_dims != int(expected_dims):
|
||||
raise EmbeddingDimensionMismatchError(
|
||||
"embedding service dimension mismatch: "
|
||||
f"service={actual_dims}, vector_store={int(expected_dims)}"
|
||||
)
|
||||
return cls(config=config, mode=resolved_mode)
|
||||
|
||||
# ================================================================
|
||||
# 属性
|
||||
@@ -1116,7 +1060,6 @@ class HyMemoryClient:
|
||||
llm_provider=self._coding_llm_provider,
|
||||
embed_service=self._embed_service,
|
||||
cache=self._cache,
|
||||
variant=variant,
|
||||
)
|
||||
self._coding_stores[variant] = store
|
||||
self._coding_writers[variant] = writer
|
||||
@@ -1886,26 +1829,21 @@ class HyMemoryClient:
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: Optional[MemoryKind] = None,
|
||||
variant: Optional[MemoryVariant] = None,
|
||||
include_raw: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
获取单条记忆。
|
||||
|
||||
Args:
|
||||
memory_id: 记忆 ID
|
||||
kind: 可选 namespace("chat" / "experience" / "fact")。
|
||||
variant: 可选生产力变体("working" / "coding")。
|
||||
include_raw: 是否允许返回 L1_RAW 原始记忆。默认 False,保持原有隐藏行为;
|
||||
lite 模式的记忆均为 L1_RAW,需显式设为 True 才能按 ID 读取。
|
||||
|
||||
Returns:
|
||||
带 kind/variant 的统一 memory dict,或 None(不存在时)。
|
||||
{"memory_id", "content", "layer", "request_id", "elapsed_ms"} 或 None(不存在时)
|
||||
"""
|
||||
return self._loop_thread.run(
|
||||
self.async_get(
|
||||
memory_id,
|
||||
kind=kind,
|
||||
variant=variant,
|
||||
)
|
||||
self.async_get(memory_id, include_raw=include_raw)
|
||||
)
|
||||
|
||||
def update(
|
||||
@@ -1974,9 +1912,6 @@ class HyMemoryClient:
|
||||
branch: Optional[str] = None,
|
||||
user_ids: Optional[List[str]] = None,
|
||||
agent_ids: Optional[List[str]] = None,
|
||||
session_ids: Optional[List[str]] = None,
|
||||
include_sources: Optional[List[str]] = None,
|
||||
exclude_sources: Optional[List[str]] = None,
|
||||
layers: Optional[List[Any]] = None,
|
||||
statuses: Optional[List[Any]] = None,
|
||||
with_evolution: bool = False,
|
||||
@@ -1999,9 +1934,6 @@ class HyMemoryClient:
|
||||
user_ids: 可选,多用户批量列举。非空时优先于 user_id,返回并集
|
||||
(分页在合并排序后统一施加)。
|
||||
agent_ids: 可选,多 agent 批量列举。非空时优先于 agent_id。
|
||||
session_ids: 可选,只返回指定 session。
|
||||
include_sources: 可选,只返回指定来源。
|
||||
exclude_sources: 可选,排除指定来源;与 include_sources 冲突时排除优先。
|
||||
layers: 可选,只返回指定层级(字符串或 MemoryLayer)。不传时返回
|
||||
L1_RAW 以外的全部层级,与改动前一致。
|
||||
statuses: 可选,只返回指定生命周期(字符串或 MemoryStatus)。不传时
|
||||
@@ -2017,9 +1949,6 @@ class HyMemoryClient:
|
||||
self.async_list_memories(
|
||||
user_id=user_id, agent_id=agent_id,
|
||||
user_ids=user_ids, agent_ids=agent_ids,
|
||||
session_ids=session_ids,
|
||||
include_sources=include_sources,
|
||||
exclude_sources=exclude_sources,
|
||||
limit=limit, offset=offset, order=order,
|
||||
workspace_id=workspace_id, branch=branch,
|
||||
layers=layers, statuses=statuses, with_evolution=with_evolution,
|
||||
@@ -2808,8 +2737,6 @@ class HyMemoryClient:
|
||||
request_id=request_id,
|
||||
source=source,
|
||||
)
|
||||
# Public receipt always identifies the productivity namespace.
|
||||
resp["variant"] = scene_variant
|
||||
# 版本演进链层(scene="coding" 独有)。放在 writer.write 之后、返回
|
||||
# 之前,因为它只依赖对话原文,与向量层写入结果无关。
|
||||
# best-effort:这层挂不上不影响本次 add 的向量层产出。
|
||||
@@ -2980,6 +2907,63 @@ class HyMemoryClient:
|
||||
include_runtime_diag=True,
|
||||
)
|
||||
|
||||
def import_raw(
|
||||
self,
|
||||
content: str,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str = "default_agent",
|
||||
session_id: str = "default_session",
|
||||
source: str = "default",
|
||||
memory_at: Optional[datetime] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
raw_id: Optional[str] = None,
|
||||
request_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Persist one exact raw record without extraction or reconciliation."""
|
||||
return self._loop_thread.run(
|
||||
self.async_import_raw(
|
||||
content,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
memory_at=memory_at,
|
||||
metadata=metadata,
|
||||
raw_id=raw_id,
|
||||
request_id=request_id,
|
||||
),
|
||||
include_runtime_diag=True,
|
||||
)
|
||||
|
||||
def get_raw(self, raw_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Read one SDK raw record through the explicit raw API."""
|
||||
return self._loop_thread.run(self.async_get_raw(raw_id))
|
||||
|
||||
def list_raw(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: Optional[str] = None,
|
||||
session_id: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
order: str = "desc",
|
||||
) -> Dict[str, Any]:
|
||||
"""List raw records for an authorized SDK owner/scope."""
|
||||
return self._loop_thread.run(
|
||||
self.async_list_raw(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
order=order,
|
||||
)
|
||||
)
|
||||
|
||||
@_ensure_internal_loop
|
||||
async def async_add_extracted(
|
||||
self,
|
||||
@@ -3132,64 +3116,163 @@ class HyMemoryClient:
|
||||
}
|
||||
|
||||
@_ensure_internal_loop
|
||||
async def _search_explicit_raw_layer(
|
||||
async def async_import_raw(
|
||||
self,
|
||||
query: str,
|
||||
content: str,
|
||||
*,
|
||||
user_ids: List[str],
|
||||
agent_ids: List[str],
|
||||
session_ids: List[str],
|
||||
include_sources: List[str],
|
||||
exclude_sources: List[str],
|
||||
limit: int,
|
||||
min_score: float,
|
||||
created_after: Optional[float],
|
||||
user_id: str,
|
||||
agent_id: str = "default_agent",
|
||||
session_id: str = "default_session",
|
||||
source: str = "default",
|
||||
memory_at: Optional[datetime] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
raw_id: Optional[str] = None,
|
||||
request_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Persist exact source content as L1_RAW without running a model."""
|
||||
from .utils.source_filter import normalize_source
|
||||
|
||||
if not isinstance(content, str) or not content:
|
||||
raise ValueError("content is required")
|
||||
if not user_id:
|
||||
raise ValueError("user_id is required")
|
||||
request_id = request_id or str(uuid.uuid4())
|
||||
replay = await self._claim_write_outcome(
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
operation="import_raw",
|
||||
)
|
||||
if replay is not None:
|
||||
return replay
|
||||
try:
|
||||
result = await self._async_import_raw_impl(
|
||||
content,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=normalize_source(source),
|
||||
memory_at=memory_at,
|
||||
metadata=metadata,
|
||||
raw_id=raw_id,
|
||||
request_id=request_id,
|
||||
)
|
||||
except BaseException:
|
||||
await self._mark_write_outcome_unknown(request_id)
|
||||
raise
|
||||
await self._finish_write_outcome(request_id, "succeeded", result)
|
||||
return result
|
||||
|
||||
async def _async_import_raw_impl(
|
||||
self,
|
||||
content: str,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
session_id: str,
|
||||
source: str,
|
||||
memory_at: Optional[datetime],
|
||||
metadata: Optional[Dict[str, Any]],
|
||||
raw_id: Optional[str],
|
||||
request_id: str,
|
||||
t0: float,
|
||||
query_embedding: Optional[List[float]],
|
||||
) -> Dict[str, Any]:
|
||||
from .models.memory import MemoryLayer, MemoryNode, MemoryStatus, SourceType
|
||||
|
||||
node_id = raw_id or str(uuid.uuid4())
|
||||
existing = await self._vector_store.get_by_id(node_id)
|
||||
if existing is not None:
|
||||
identity = (
|
||||
existing.layer == MemoryLayer.L1_RAW,
|
||||
existing.content == content,
|
||||
existing.user_id == user_id,
|
||||
existing.agent_id == agent_id,
|
||||
existing.session_id == session_id,
|
||||
existing.source == source,
|
||||
)
|
||||
if not all(identity):
|
||||
raise ValueError("raw_id is already bound to different source content")
|
||||
return {
|
||||
"success": True,
|
||||
"raw_id": node_id,
|
||||
"request_id": request_id,
|
||||
"replayed": True,
|
||||
}
|
||||
|
||||
custom = dict(metadata or {})
|
||||
custom["request_id"] = request_id
|
||||
custom["input_kind"] = "raw_exact_import"
|
||||
embedding = await self._embed_service.embed(content)
|
||||
node = MemoryNode(
|
||||
node_id=node_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
layer=MemoryLayer.L1_RAW,
|
||||
content=content,
|
||||
source_type=SourceType.EXPLICIT,
|
||||
status=MemoryStatus.ACTIVE,
|
||||
is_latest=True,
|
||||
embedding=embedding,
|
||||
memory_at=memory_at,
|
||||
custom=custom,
|
||||
)
|
||||
stored_id = await self._vector_store.upsert(node)
|
||||
if stored_id != node_id:
|
||||
raise RuntimeError("raw import changed the requested identity")
|
||||
return {"success": True, "raw_id": node_id, "request_id": request_id}
|
||||
|
||||
@_ensure_internal_loop
|
||||
async def async_get_raw(self, raw_id: str) -> Optional[Dict[str, Any]]:
|
||||
from .models.memory import MemoryLayer
|
||||
|
||||
node = await self._vector_store.get_by_id(raw_id)
|
||||
if node is None or node.layer != MemoryLayer.L1_RAW:
|
||||
return None
|
||||
return self._raw_node_to_public(node)
|
||||
|
||||
@_ensure_internal_loop
|
||||
async def async_list_raw(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: Optional[str] = None,
|
||||
session_id: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
order: str = "desc",
|
||||
) -> Dict[str, Any]:
|
||||
"""Implement `search(..., layers=['l1_raw'])` without a second API."""
|
||||
from .models.memory import MemoryLayer, MemoryStatus
|
||||
|
||||
embedding = query_embedding or await self._embed_service.embed(query)
|
||||
fetch = min(max(limit * (10 if session_ids else 2), limit), 10_000)
|
||||
hits = await self._vector_store.search(
|
||||
query_embedding=embedding,
|
||||
user_ids=user_ids,
|
||||
agent_ids=agent_ids,
|
||||
include_sources=include_sources,
|
||||
exclude_sources=exclude_sources,
|
||||
layers=[MemoryLayer.L1_RAW],
|
||||
limit=fetch,
|
||||
score_threshold=min_score,
|
||||
if not user_id:
|
||||
raise ValueError("user_id is required")
|
||||
if limit < 1 or offset < 0:
|
||||
raise ValueError("limit must be positive and offset must be non-negative")
|
||||
nodes = await self._vector_store.list_by_user(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
status_filter=[MemoryStatus.ACTIVE],
|
||||
layers=[MemoryLayer.L1_RAW],
|
||||
include_sources=[source] if source else None,
|
||||
limit=100_000,
|
||||
only_latest=False,
|
||||
created_after=created_after,
|
||||
with_vectors=False,
|
||||
allow_raw_layer=True,
|
||||
)
|
||||
sessions = set(session_ids)
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for hit in hits:
|
||||
node = hit.get("node")
|
||||
if node is None or node.layer != MemoryLayer.L1_RAW:
|
||||
continue
|
||||
if sessions and node.session_id not in sessions:
|
||||
continue
|
||||
row = self._raw_node_to_public(node)
|
||||
rows.append(
|
||||
{
|
||||
"memory_id": row.pop("raw_id"),
|
||||
**row,
|
||||
"layer": MemoryLayer.L1_RAW.value,
|
||||
"status": MemoryStatus.ACTIVE.value,
|
||||
"score": round(float(hit.get("score", 0.0)), 4),
|
||||
}
|
||||
)
|
||||
if len(rows) >= limit:
|
||||
break
|
||||
if session_id:
|
||||
nodes = [node for node in nodes if node.session_id == session_id]
|
||||
nodes = self._sort_memory_nodes(nodes, order=order)
|
||||
total = len(nodes)
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"memories": {"schema": [], "proactive": [], "normal": rows},
|
||||
"elapsed_ms": round((time.perf_counter() - t0) * 1000, 2),
|
||||
"raw": [
|
||||
self._raw_node_to_public(node)
|
||||
for node in nodes[offset: offset + limit]
|
||||
],
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -3852,22 +3935,6 @@ class HyMemoryClient:
|
||||
effective_search_layers = [MemoryLayer.L1_RAW] if lite_mode else layers
|
||||
effective_schema_limit = 0 if lite_mode else schema_limit
|
||||
effective_intention_limit = 0 if lite_mode else intention_limit
|
||||
normalized_search_layers = self._normalize_layers(effective_search_layers)
|
||||
if normalized_search_layers == [MemoryLayer.L1_RAW]:
|
||||
return await self._search_explicit_raw_layer(
|
||||
effective_target_query,
|
||||
user_ids=effective_user_ids,
|
||||
agent_ids=effective_agent_ids,
|
||||
session_ids=effective_session_ids,
|
||||
include_sources=effective_include_sources,
|
||||
exclude_sources=effective_exclude_sources,
|
||||
limit=limit,
|
||||
min_score=min_score,
|
||||
created_after=effective_created_after,
|
||||
request_id=request_id,
|
||||
t0=t0,
|
||||
query_embedding=query_embedding,
|
||||
)
|
||||
|
||||
read_req = ReadRequest(
|
||||
query=effective_target_query,
|
||||
@@ -3888,7 +3955,7 @@ class HyMemoryClient:
|
||||
layers=(
|
||||
[
|
||||
layer.value
|
||||
for layer in normalized_search_layers
|
||||
for layer in self._normalize_layers(effective_search_layers)
|
||||
]
|
||||
if effective_search_layers else None
|
||||
),
|
||||
@@ -3913,7 +3980,7 @@ class HyMemoryClient:
|
||||
_wanted_layers = (
|
||||
{
|
||||
layer.value
|
||||
for layer in normalized_search_layers
|
||||
for layer in self._normalize_layers(effective_search_layers)
|
||||
}
|
||||
if effective_search_layers else None
|
||||
)
|
||||
@@ -4037,100 +4104,35 @@ class HyMemoryClient:
|
||||
# CRUD API(异步)— get / update / delete / delete_all
|
||||
# ================================================================
|
||||
|
||||
async def _coding_store_for_exact_get(self, variant: str):
|
||||
"""Return an existing namespace store without creating an empty DB."""
|
||||
stores = getattr(self, "_coding_stores", None) or {}
|
||||
store = stores.get(variant)
|
||||
if store is not None:
|
||||
return store
|
||||
|
||||
from .coding.store import CodingMemoryStore
|
||||
|
||||
config = getattr(self, "_config", None)
|
||||
coding_config = getattr(config, "coding", None)
|
||||
if config is None or coding_config is None:
|
||||
return None
|
||||
if not CodingMemoryStore.database_exists(
|
||||
config,
|
||||
db_path=coding_config.db_path,
|
||||
variant=variant,
|
||||
):
|
||||
return None
|
||||
await self._get_coding_writer(variant=variant)
|
||||
return (getattr(self, "_coding_stores", None) or {}).get(variant)
|
||||
|
||||
@staticmethod
|
||||
def _format_productivity_get(
|
||||
item: Any,
|
||||
*,
|
||||
kind: str,
|
||||
variant: str,
|
||||
request_id: str,
|
||||
elapsed_ms: float,
|
||||
) -> Dict[str, Any]:
|
||||
payload = item.to_dict()
|
||||
item_id = (
|
||||
payload.get("experience_id")
|
||||
if kind == "experience"
|
||||
else payload.get("fact_id")
|
||||
)
|
||||
return {
|
||||
"memory_id": item_id,
|
||||
"kind": kind,
|
||||
"variant": variant,
|
||||
"scene": "productivity",
|
||||
"content": (
|
||||
payload.get("solution", "")
|
||||
if kind == "experience"
|
||||
else payload.get("statement", "")
|
||||
),
|
||||
"layer": kind,
|
||||
**payload,
|
||||
"request_id": request_id,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}
|
||||
|
||||
@_ensure_internal_loop
|
||||
async def async_get(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: Optional[MemoryKind] = None,
|
||||
variant: Optional[MemoryVariant] = None,
|
||||
include_raw: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
获取单条记忆(异步)。
|
||||
|
||||
默认查询 chat、working/coding experience 和 working/coding fact。
|
||||
若同一 ID 命中多个 namespace,则 fail closed;调用方必须用 kind/variant
|
||||
显式消歧。
|
||||
VDB + Graph 双路并行查询,哪个有结果返回哪个。
|
||||
|
||||
Args:
|
||||
memory_id: 记忆 ID
|
||||
kind: 可选 namespace("chat" / "experience" / "fact")。
|
||||
variant: 可选生产力变体("working" / "coding")。
|
||||
include_raw: 是否允许返回 L1_RAW 原始记忆,默认 False。
|
||||
|
||||
Returns:
|
||||
统一 memory dict,或 None。
|
||||
{"memory_id", "content", "layer", "source", "request_id", "elapsed_ms"} 或 None
|
||||
"""
|
||||
if kind not in (None, "chat", "experience", "fact"):
|
||||
raise ValueError(
|
||||
"kind must be one of: chat, experience, fact"
|
||||
)
|
||||
if variant not in (None, "working", "coding"):
|
||||
raise ValueError("variant must be one of: working, coding")
|
||||
if kind == "chat" and variant is not None:
|
||||
raise ValueError("variant is only valid for experience/fact lookups")
|
||||
import asyncio as _asyncio_get
|
||||
from .models.memory import MemoryLayer
|
||||
|
||||
request_id = str(uuid.uuid4())
|
||||
set_request_id(request_id)
|
||||
t0 = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"[get] memory_id=%s kind=%s variant=%s",
|
||||
memory_id, kind, variant,
|
||||
)
|
||||
logger.info(f"[get] memory_id={memory_id}")
|
||||
|
||||
# 双路并行
|
||||
async def _get_vdb():
|
||||
return await self._vector_store.get_by_id(memory_id)
|
||||
|
||||
@@ -4143,63 +4145,39 @@ class HyMemoryClient:
|
||||
logger.debug(f"[get] graph_store query failed: {e}")
|
||||
return None
|
||||
|
||||
query_chat = kind in (None, "chat") and variant is None
|
||||
coding_kinds = (
|
||||
("experience", "fact") if kind is None else (kind,)
|
||||
)
|
||||
coding_variants = (variant,) if variant is not None else ("working", "coding")
|
||||
vdb_node, graph_node = await _asyncio_get.gather(_get_vdb(), _get_graph())
|
||||
|
||||
stores = {}
|
||||
if kind != "chat":
|
||||
for candidate_variant in coding_variants:
|
||||
store = await self._coding_store_for_exact_get(candidate_variant)
|
||||
if store is not None:
|
||||
stores[candidate_variant] = store
|
||||
|
||||
tasks = []
|
||||
task_names = []
|
||||
if query_chat:
|
||||
tasks.extend((_get_vdb(), _get_graph()))
|
||||
task_names.extend(("chat:vdb", "chat:graph"))
|
||||
for candidate_variant, store in stores.items():
|
||||
if "experience" in coding_kinds:
|
||||
tasks.append(store.get_by_id(memory_id))
|
||||
task_names.append(f"{candidate_variant}:experience")
|
||||
if "fact" in coding_kinds:
|
||||
tasks.append(store.get_fact(memory_id))
|
||||
task_names.append(f"{candidate_variant}:fact")
|
||||
|
||||
values = await asyncio.gather(*tasks) if tasks else []
|
||||
by_name = dict(zip(task_names, values))
|
||||
vdb_node = by_name.get("chat:vdb")
|
||||
graph_node = by_name.get("chat:graph")
|
||||
|
||||
matches = []
|
||||
chat_node = vdb_node if vdb_node is not None else graph_node
|
||||
if chat_node is not None:
|
||||
matches.append(("chat", chat_node))
|
||||
for name, value in by_name.items():
|
||||
if not name.startswith("chat:") and value is not None:
|
||||
matches.append((name, value))
|
||||
|
||||
if len(matches) > 1:
|
||||
raise AmbiguousMemoryIdError(
|
||||
memory_id, [namespace for namespace, _ in matches]
|
||||
)
|
||||
# L1_RAW 默认不对外可见:按 ID 直取走主键查询、绕过向量库的 layer
|
||||
# filter,故在此统一挡掉。SDK 调用方可通过 include_raw=True 显式读取;
|
||||
# HTTP 接口不传该参数,继续保持隐藏。内部写链路不经过此方法。
|
||||
#
|
||||
# 置于 gather 之后、埋点之前:命中 raw 仍照常落 GET 埋点(保留可观测性,
|
||||
# 便于发现有调用方拿 raw id 试探),只是不把内容返回给调用方。
|
||||
# 对调用方的表现与"该 id 不存在"完全一致,不额外暴露"存在但被挡"的信息。
|
||||
if (
|
||||
not include_raw
|
||||
and vdb_node is not None
|
||||
and vdb_node.layer == MemoryLayer.L1_RAW
|
||||
):
|
||||
vdb_node = None
|
||||
if (
|
||||
not include_raw
|
||||
and graph_node is not None
|
||||
and getattr(graph_node, "layer", None) == MemoryLayer.L1_RAW
|
||||
):
|
||||
graph_node = None
|
||||
|
||||
elapsed = round((time.perf_counter() - t0) * 1000, 2)
|
||||
_hit_namespace, _hit_node = matches[0] if matches else ("", None)
|
||||
_source = (
|
||||
"vdb" if vdb_node is not None
|
||||
else ("graph" if graph_node is not None else _hit_namespace)
|
||||
)
|
||||
|
||||
# pipeline trace(best-effort,受 MEMORY_PIPELINE_TRACE_ENABLED 控制)
|
||||
_hit_node = vdb_node if vdb_node is not None else graph_node
|
||||
_source = "vdb" if vdb_node is not None else ("graph" if graph_node is not None else "")
|
||||
_ident = self._backend_identity()
|
||||
_parsed = {
|
||||
"memory_id": memory_id,
|
||||
**_ident,
|
||||
"hit": _hit_node is not None,
|
||||
"source": _source,
|
||||
"namespace": _hit_namespace,
|
||||
}
|
||||
if _hit_node is not None:
|
||||
try:
|
||||
@@ -4216,25 +4194,10 @@ class HyMemoryClient:
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
if not matches:
|
||||
logger.info(f"[get] not found: memory_id={memory_id}")
|
||||
return None
|
||||
|
||||
if _hit_namespace != "chat":
|
||||
hit_variant, hit_kind = _hit_namespace.split(":", 1)
|
||||
return self._format_productivity_get(
|
||||
_hit_node,
|
||||
kind=hit_kind,
|
||||
variant=hit_variant,
|
||||
request_id=request_id,
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
# VDB 优先(如果两边都有,以 VDB 为准)
|
||||
if vdb_node is not None:
|
||||
return {
|
||||
"memory_id": vdb_node.node_id,
|
||||
"kind": "chat",
|
||||
"variant": None,
|
||||
"content": vdb_node.content,
|
||||
"layer": vdb_node.layer.value,
|
||||
"status": vdb_node.status.value if vdb_node.status else "active",
|
||||
@@ -4248,13 +4211,10 @@ class HyMemoryClient:
|
||||
# 不透出 source_raw_memory_id:见 async_search 同处说明。
|
||||
"user_id": vdb_node.user_id,
|
||||
"agent_id": vdb_node.agent_id,
|
||||
"session_id": vdb_node.session_id,
|
||||
"owner": vdb_node.owner,
|
||||
"confidence": vdb_node.confidence,
|
||||
"custom": vdb_node.custom or {},
|
||||
"meta": list((vdb_node.custom or {}).get("meta") or []),
|
||||
"source": vdb_node.source,
|
||||
"retrieval_source": "vdb",
|
||||
"source": "vdb",
|
||||
"request_id": request_id,
|
||||
"elapsed_ms": elapsed,
|
||||
}
|
||||
@@ -4262,8 +4222,6 @@ class HyMemoryClient:
|
||||
if graph_node is not None:
|
||||
return {
|
||||
"memory_id": graph_node.node_id,
|
||||
"kind": "chat",
|
||||
"variant": None,
|
||||
"content": graph_node.content,
|
||||
"layer": graph_node.layer.value if graph_node.layer else "",
|
||||
"status": graph_node.status.value if graph_node.status else "active",
|
||||
@@ -4277,16 +4235,14 @@ class HyMemoryClient:
|
||||
# 不透出 source_raw_memory_id:见 async_search 同处说明。
|
||||
"user_id": graph_node.user_id,
|
||||
"agent_id": graph_node.agent_id,
|
||||
"session_id": getattr(graph_node, "session_id", "") or "",
|
||||
"owner": getattr(graph_node, "owner", None),
|
||||
"confidence": graph_node.confidence,
|
||||
"custom": getattr(graph_node, "custom", {}) or {},
|
||||
"source": getattr(graph_node, "source", "") or "",
|
||||
"retrieval_source": "graph",
|
||||
"source": "graph",
|
||||
"request_id": request_id,
|
||||
"elapsed_ms": elapsed,
|
||||
}
|
||||
|
||||
logger.info(f"[get] not found: memory_id={memory_id}")
|
||||
return None
|
||||
|
||||
@_ensure_internal_loop
|
||||
@@ -4855,7 +4811,6 @@ class HyMemoryClient:
|
||||
"user_id": node.user_id,
|
||||
"agent_id": node.agent_id,
|
||||
"session_id": getattr(node, "session_id", None),
|
||||
"source": getattr(node, "source", None),
|
||||
"speculate": getattr(node, "speculate", None),
|
||||
# 不透出 source_raw_memory_id:见 async_search 同处说明。
|
||||
"tags": node.tags or [],
|
||||
@@ -4927,9 +4882,6 @@ class HyMemoryClient:
|
||||
order: str,
|
||||
user_ids: Optional[List[str]] = None,
|
||||
agent_ids: Optional[List[str]] = None,
|
||||
session_ids: Optional[List[str]] = None,
|
||||
include_sources: Optional[List[str]] = None,
|
||||
exclude_sources: Optional[List[str]] = None,
|
||||
layers: Optional[List] = None,
|
||||
statuses: Optional[List] = None,
|
||||
with_evolution: bool = False,
|
||||
@@ -4955,9 +4907,6 @@ class HyMemoryClient:
|
||||
status_filter=effective_statuses,
|
||||
layers=layers,
|
||||
with_vectors=False,
|
||||
allow_raw_layer=bool(
|
||||
layers is not None and MemoryLayer.L1_RAW in layers
|
||||
),
|
||||
)
|
||||
|
||||
# L0 是 user-level,不应被 agent 过滤。默认列表或显式包含 L0 时,额外按
|
||||
@@ -4979,22 +4928,6 @@ class HyMemoryClient:
|
||||
])
|
||||
all_nodes.extend(hit["node"] for hit in l0_hits if hit.get("node") is not None)
|
||||
|
||||
if session_ids:
|
||||
wanted_sessions = set(session_ids)
|
||||
all_nodes = [
|
||||
node for node in all_nodes if node.session_id in wanted_sessions
|
||||
]
|
||||
if include_sources:
|
||||
wanted_sources = set(include_sources)
|
||||
all_nodes = [
|
||||
node for node in all_nodes if node.source in wanted_sources
|
||||
]
|
||||
if exclude_sources:
|
||||
blocked_sources = set(exclude_sources)
|
||||
all_nodes = [
|
||||
node for node in all_nodes if node.source not in blocked_sources
|
||||
]
|
||||
|
||||
all_nodes = self._sort_memory_nodes(all_nodes, order=order)
|
||||
total = len(all_nodes)
|
||||
page_nodes = all_nodes[offset: offset + limit]
|
||||
@@ -5058,9 +4991,6 @@ class HyMemoryClient:
|
||||
order: str,
|
||||
user_ids: Optional[List[str]] = None,
|
||||
agent_ids: Optional[List[str]] = None,
|
||||
session_ids: Optional[List[str]] = None,
|
||||
include_sources: Optional[List[str]] = None,
|
||||
exclude_sources: Optional[List[str]] = None,
|
||||
layers: Optional[List] = None,
|
||||
statuses: Optional[List] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
@@ -5108,21 +5038,6 @@ class HyMemoryClient:
|
||||
node for node in graph_nodes
|
||||
if getattr(node, "status", None) in effective_statuses
|
||||
]
|
||||
if session_ids:
|
||||
wanted_sessions = set(session_ids)
|
||||
graph_nodes = [
|
||||
node for node in graph_nodes if node.session_id in wanted_sessions
|
||||
]
|
||||
if include_sources:
|
||||
wanted_sources = set(include_sources)
|
||||
graph_nodes = [
|
||||
node for node in graph_nodes if node.source in wanted_sources
|
||||
]
|
||||
if exclude_sources:
|
||||
blocked_sources = set(exclude_sources)
|
||||
graph_nodes = [
|
||||
node for node in graph_nodes if node.source not in blocked_sources
|
||||
]
|
||||
|
||||
graph_nodes = self._sort_memory_nodes(graph_nodes, order=order)
|
||||
total = len(graph_nodes)
|
||||
@@ -5234,9 +5149,6 @@ class HyMemoryClient:
|
||||
branch: Optional[str] = None,
|
||||
user_ids: Optional[List[str]] = None,
|
||||
agent_ids: Optional[List[str]] = None,
|
||||
session_ids: Optional[List[str]] = None,
|
||||
include_sources: Optional[List[str]] = None,
|
||||
exclude_sources: Optional[List[str]] = None,
|
||||
layers: Optional[List[Any]] = None,
|
||||
statuses: Optional[List[Any]] = None,
|
||||
with_evolution: bool = False,
|
||||
@@ -5313,9 +5225,6 @@ class HyMemoryClient:
|
||||
vdb = await self._list_vdb_bucket(
|
||||
user_id=user_id, agent_id=agent_id,
|
||||
user_ids=user_ids, agent_ids=agent_ids,
|
||||
session_ids=session_ids,
|
||||
include_sources=include_sources,
|
||||
exclude_sources=exclude_sources,
|
||||
limit=limit, offset=offset, order=order,
|
||||
layers=effective_layers, statuses=effective_statuses,
|
||||
with_evolution=with_evolution,
|
||||
@@ -5323,9 +5232,6 @@ class HyMemoryClient:
|
||||
graph = await self._list_graph_bucket(
|
||||
user_id=user_id, agent_id=agent_id,
|
||||
user_ids=user_ids, agent_ids=agent_ids,
|
||||
session_ids=session_ids,
|
||||
include_sources=include_sources,
|
||||
exclude_sources=exclude_sources,
|
||||
limit=limit, offset=offset, order=order,
|
||||
layers=effective_layers, statuses=effective_statuses,
|
||||
)
|
||||
|
||||
@@ -416,36 +416,6 @@ class CodingMemoryStore:
|
||||
|
||||
self._initialized = False
|
||||
|
||||
@staticmethod
|
||||
def database_path(
|
||||
config: MemoryConfig,
|
||||
db_path: Optional[str] = None,
|
||||
variant: str = "working",
|
||||
) -> str:
|
||||
"""Return the authoritative SQLite path without creating anything."""
|
||||
if db_path is None:
|
||||
data_dir = (
|
||||
os.path.dirname(config.history.db_path)
|
||||
if config.history.db_path else "./data"
|
||||
)
|
||||
db_path = os.path.join(data_dir, "coding_memory.db")
|
||||
normalized = (
|
||||
"coding"
|
||||
if str(variant or "").strip().lower() == "coding"
|
||||
else "working"
|
||||
)
|
||||
return _variant_db_path(db_path, "_cv" if normalized == "coding" else "")
|
||||
|
||||
@classmethod
|
||||
def database_exists(
|
||||
cls,
|
||||
config: MemoryConfig,
|
||||
db_path: Optional[str] = None,
|
||||
variant: str = "working",
|
||||
) -> bool:
|
||||
"""Whether a coding namespace has an existing SQLite database."""
|
||||
return os.path.isfile(cls.database_path(config, db_path, variant))
|
||||
|
||||
async def initialize(self) -> None:
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
@@ -50,7 +50,6 @@ class CodingWriter:
|
||||
embed_service: EmbedService,
|
||||
fact_reconciler: Optional[WorkingFactReconciler] = None,
|
||||
cache: Any = None,
|
||||
variant: str = "working",
|
||||
):
|
||||
self.store = store
|
||||
self.extractor = extractor
|
||||
@@ -60,11 +59,6 @@ class CodingWriter:
|
||||
self.fact_reconciler = fact_reconciler or WorkingFactReconciler(
|
||||
store, embed_service, llm_provider
|
||||
)
|
||||
self.variant = (
|
||||
"coding"
|
||||
if str(variant or "").strip().lower() == "coding"
|
||||
else "working"
|
||||
)
|
||||
# cache 仅用于把 request_id 串给各阶段(各阶段自己打点);writer 不集中打点。
|
||||
self.cache = cache
|
||||
|
||||
@@ -223,7 +217,6 @@ class CodingWriter:
|
||||
return {
|
||||
"success": True,
|
||||
"scene": "productivity",
|
||||
"variant": self.variant,
|
||||
"request_id": request_id,
|
||||
"elapsed_ms": round((time.perf_counter() - t0) * 1000, 2),
|
||||
"ops": [op.to_dict() for op in ops],
|
||||
@@ -285,11 +278,11 @@ class CodingWriter:
|
||||
f"[coding-write] crosslink fact {f.fact_id} failed: {e}"
|
||||
)
|
||||
|
||||
def _empty_response(self, request_id: str, t0: float) -> Dict[str, Any]:
|
||||
@staticmethod
|
||||
def _empty_response(request_id: str, t0: float) -> Dict[str, Any]:
|
||||
return {
|
||||
"success": True,
|
||||
"scene": "productivity",
|
||||
"variant": self.variant,
|
||||
"request_id": request_id,
|
||||
"elapsed_ms": round((time.perf_counter() - t0) * 1000, 2),
|
||||
"ops": [],
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
"""Public structural contract for injected embedding services."""
|
||||
|
||||
from typing import Any, Dict, List, Protocol, Tuple, runtime_checkable
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EmbeddingService(Protocol):
|
||||
"""Embedding service accepted by :class:`HyMemoryClient`.
|
||||
|
||||
Implementations are borrowed by the client: their lifecycle remains owned
|
||||
by the caller and ``HyMemoryClient.close()`` will not close them.
|
||||
"""
|
||||
|
||||
async def embed(self, text: str, use_cache: bool = True) -> List[float]:
|
||||
...
|
||||
|
||||
async def embed_with_meta(
|
||||
self, text: str, use_cache: bool = True
|
||||
) -> Tuple[List[float], Dict[str, Any]]:
|
||||
...
|
||||
|
||||
async def embed_batch(
|
||||
self, texts: List[str], use_cache: bool = True
|
||||
) -> List[List[float]]:
|
||||
...
|
||||
|
||||
async def embed_queued(
|
||||
self, text: str, use_cache: bool = True
|
||||
) -> List[float]:
|
||||
...
|
||||
|
||||
async def embed_queued_with_meta(
|
||||
self, text: str, use_cache: bool = True
|
||||
) -> Tuple[List[float], Dict[str, Any]]:
|
||||
...
|
||||
|
||||
def get_embedding_dims(self) -> int:
|
||||
...
|
||||
|
||||
|
||||
# Explicit alias for callers that prefer the conventional Protocol suffix.
|
||||
EmbeddingServiceProtocol = EmbeddingService
|
||||
|
||||
|
||||
class EmbeddingDimensionMismatchError(ValueError):
|
||||
"""Raised before initialization when service and store dimensions differ."""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EmbeddingService",
|
||||
"EmbeddingServiceProtocol",
|
||||
"EmbeddingDimensionMismatchError",
|
||||
]
|
||||
@@ -19,7 +19,6 @@ import logging
|
||||
from .base import WritePipeline, ReadPipeline
|
||||
from ..config import MemoryConfig
|
||||
from ..core.embed_service import EmbedService
|
||||
from ..embedding import EmbeddingService
|
||||
from ..data.vector_store import create_vector_store
|
||||
from ..data.vector_store_base import VectorStoreBase
|
||||
|
||||
@@ -51,12 +50,7 @@ class ComponentFactory:
|
||||
s2 = await factory.get_system2_writer() # System2Writer (ultra mode)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Optional[MemoryConfig] = None,
|
||||
*,
|
||||
embedding_service: Optional[EmbeddingService] = None,
|
||||
):
|
||||
def __init__(self, config: Optional[MemoryConfig] = None):
|
||||
self.config = config or MemoryConfig.from_env()
|
||||
|
||||
# 实例缓存(单例)
|
||||
@@ -68,7 +62,7 @@ class ComponentFactory:
|
||||
self._readers: Dict[str, ReadPipeline] = {}
|
||||
|
||||
# 共享资源
|
||||
self._shared_embed_service: Optional[EmbeddingService] = embedding_service
|
||||
self._shared_embed_service: Optional[EmbedService] = None
|
||||
self._shared_vector_store: Optional[VectorStoreBase] = None
|
||||
self._shared_vector_store_initialized: bool = False
|
||||
self._shared_graph_store = None
|
||||
@@ -78,7 +72,7 @@ class ComponentFactory:
|
||||
# 共享资源
|
||||
# ================================================================
|
||||
|
||||
def _get_shared_embed_service(self) -> EmbeddingService:
|
||||
def _get_shared_embed_service(self) -> EmbedService:
|
||||
if self._shared_embed_service is None:
|
||||
self._shared_embed_service = EmbedService(self.config)
|
||||
return self._shared_embed_service
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
"""Public types and errors for exact memory-ID lookup."""
|
||||
|
||||
from typing import Literal, Sequence
|
||||
|
||||
|
||||
MemoryKind = Literal["chat", "experience", "fact"]
|
||||
MemoryVariant = Literal["working", "coding"]
|
||||
|
||||
|
||||
class AmbiguousMemoryIdError(LookupError):
|
||||
"""An ID exists in more than one public memory namespace."""
|
||||
|
||||
def __init__(self, memory_id: str, namespaces: Sequence[str]):
|
||||
self.memory_id = memory_id
|
||||
self.namespaces = tuple(namespaces)
|
||||
joined = ", ".join(self.namespaces)
|
||||
super().__init__(
|
||||
f"memory ID {memory_id!r} is ambiguous across namespaces: {joined}; "
|
||||
"pass kind and/or variant to disambiguate"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["MemoryKind", "MemoryVariant", "AmbiguousMemoryIdError"]
|
||||
@@ -8,7 +8,6 @@ Uses a two-tier strategy:
|
||||
This avoids langdetect's known weakness with short Chinese text.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Optional
|
||||
@@ -136,30 +135,6 @@ def detect_language(text: str) -> str:
|
||||
return "zh" if ratio > 0.1 else "en"
|
||||
|
||||
|
||||
def _spoken_text(value: str) -> str:
|
||||
"""Ignore text-block metadata when a message content is structured JSON."""
|
||||
text = value.strip()
|
||||
if text.startswith("["):
|
||||
try:
|
||||
blocks = json.loads(text)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
else:
|
||||
if (
|
||||
isinstance(blocks, list)
|
||||
and blocks
|
||||
and all(
|
||||
isinstance(block, dict)
|
||||
and isinstance(block.get("type"), str)
|
||||
and block["type"] in {"text", "input_text", "output_text"}
|
||||
and isinstance(block.get("text"), str)
|
||||
for block in blocks
|
||||
)
|
||||
):
|
||||
return "\n".join(block["text"] for block in blocks)
|
||||
return text
|
||||
|
||||
|
||||
def extract_content_for_detection(
|
||||
content: str = "",
|
||||
messages: Optional[List] = None,
|
||||
@@ -178,57 +153,20 @@ def extract_content_for_detection(
|
||||
Concatenated content text suitable for language detection.
|
||||
"""
|
||||
if content and content.strip():
|
||||
value = content.strip()
|
||||
# Writer serializes message batches to JSON before calling Extractor.
|
||||
# Detect the spoken text, not the leading ASCII keys (role/content),
|
||||
# without changing the raw evidence sent to the model.
|
||||
if value.startswith("["):
|
||||
try:
|
||||
decoded = json.loads(value)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
else:
|
||||
if (
|
||||
isinstance(decoded, list)
|
||||
and decoded
|
||||
and all(
|
||||
isinstance(item, dict)
|
||||
and isinstance(item.get("role"), str)
|
||||
and isinstance(item.get("content"), str)
|
||||
for item in decoded
|
||||
)
|
||||
):
|
||||
# In a mixed batch, the latest user turn is the clearest
|
||||
# instruction for the language; assistant/tool metadata
|
||||
# must not decide it first.
|
||||
user_text = [
|
||||
_spoken_text(item["content"])
|
||||
for item in decoded
|
||||
if item["role"] == "user" and item["content"].strip()
|
||||
]
|
||||
if user_text:
|
||||
return user_text[-1]
|
||||
spoken = "\n".join(_spoken_text(item["content"]) for item in decoded)
|
||||
if spoken.strip():
|
||||
return spoken
|
||||
return _spoken_text(value)
|
||||
return content.strip()
|
||||
|
||||
if messages:
|
||||
parts = []
|
||||
user_parts = []
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict):
|
||||
c, role = msg.get("content", ""), msg.get("role")
|
||||
c = msg.get("content", "")
|
||||
elif hasattr(msg, "content"):
|
||||
c, role = msg.content, getattr(msg, "role", None)
|
||||
c = msg.content
|
||||
else:
|
||||
continue
|
||||
if isinstance(c, str) and c.strip():
|
||||
text = _spoken_text(c)
|
||||
parts.append(text)
|
||||
if role == "user":
|
||||
user_parts.append(text)
|
||||
return user_parts[-1] if user_parts else "\n".join(parts)
|
||||
if c and c.strip():
|
||||
parts.append(c.strip())
|
||||
return "\n".join(parts)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Contributing
|
||||
|
||||
感谢你参与 MemHub。
|
||||
感谢你参与 One Memory。
|
||||
|
||||
## 开发环境
|
||||
|
||||
@@ -27,10 +27,3 @@ pnpm check
|
||||
|
||||
涉及外部源码时,必须记录项目、固定版本或 commit、许可证、采用文件和修改说明。不得提交
|
||||
密钥、内部地址、真实对话、个人路径、模型权重或签名证书。
|
||||
|
||||
## 界面翻译
|
||||
|
||||
新增或修改界面语言时遵循
|
||||
[`docs/LOCALIZATION.md`](docs/LOCALIZATION.md):使用稳定语义 key,保持所有
|
||||
locale 的 key 与占位符一致,不翻译用户内容、路径、标识符或外部错误正文。语言
|
||||
catalog 与无关产品行为应拆成不同 PR。
|
||||
+88
-30
@@ -2,25 +2,28 @@
|
||||
|
||||
更新:2026-10-21
|
||||
|
||||
架构边界以 [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) 为准。
|
||||
**文档约定:** 以 [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) **Top Rules** 为现行权威。
|
||||
~~删除线~~ = 历史产品合同(曾要求走 Evidence / Governance 台账),**不再作为目标**;SDK 用法示例仍可用于开发调试。
|
||||
|
||||
---
|
||||
|
||||
## Architecture Principles
|
||||
## Top Rules(与架构一致 · 新)
|
||||
|
||||
1. Memory 读写与存储 backend **只经 SDK**(config / env)。
|
||||
2. MemHub 运行时对 memory 的原子操作 **只经 SDK 公共 Client**;缺正常业务
|
||||
接口先补 SDK。Runtime writers 停止后的 App 启动迁移可直接操作底层存储,
|
||||
但不得把迁移专用方法加入公共 Client。
|
||||
3. MemHub **不存 memory 实体**;**可以存 `memory_id`** 做组织/回执。原文进入 **SDK raw**。
|
||||
4. SDK raw 只保留滚动三天;一条 raw 对应一个完整 turn,只含真人 user 与
|
||||
assistant,tool 不持久化。提取后的高层 memory 不受该窗口影响。
|
||||
2. MemHub 对 memory 的原子操作 **只经 SDK**;缺接口先补 SDK。
|
||||
3. MemHub **不存 memory 实体**;**可以存 `memory_id`** 做组织/回执。原文进 **SDK raw**,不进 MemHub Evidence 权威库。
|
||||
|
||||
生产入口通过共享 Runtime 租约取得公开 `HyMemoryClient`;MemHub 只在调用前执行
|
||||
模型配置、密钥和 harness/model 门禁,不维护第二套 memory 正文权威。
|
||||
生产入口仍建议经 `HyMemoryBridge`(共享 Runtime、模型配置、密钥、harness/model 门禁),但 bridge **不得**再把 SDK 结果镜像成第二套 memory 正文权威。
|
||||
|
||||
来源授权、模型授权和宿主召回授权仍是 MemHub 管理面门禁;memory
|
||||
数据平面的权威在 SDK。
|
||||
---
|
||||
|
||||
## ~~旧开篇(已废弃)~~
|
||||
|
||||
~~本文说明 MemHub 当前使用的 Hy Memory SDK… 仓库内仍以 One Memory 命名的 Evidence、权限和 Governance 层属于 MemHub 的历史兼容模块;产品代码不应绕过这些层直接调用 SDK。~~
|
||||
|
||||
~~MemHub 在本地 Runtime 内嵌入 SDK,但不绕过自身的来源授权、模型授权、Evidence、Governance 和撤权边界。~~
|
||||
|
||||
**新:** 来源授权、模型授权、宿主召回授权仍是 MemHub 管理面门禁。~~Evidence / Governance 记忆台账~~ 按 Top Rules 拆除;memory 真相在 SDK。
|
||||
|
||||
## 当前版本状态
|
||||
|
||||
@@ -35,15 +38,53 @@ reader: hybrid
|
||||
|
||||
## MemHub 运行时边界
|
||||
|
||||
MemHub 通过公开 `HyMemoryClient` 配置使用可复现的本地策略:
|
||||
### 新
|
||||
|
||||
MemHub 对 SDK 使用公开、可复现的本地策略(经 bridge / `hy_runtime_config` 注入):
|
||||
|
||||
- vector / graph / cache / history / coding 等 **一律 SDK config/env**;
|
||||
- 模型与 embedding 来自版本化 Model Profile;密钥进安全存储;
|
||||
- MemHub 管理库可存 harness/model/队列状态与 **`memory_id` 引用**。
|
||||
|
||||
MemHub 不再通过 import-time 环境变量覆盖 SDK 的提取、reconcile、episode、
|
||||
intention、few-shot、summary 或去重策略。产品只配置模型/密钥/endpoint、本地
|
||||
backend 路径、公开 embedding service,以及每次公开 Client 调用的参数。
|
||||
### ~~旧~~
|
||||
|
||||
~~- MemHub 不绕过 Evidence、Governance…~~
|
||||
~~- 生产路径不能用直接 `client.add()` 代替 Governance 检查。~~
|
||||
|
||||
发布包默认 SDK 行为环境变量仍集中在
|
||||
`apps/core/src/one_memory/hy_runtime_config.py`,并由下表锁定:
|
||||
|
||||
| 变量 | 值 |
|
||||
|---|---|
|
||||
| `MEMORY_MODE` | `ultra` |
|
||||
| `MEMORY_ENABLE_GRAPH` | `true` |
|
||||
| `MEMORY_GRAPH_RERANK_ENABLED` | `false` |
|
||||
| `MEMORY_GRAPH_TEMPORAL_SPINE` | `true` |
|
||||
| `MEMORY_TOPIC_SUMMARY_ENABLED` | `false` |
|
||||
| `MEMORY_EPISODE_SUMMARY_ENABLED` | `false` |
|
||||
| `MEMORY_EPISODE_RECALL_ENABLED` | `true` |
|
||||
| `MEMORY_DYNAMIC_EPISODE_ENABLED` | `true` |
|
||||
| `MEMORY_SESSION_EPISODE_ENABLED` | `false` |
|
||||
| `MEMORY_SESSION_EPISODE_CONTEXT_ENABLED` | `false` |
|
||||
| `MEMORY_EPISODE_CAPSULE_CONTEXT_ENABLED` | `true` |
|
||||
| `MEMORY_EPISODE_CAPSULE_QUERY_MIN` | `0.65` |
|
||||
| `MEMORY_EPISODE_CAPSULE_ASSIGNMENT_MIN` | `0.60` |
|
||||
| `MEMORY_EPISODE_CAPSULE_FIRST_RANK_MAX` | `3` |
|
||||
| `MEMORY_EPISODE_CAPSULE_MEMBER_LIMIT` | `3` |
|
||||
| `MEMORY_ENABLE_INTENTION` | `false` |
|
||||
| `MEMORY_ENABLE_QUERY_REWRITE` | `false` |
|
||||
| `MEMORY_FEW_SHOT_ENABLED` | `true` |
|
||||
| `MEMORY_HYBRID_V2_QUERY_EXPAND` | `1` |
|
||||
| `MEMORY_HYBRID_V2_GEO_INTENT_PRIOR` | `0.6` |
|
||||
| `MEMORY_HYBRID_V2_DIVERSITY` | `1` |
|
||||
| `MEMORY_HYBRID_V2_TEMPORAL_ADMIT_PRIOR` | `0.55` |
|
||||
| `MEMORY_HYBRID_V2_TEMPORAL_BOOST` | `0.18` |
|
||||
| `MEMORY_CROSS_DOMAIN_CON_THRESHOLD` | `0.5` |
|
||||
| `MEMORY_READER_ENABLE_SUMMARY` | `false` |
|
||||
| `MEMORY_SUMMARY_ENABLED_IN_SYS2` | `false` |
|
||||
| `MEMORY_LOCAL_RERANK_ENABLED` | `false` |
|
||||
| `RECONCILE_ENABLED` | `true` |
|
||||
| `HY_MEMORY_READER` | `hybrid` |
|
||||
|
||||
独立脚本直连 SDK 时须自配存储目录与模型端点。
|
||||
|
||||
@@ -51,8 +92,8 @@ backend 路径、公开 embedding service,以及每次公开 Client 调用的
|
||||
|
||||
下面是独立调试示例。
|
||||
|
||||
产品代码经 `hy_sdk_runtime.SdkClientHandle` 获取已配置的公开 client;memory I/O
|
||||
由 SDK 完成。
|
||||
**新:** 产品代码经 `HyMemoryBridge` 获取已配置的 client(管理面门禁),memory I/O 仍是 SDK。
|
||||
~~旧:必须额外提供 Evidence 绑定、写后核验进 Governance。~~
|
||||
|
||||
```python
|
||||
from hy_memory import HyMemoryClient, MemoryConfig
|
||||
@@ -85,7 +126,7 @@ client = HyMemoryClient(config=config, mode="ultra")
|
||||
|
||||
## Write:写入对话
|
||||
|
||||
推荐把一段连续对话作为 OpenAI messages 写入;含原文时由 SDK 接管 raw:
|
||||
推荐把一段连续对话作为 OpenAI messages 写入(**含原文时由 SDK 接管 raw**,不再写入 MemHub Evidence 权威表):
|
||||
|
||||
```python
|
||||
from datetime import UTC, datetime
|
||||
@@ -104,6 +145,7 @@ result = client.add(
|
||||
request_id=str(uuid4()),
|
||||
metadata={
|
||||
"source_id": "cursor",
|
||||
# ~~evidence_ids 指向 MemHub Evidence 表~~ → 新:按 SDK raw / 元数据合同填写;勿依赖 MemHub evidence 权威
|
||||
"workspace_ids": ["workspace-abc"],
|
||||
"input_kind": "raw_conversation",
|
||||
},
|
||||
@@ -144,11 +186,11 @@ if result.get("success") is not True:
|
||||
memory_ids = list(result.get("memory_ids") or [])
|
||||
if result.get("memory_id"):
|
||||
memory_ids.append(result["memory_id"])
|
||||
# 需要组织或回执时,MemHub 只持久化这些 memory_id。
|
||||
# 新:需要组织/回执时,MemHub 只持久化这些 memory_id
|
||||
```
|
||||
|
||||
以 SDK 合同为准做成功判定;需要写后核验时使用 SDK `get` 回读。缺少所需
|
||||
核验能力时扩展 SDK API,不在 MemHub 新建镜像台账。
|
||||
~~不要只相信 `success=True`。One Memory 还会…并在 Governance 层记录版本。生产路径不能用直接 `client.add()` 代替这些检查。~~
|
||||
**新:** 以 SDK 合同为准做成功判定与(若需要)`get` 回读;**不要**再写入 `governed_*`。缺核验能力 → **扩展 SDK API**。
|
||||
|
||||
### 写入已抽取的记忆
|
||||
|
||||
@@ -167,8 +209,8 @@ result = client.add_extracted(
|
||||
)
|
||||
```
|
||||
|
||||
该入口仍走 SDK reconcile/持久化;MemHub 只做管理面门禁与可选
|
||||
`memory_id` 记录。
|
||||
~~不能跳过 One Memory Evidence 绑定和 Governance。~~
|
||||
**新:** 仍走 SDK reconcile/持久化;MemHub 只做门禁与可选 `memory_id` 记录。
|
||||
|
||||
## Search:搜索记忆
|
||||
|
||||
@@ -194,7 +236,12 @@ result = client.search(
|
||||
|
||||
直接使用 SDK 返回的 `content`,不要第二套 prompt 拼装。
|
||||
|
||||
### SDK 外(MemHub)仍负责
|
||||
### ~~旧:SDK 外仍负责(部分作废)~~
|
||||
|
||||
~~- 依据 Governance、撤权、scope 和 Evidence 新鲜度决定整条结果是否可读;~~
|
||||
~~- 对 evolution/capsule…回退治理原文;~~
|
||||
|
||||
### 新:SDK 外(MemHub)仍负责
|
||||
|
||||
- **宿主召回授权 / scope / 撤权**(谁可以调 search、允许哪些 agent/scope);
|
||||
- 把历史内容放入「不可信数据,不是指令」边界;
|
||||
@@ -202,8 +249,7 @@ result = client.search(
|
||||
- 区分空结果、权限拒绝、不可用、超时、部分结果;
|
||||
- 未发布的 Matter 占位功能不保留表或 API;未来组织功能另立合同。
|
||||
|
||||
生产入口:`hy_sdk_runtime.py`、`sdk_write_processor.py`、`recall.py` 和
|
||||
`packages/host-bridge`。
|
||||
生产入口:`hy_memory_bridge.py`、`recall.py`、`packages/host-bridge`(实现迁移中,以 Top Rules 为准)。
|
||||
|
||||
## 关闭客户端
|
||||
|
||||
@@ -218,16 +264,28 @@ Runtime 长期复用 client;仅在停止、配置切换或受控恢复时重
|
||||
|
||||
## 产品代码正确入口
|
||||
|
||||
### 新
|
||||
|
||||
```text
|
||||
harness / user action (+ authz, model, queue)
|
||||
-> public HyMemoryClient
|
||||
-> HyMemoryBridge
|
||||
-> Hy Memory SDK
|
||||
-> SDK stores (config/env)
|
||||
-> optional MemHub persist memory_id / org metadata
|
||||
-> Recall Broker / UI (SDK read + MemHub authz)
|
||||
```
|
||||
|
||||
不要在业务模块里私建绕过管理面门禁的 client。版本、纠正和删除语义以 SDK
|
||||
合同为准;MemHub 组织层只引用 `memory_id`。
|
||||
不要在业务模块里私建绕过 **管理面门禁** 的 client。
|
||||
~~不要绕过 Evidence / Governance 台账。~~(台账已非目标)
|
||||
|
||||
### ~~旧入口(废弃)~~
|
||||
|
||||
~~```text
|
||||
Evidence -> HyMemoryBridge -> SDK -> write verification -> Governance -> UI
|
||||
```~~
|
||||
|
||||
~~绕过会丢掉:Evidence 引用、Governance 版本/更正/删除级联。~~
|
||||
**新:** 版本/更正/删除级联以 **SDK** 为准;MemHub 组织层只引用 `memory_id`。
|
||||
|
||||
Hy Memory SDK 是 MemHub 内置可信引擎,管理本机 Hy Memory 数据。
|
||||
来源授权决定哪些宿主数据可进入写入路径;宿主 Recall Grant 决定外部应用可读范围。
|
||||
|
||||
+8
-10
@@ -18,13 +18,13 @@ If you run into a problem, please open an Issue with the version, platform,repro
|
||||
|
||||
## 发行与平台合同
|
||||
|
||||
源码当前固定使用公开 PyPI 正式版 `hy-memory==2.0.5`。MemHub 源码版本为 `1.0.5`
|
||||
(build 14),GitHub 仓库为
|
||||
当前重构验证固定使用开发预览 `hy-memory==2.0.5.dev3`。MemHub 源码版本为 `1.0.4`
|
||||
(build 13),GitHub 仓库为
|
||||
[`Tencent-Hunyuan/Hy-Memory`](https://github.com/Tencent-Hunyuan/Hy-Memory),npm 包名为
|
||||
`memhub-cli`。当前代码架构见
|
||||
[`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)。
|
||||
|
||||
`v1.0.5` 的发行目标是 Linux x64/arm64 Runtime 与 macOS arm64/x64 App 和 Runtime。
|
||||
`v1.0.4` 的发行目标是 Linux x64/arm64 Runtime 与 macOS arm64/x64 App 和 Runtime。
|
||||
Linux 制品由 GitHub Actions 在原生架构 runner 上构建并完成生命周期验收;macOS
|
||||
制品由发行流水线完成 Developer ID 签名、公证和验收后手动上传。Windows 不支持。
|
||||
只有同时出现在 GitHub Release 与 `release-manifest.json` 中、且通过公开校验的制品才属于
|
||||
@@ -38,26 +38,24 @@ Linux 制品由 GitHub Actions 在原生架构 runner 上构建并完成生命
|
||||
|
||||
- 一次安装、一个 Runtime、一份数据、多种客户端。
|
||||
- Hy Memory SDK 是 memory/raw/vector/graph/coding 数据面的唯一权威;Python Core
|
||||
管理授权、模型、异步队列和 Runtime 状态。
|
||||
管理授权、模型、异步队列、Runtime 状态与 Matters 引用。
|
||||
- 来源读取、模型处理和宿主读取是相互独立的授权边界。
|
||||
- 正式写链路为 `memory_write_jobs -> SDK add() -> raw/memory ID receipt`。
|
||||
- 正式写链路为 `memory_write_jobs -> SDK add() -> raw/memory ID receipt`;
|
||||
Matter/Wiki 只保存并解析 SDK ID 引用。
|
||||
- 产品不包含屏幕、Accessibility / AXTree、OCR、键鼠、剪贴板或后台录音采集。
|
||||
|
||||
## 仓库结构
|
||||
|
||||
- `apps/core`:Python / FastAPI Kernel,负责授权、模型、队列、SDK Runtime 边界与 API。
|
||||
- `apps/core`:Python / FastAPI Kernel,负责授权、模型、队列、SDK bridge、Matters 与 API。
|
||||
- `apps/cli`:安装、控制和诊断共享 Runtime 的 `one-memory` CLI。
|
||||
- `apps/web`:React / TypeScript 管理界面。
|
||||
- `apps/desktop`:Electron 桌面壳与图形安装入口。
|
||||
- `packages/contracts`:公共 API 类型。
|
||||
- `packages/i18n`:Web 与 Electron 共用的界面语言合同和 catalog。
|
||||
- `packages/host-bridge`:AI App 的 Skill、Hook、插件和 MCP 接入。
|
||||
- `packages/runtime-control`:Core、Host Gateway 和 Runtime 生命周期控制。
|
||||
|
||||
架构详情见 [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) 和
|
||||
[`docs/contracts/runtime-unification.md`](docs/contracts/runtime-unification.md)。
|
||||
界面语言架构与社区翻译规则见
|
||||
[`docs/LOCALIZATION.md`](docs/LOCALIZATION.md)。
|
||||
产品、测试和一线支持排查导入与记忆沉淀时,使用
|
||||
[`docs/PRODUCT_LOG_TROUBLESHOOTING.md`](docs/PRODUCT_LOG_TROUBLESHOOTING.md)。
|
||||
|
||||
@@ -117,7 +115,7 @@ MemHub 的 SQLite 数据默认位于 `~/.one-memory`,不在 npm 包或 Runtime
|
||||
|
||||
## Hy Memory
|
||||
|
||||
SDK 依赖固定为公开 PyPI 正式版 `hy-memory==2.0.5`。SDK 配置、权限边界及 search
|
||||
当前开发依赖固定为公开 PyPI 开发预览 `hy-memory==2.0.5.dev3`。SDK 配置、权限边界及 search
|
||||
返回格式见 [`MEMORY.md`](MEMORY.md)。HY search 返回的 `content` 已包含 SDK 统一生成的
|
||||
逻辑时间和演化上下文;调用方不应再次转换时间戳或重复拼接 `When`。
|
||||
|
||||
|
||||
+3
-25
@@ -3,9 +3,9 @@
|
||||
"dataLicense": "CC0-1.0",
|
||||
"SPDXID": "SPDXRef-DOCUMENT",
|
||||
"name": "one-memory",
|
||||
"documentNamespace": "https://one-memory.local/spdx/2026-10-22T13:18:16Z",
|
||||
"documentNamespace": "https://one-memory.local/spdx/2026-10-17T13:52:48Z",
|
||||
"creationInfo": {
|
||||
"created": "2026-10-22T13:18:16Z",
|
||||
"created": "2026-10-17T13:52:48Z",
|
||||
"creators": [
|
||||
"Tool: one-memory-generate-sbom"
|
||||
]
|
||||
@@ -88,23 +88,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"SPDXID": "SPDXRef-Package-npm-i18next",
|
||||
"name": "i18next",
|
||||
"versionInfo": "25.10.10",
|
||||
"downloadLocation": "NOASSERTION",
|
||||
"filesAnalyzed": false,
|
||||
"licenseConcluded": "NOASSERTION",
|
||||
"licenseDeclared": "NOASSERTION",
|
||||
"copyrightText": "NOASSERTION",
|
||||
"externalRefs": [
|
||||
{
|
||||
"referenceCategory": "PACKAGE-MANAGER",
|
||||
"referenceType": "purl",
|
||||
"referenceLocator": "pkg:npm/i18next@25.10.10"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"SPDXID": "SPDXRef-Package-npm-lucide-react",
|
||||
"name": "lucide-react",
|
||||
@@ -2352,7 +2335,7 @@
|
||||
{
|
||||
"SPDXID": "SPDXRef-Package-hy-memory",
|
||||
"name": "hy-memory",
|
||||
"versionInfo": "2.0.5",
|
||||
"versionInfo": "2.0.5.dev3",
|
||||
"downloadLocation": "NOASSERTION",
|
||||
"filesAnalyzed": false,
|
||||
"licenseConcluded": "Apache-2.0",
|
||||
@@ -2397,11 +2380,6 @@
|
||||
"relationshipType": "DEPENDS_ON",
|
||||
"relatedSpdxElement": "SPDXRef-Package-npm-https-proxy-agent"
|
||||
},
|
||||
{
|
||||
"spdxElementId": "SPDXRef-Package-one-memory",
|
||||
"relationshipType": "DEPENDS_ON",
|
||||
"relatedSpdxElement": "SPDXRef-Package-npm-i18next"
|
||||
},
|
||||
{
|
||||
"spdxElementId": "SPDXRef-Package-one-memory",
|
||||
"relationshipType": "DEPENDS_ON",
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
MemHub itself is licensed under Apache-2.0 (see `LICENSE`).
|
||||
It bundles the runtime dependencies listed below.
|
||||
Generated from the installed dependency tree on 2026-10-22T13:18:16Z
|
||||
Generated from the installed dependency tree on 2026-10-17T13:52:48Z
|
||||
by `python3 scripts/generate_sbom.py`. Do not edit by hand.
|
||||
|
||||
Per-package license texts are not reproduced here yet; the license
|
||||
column is filled in as part of release review, and the SBOM records
|
||||
`NOASSERTION` until then rather than guessing.
|
||||
|
||||
## hy-memory 2.0.5
|
||||
## hy-memory 2.0.5.dev3
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
@@ -112,13 +112,12 @@ Adaptation boundary:
|
||||
|
||||
No Memmy brand, icon, font or mascot assets are distributed.
|
||||
|
||||
## JavaScript (npm) — 11 packages
|
||||
## JavaScript (npm) — 10 packages
|
||||
|
||||
- @tanstack/react-query 5.103.2
|
||||
- axios 1.20.0
|
||||
- fflate 0.8.3
|
||||
- https-proxy-agent 7.0.0
|
||||
- i18next 25.10.10
|
||||
- lucide-react 1.43.0
|
||||
- proxy-from-env 2.1.0
|
||||
- react 19.2.8
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@one-memory/cli",
|
||||
"version": "1.0.5",
|
||||
"version": "1.0.4",
|
||||
"description": "MemHub bootstrap CLI for installing and managing the local-first shared memory runtime",
|
||||
"license": "Apache-2.0",
|
||||
"repository": {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { inflateRawSync } from "node:zlib";
|
||||
import { DiagnosticLogger } from "@one-memory/runtime-control";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
|
||||
@@ -30,27 +29,6 @@ async function fixture() {
|
||||
return dataDir;
|
||||
}
|
||||
|
||||
function readZipEntry(archive: Buffer, expected: string): string {
|
||||
let offset = 0;
|
||||
while (archive.readUInt32LE(offset) === 0x04034b50) {
|
||||
const method = archive.readUInt16LE(offset + 8);
|
||||
const compressedSize = archive.readUInt32LE(offset + 18);
|
||||
const nameLength = archive.readUInt16LE(offset + 26);
|
||||
const extraLength = archive.readUInt16LE(offset + 28);
|
||||
const nameStart = offset + 30;
|
||||
const dataStart = nameStart + nameLength + extraLength;
|
||||
const name = archive.subarray(nameStart, nameStart + nameLength).toString();
|
||||
const compressed = archive.subarray(dataStart, dataStart + compressedSize);
|
||||
if (name === expected) {
|
||||
if (method === 0) return compressed.toString();
|
||||
if (method === 8) return inflateRawSync(compressed).toString();
|
||||
throw new Error("unsupported zip compression");
|
||||
}
|
||||
offset = dataStart + compressedSize;
|
||||
}
|
||||
throw new Error(`missing ${expected}`);
|
||||
}
|
||||
|
||||
it("prints matching diagnostic events as NDJSON without starting Runtime", async () => {
|
||||
const dataDir = await fixture();
|
||||
let stdout = "";
|
||||
@@ -96,25 +74,6 @@ it("exports one support zip to the requested downloads directory", async () => {
|
||||
});
|
||||
expect(receipt.path).toMatch(/MemHub-日志-3800-\d{8}-\d{4}\.zip$/);
|
||||
expect(await readdir(downloads)).toEqual([path.basename(receipt.path)]);
|
||||
const archive = await readFile(receipt.path);
|
||||
expect(JSON.parse(readZipEntry(archive, "summary.json")).commit).toBeNull();
|
||||
});
|
||||
|
||||
it("fails closed for an explicitly supplied invalid commit", async () => {
|
||||
const dataDir = await fixture();
|
||||
const downloads = await mkdtemp(path.join(os.tmpdir(), "om-cli-downloads-"));
|
||||
roots.push(downloads);
|
||||
vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
|
||||
await expect(
|
||||
run(["logs", "export", "--json"], {
|
||||
ONE_MEMORY_DATA_DIR: dataDir,
|
||||
ONE_MEMORY_EXPORT_DIR: downloads,
|
||||
ONE_MEMORY_BUILD_NUMBER: "3800",
|
||||
ONE_MEMORY_COMMIT: "unknown",
|
||||
}),
|
||||
).rejects.toThrow("diagnostic_commit_invalid");
|
||||
expect(await readdir(downloads)).toEqual([]);
|
||||
});
|
||||
|
||||
it("never emits a secret from a malformed manually written event", async () => {
|
||||
|
||||
@@ -1048,21 +1048,15 @@ async function runCommand(
|
||||
entry: "cli",
|
||||
}));
|
||||
const target = path.join(downloads, diagnosticZipFilename(build));
|
||||
const environmentCommit = environment.ONE_MEMORY_COMMIT?.trim();
|
||||
const embeddedCommit =
|
||||
typeof __ONE_MEMORY_COMMIT__ === "string"
|
||||
? __ONE_MEMORY_COMMIT__.trim()
|
||||
: "";
|
||||
const commit =
|
||||
environmentCommit ||
|
||||
(embeddedCommit && embeddedCommit !== "unknown"
|
||||
? embeddedCommit
|
||||
: undefined);
|
||||
const receipt = await exportDiagnosticZip({
|
||||
dataDir: config.dataDir,
|
||||
target,
|
||||
productVersion: build,
|
||||
...(commit === undefined ? {} : { commit }),
|
||||
commit:
|
||||
environment.ONE_MEMORY_COMMIT?.trim() ||
|
||||
(typeof __ONE_MEMORY_COMMIT__ === "string"
|
||||
? __ONE_MEMORY_COMMIT__
|
||||
: "unknown"),
|
||||
coreSha256: await sha256File(config.coreCommand).catch(() => undefined),
|
||||
home: os.homedir(),
|
||||
crashReportsDir: path.join(
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
import {
|
||||
CANDIDATE_KINDS,
|
||||
type CandidateKind,
|
||||
type GovernedMemory,
|
||||
type HomeSummary,
|
||||
type ImportIssue,
|
||||
type ImportJob,
|
||||
type LiveSyncHealth,
|
||||
type MemoryDetail,
|
||||
type MemoryEvidenceExcerpt,
|
||||
type MemoryLibraryPage,
|
||||
type MemoryProcessingSummary,
|
||||
type ModelProfile,
|
||||
type SourceResponse,
|
||||
import type {
|
||||
GovernedMemory,
|
||||
HomeSummary,
|
||||
ImportIssue,
|
||||
ImportJob,
|
||||
LiveSyncHealth,
|
||||
MemoryDetail,
|
||||
MemoryEvidenceExcerpt,
|
||||
MemoryLibraryPage,
|
||||
MemoryProcessingSummary,
|
||||
ModelProfile,
|
||||
SourceResponse,
|
||||
} from "@one-memory/contracts";
|
||||
/** The only memory states a response may carry, and the only ones a filter may ask for. */
|
||||
export const MEMORY_STATUSES = ["confirmed", "deleted", "superseded"] as const;
|
||||
@@ -76,6 +74,11 @@ const SYNC_COMPLETENESS = [
|
||||
"additions_only",
|
||||
"one_time_snapshot",
|
||||
] as const;
|
||||
const CANDIDATE_KINDS = [
|
||||
"default",
|
||||
"environment_override",
|
||||
"user_selected",
|
||||
] as const;
|
||||
const LOCATION_STATES = [
|
||||
"found",
|
||||
"not_found",
|
||||
@@ -97,7 +100,7 @@ export interface ValidatedMemoryTask {
|
||||
export interface SourceConnectionCandidate {
|
||||
source_id: string;
|
||||
display_name: string;
|
||||
candidate_kind: CandidateKind;
|
||||
candidate_kind: (typeof CANDIDATE_KINDS)[number];
|
||||
candidate_root_id: string;
|
||||
candidate_revision?: number;
|
||||
requires_override_confirmation: boolean;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdtemp, rm, symlink } from "node:fs/promises";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DiagnosticLogger } from "@one-memory/runtime-control";
|
||||
@@ -41,15 +41,11 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
|
||||
session_id: "11111111-1111-4111-8111-111111111111",
|
||||
details: {
|
||||
count: 1,
|
||||
reason: "/Users/me/private-query.txt",
|
||||
status: "sk-live-secret-token",
|
||||
source_id: "credential.private",
|
||||
prompt: "secret-token-123",
|
||||
authorization: "Bearer secret-token-123",
|
||||
} as unknown as Record<string, string | number | boolean | null>,
|
||||
safe_stack: [
|
||||
"packages/runtime-control/src/example.ts:42:7",
|
||||
"packages/runtime/sk-live-secret-token.ts:1:1",
|
||||
"/Users/me/private.py:1",
|
||||
"C:\\Users\\me\\private.py:1",
|
||||
],
|
||||
@@ -60,8 +56,6 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
|
||||
"packages/runtime-control/src/example.ts:42:7",
|
||||
]);
|
||||
expect(JSON.stringify(redacted)).not.toContain("secret-token-123");
|
||||
expect(JSON.stringify(redacted)).not.toContain("sk-live-secret-token");
|
||||
expect(JSON.stringify(redacted)).not.toContain("credential.private");
|
||||
expect(JSON.stringify(redacted)).not.toContain("/Users/me");
|
||||
expect(JSON.stringify(redacted)).not.toContain("C:\\\\Users");
|
||||
});
|
||||
@@ -138,27 +132,3 @@ it("follows a newly appended event without stopping Runtime", async () => {
|
||||
});
|
||||
expect(seen).toEqual(["service.ready", "core.failed"]);
|
||||
});
|
||||
|
||||
it("fails follow mode when the diagnostic directory is unsafe", async () => {
|
||||
const dataDir = await mkdtemp(
|
||||
path.join(os.tmpdir(), "om-cli-follow-unsafe-"),
|
||||
);
|
||||
const external = await mkdtemp(
|
||||
path.join(os.tmpdir(), "om-cli-follow-external-"),
|
||||
);
|
||||
roots.push(dataDir, external);
|
||||
await symlink(external, path.join(dataDir, "logs"));
|
||||
const query = parseLogQuery(["--follow"]);
|
||||
expect(query).not.toBeNull();
|
||||
if (!query) throw new Error("query_missing");
|
||||
await expect(
|
||||
followLogs({
|
||||
dataDir,
|
||||
query,
|
||||
signal: new AbortController().signal,
|
||||
maxEvents: 1,
|
||||
emit: () => undefined,
|
||||
wait: async () => undefined,
|
||||
}),
|
||||
).rejects.toThrow("log_directory_unreadable");
|
||||
});
|
||||
@@ -3,7 +3,6 @@ import path from "node:path";
|
||||
import type { DiagnosticEventV1 } from "@one-memory/runtime-control";
|
||||
import {
|
||||
diagnosticLogFiles,
|
||||
projectDiagnosticEventForSafeOutput,
|
||||
readDiagnosticEvents,
|
||||
} from "@one-memory/runtime-control";
|
||||
|
||||
@@ -25,6 +24,54 @@ export interface LogQuery {
|
||||
|
||||
const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
|
||||
const NAME = /^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$/;
|
||||
const SAFE_DETAIL_KEYS = new Set([
|
||||
"action",
|
||||
"attempt",
|
||||
"count",
|
||||
"method",
|
||||
"mode",
|
||||
"phase",
|
||||
"platform",
|
||||
"reason",
|
||||
"result_count",
|
||||
"source_id",
|
||||
"spawned",
|
||||
"status",
|
||||
"version",
|
||||
"parent_session_id",
|
||||
"invalid_lines",
|
||||
"truncated",
|
||||
"runtime_state",
|
||||
"core_state",
|
||||
"broker_state",
|
||||
"instance_id",
|
||||
"open_file_count",
|
||||
"memory_bytes",
|
||||
"thread_count",
|
||||
"database_error_count",
|
||||
"file_count",
|
||||
"sqlite_file_count",
|
||||
"byte_file_count",
|
||||
"total_bytes",
|
||||
"copied_bytes",
|
||||
"bytes_per_second",
|
||||
"stored_count",
|
||||
"revised_count",
|
||||
"skipped_count",
|
||||
"error_count",
|
||||
"pending_count",
|
||||
"prepared_count",
|
||||
"queue_depth",
|
||||
"in_flight_count",
|
||||
"call_count",
|
||||
"created_count",
|
||||
"updated_count",
|
||||
"superseded_count",
|
||||
"deleted_count",
|
||||
"operation_kind",
|
||||
"file_kind",
|
||||
]);
|
||||
|
||||
export function parseLogQuery(arguments_: readonly string[]): LogQuery | null {
|
||||
if (arguments_.filter((item) => item === "--json").length > 1) return null;
|
||||
const values: Partial<LogQuery> = { errorsOnly: false, follow: false };
|
||||
@@ -190,13 +237,63 @@ export function renderLogText(
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
/** Defence in depth for the CLI read surface. Local writers accept arbitrary
|
||||
* bounded scalar keys, so every outward read surface must reuse the same
|
||||
* field-and-value projection as the support-bundle exporter. */
|
||||
/** Defence in depth for the CLI read surface. `DiagnosticLogger` already
|
||||
* allow-lists keys, but a future writer, a manually edited log, or a malformed
|
||||
* line must not turn `memhub logs` into an exfiltration path. */
|
||||
export function redactDiagnosticEvent(
|
||||
event: DiagnosticEventV1,
|
||||
): DiagnosticEventV1 {
|
||||
return projectDiagnosticEventForSafeOutput(event);
|
||||
const redacted: DiagnosticEventV1 = {
|
||||
schema_version: 1,
|
||||
timestamp: event.timestamp,
|
||||
level: event.level,
|
||||
service: event.service,
|
||||
component: event.component,
|
||||
event: event.event,
|
||||
outcome: event.outcome,
|
||||
session_id: event.session_id,
|
||||
...(event.trace_id === undefined ? {} : { trace_id: event.trace_id }),
|
||||
...(event.incident_id === undefined
|
||||
? {}
|
||||
: { incident_id: event.incident_id }),
|
||||
...(event.instance_id === undefined
|
||||
? {}
|
||||
: { instance_id: event.instance_id }),
|
||||
...(event.operation_id === undefined
|
||||
? {}
|
||||
: { operation_id: event.operation_id }),
|
||||
...(event.request_id === undefined ? {} : { request_id: event.request_id }),
|
||||
...(event.job_id === undefined ? {} : { job_id: event.job_id }),
|
||||
...(event.task_id === undefined ? {} : { task_id: event.task_id }),
|
||||
...(event.duration_ms === undefined
|
||||
? {}
|
||||
: { duration_ms: event.duration_ms }),
|
||||
...(event.state_before === undefined
|
||||
? {}
|
||||
: { state_before: event.state_before }),
|
||||
...(event.state_after === undefined
|
||||
? {}
|
||||
: { state_after: event.state_after }),
|
||||
...(event.error_code === undefined ? {} : { error_code: event.error_code }),
|
||||
...(event.error_type === undefined ? {} : { error_type: event.error_type }),
|
||||
...(event.safe_stack === undefined
|
||||
? {}
|
||||
: {
|
||||
safe_stack: event.safe_stack.filter(
|
||||
(frame) =>
|
||||
!frame.startsWith("/") &&
|
||||
!/^[A-Za-z]:\\/.test(frame) &&
|
||||
!frame.includes(".."),
|
||||
),
|
||||
}),
|
||||
};
|
||||
const details = Object.fromEntries(
|
||||
Object.entries(event.details ?? {}).filter(([key]) =>
|
||||
SAFE_DETAIL_KEYS.has(key),
|
||||
),
|
||||
);
|
||||
if (Object.keys(details).length > 0) redacted.details = details;
|
||||
return redacted;
|
||||
}
|
||||
|
||||
export async function followLogs(options: {
|
||||
@@ -218,7 +315,6 @@ export async function followLogs(options: {
|
||||
...options.query,
|
||||
tail: 5_000,
|
||||
});
|
||||
if (report.failure) throw new Error(report.failure);
|
||||
for (const event of report.events) {
|
||||
const identity = JSON.stringify(event);
|
||||
if (seen.has(identity)) continue;
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
||||
|
||||
[project]
|
||||
name = "one-memory"
|
||||
version = "1.0.5"
|
||||
version = "1.0.4"
|
||||
description = "MemHub local-first memory core"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -12,7 +12,7 @@ dependencies = [
|
||||
"fastapi>=0.141.1",
|
||||
"filelock>=3.32.4",
|
||||
# Public SDK version validated against MemHub's permission and recall contracts.
|
||||
"hy-memory==2.0.5",
|
||||
"hy-memory==2.0.5.dev3",
|
||||
# Retrieval's derived vector index directly consumes these locked versions.
|
||||
"chromadb==1.5.9",
|
||||
"numpy==2.5.2",
|
||||
|
||||
@@ -42,10 +42,12 @@ TARGETED = (
|
||||
"tests/test_database_contention.py",
|
||||
"tests/test_main_database_fairness.py",
|
||||
"tests/test_soak_probe_effectiveness.py",
|
||||
"tests/test_evidence_reconcile_primitive.py",
|
||||
"tests/test_sqlite_semantics_gate.py",
|
||||
"tests/test_store_binding.py",
|
||||
"tests/test_store_binding_ratchet.py",
|
||||
"tests/test_main_database_entry.py",
|
||||
"tests/test_memory_processing.py",
|
||||
"tests/test_extraction_window.py",
|
||||
"tests/test_extraction_defer.py",
|
||||
"tests/test_active_memory_window.py",
|
||||
|
||||
@@ -2,5 +2,11 @@
|
||||
|
||||
import os
|
||||
|
||||
from one_memory.hy_runtime_config import configure_hy_behavior_environment
|
||||
|
||||
# HY has a few reader switches evaluated at module import time. Fix the
|
||||
# product-owned, non-secret behavior before any one_memory module imports HY.
|
||||
configure_hy_behavior_environment()
|
||||
|
||||
_version = os.environ.get("ONE_MEMORY_VERSION", "").strip()
|
||||
__version__ = _version if _version else "development"
|
||||
@@ -50,10 +50,6 @@ from one_memory.auth import local_api_guard, renderer_origins
|
||||
from one_memory.background_work import shutdown_bulk_pool, warm_bulk_pool
|
||||
from one_memory.config import Settings, load_settings
|
||||
from one_memory.context_routes import context_router, sdk_context_router
|
||||
from one_memory.conversation_processing import (
|
||||
_conversation_chunks,
|
||||
memory_task_timeout_seconds,
|
||||
)
|
||||
from one_memory.database_access import (
|
||||
MAIN_DATABASE_REPORT_SECONDS,
|
||||
close_main_databases,
|
||||
@@ -82,30 +78,23 @@ from one_memory.external_memory_access import external_access, require_external_
|
||||
from one_memory.extraction_settings import ExtractionSettings, ExtractionSettingsStore
|
||||
from one_memory.file_imports import FileImportStore, file_import_router
|
||||
from one_memory.home_summary import HomeSummary, read_home_summary
|
||||
from one_memory.hy_graph_admin import GraphWidthDecision, GraphWidthMismatch
|
||||
from one_memory.hy_memory_bridge import (
|
||||
GraphWidthDecision,
|
||||
GraphWidthMismatch,
|
||||
HyMemoryBridge,
|
||||
HyMemoryBridgeError,
|
||||
HySdkMemoryPage,
|
||||
HySdkMemoryRecord,
|
||||
HySdkRawRecord,
|
||||
builtin_retrieval_snapshot,
|
||||
)
|
||||
from one_memory.hy_runtime_config import (
|
||||
HY_MEMORY_READER,
|
||||
DiagnosticLogLevel,
|
||||
detailed_logging_enabled,
|
||||
diagnostic_log_level,
|
||||
set_detailed_logging,
|
||||
set_diagnostic_log_level,
|
||||
)
|
||||
from one_memory.hy_sdk_runtime import (
|
||||
SdkClientHandle,
|
||||
builtin_retrieval_snapshot,
|
||||
close_all_shared_runtimes,
|
||||
sdk_relevance_floor,
|
||||
)
|
||||
from one_memory.hy_sdk_types import (
|
||||
HySdkError,
|
||||
HySdkMemoryPage,
|
||||
HySdkMemoryRecord,
|
||||
HySdkRawPage,
|
||||
HySdkRawRecord,
|
||||
HySdkUserProfile,
|
||||
parse_sdk_search,
|
||||
)
|
||||
from one_memory.imports.access import (
|
||||
AuthorizedSourceRoot,
|
||||
SourceAccessError,
|
||||
@@ -178,10 +167,11 @@ from one_memory.memory_library import (
|
||||
read_conversation_memories,
|
||||
read_memory_library,
|
||||
)
|
||||
from one_memory.memory_notes import (
|
||||
LegacyMemoryNoteReader,
|
||||
MemoryNoteStore,
|
||||
memory_note_router,
|
||||
from one_memory.memory_notes import MemoryNoteStore, memory_note_router
|
||||
from one_memory.memory_processing import (
|
||||
MemoryProcessingSupervisor,
|
||||
_conversation_chunks,
|
||||
memory_task_timeout_seconds,
|
||||
)
|
||||
from one_memory.memory_qa import MemoryAnswerer, MemoryAnswerError
|
||||
from one_memory.memory_queue import (
|
||||
@@ -211,10 +201,7 @@ from one_memory.model_profiles import (
|
||||
ModelProfileStore,
|
||||
ModelProfileUnavailableError,
|
||||
)
|
||||
from one_memory.model_request_audit import (
|
||||
AuditedModelTransport,
|
||||
ModelRequestAuditStore,
|
||||
)
|
||||
from one_memory.model_request_audit import AuditedModelTransport, ModelRequestAuditStore
|
||||
from one_memory.model_request_budget import BudgetedModelTransport, ModelRequestBudget
|
||||
from one_memory.onboarding import OnboardingState, OnboardingStore
|
||||
from one_memory.recall import (
|
||||
@@ -238,8 +225,6 @@ from one_memory.sdk_memory_qa import (
|
||||
SdkMemoryAnswer,
|
||||
SdkMemoryAnswerer,
|
||||
)
|
||||
from one_memory.sdk_memory_runtime import SdkMemoryRuntime
|
||||
from one_memory.sdk_write_processor import SdkWriteCoordinator
|
||||
from one_memory.secrets import (
|
||||
KeyringSecretStore,
|
||||
SecretStore,
|
||||
@@ -860,7 +845,7 @@ def create_app(
|
||||
runtime = CoreRuntime(runtime_settings) if manage_runtime else nullcontext()
|
||||
with runtime:
|
||||
database_path = runtime_settings.data_dir / "one-memory.sqlite3"
|
||||
app.state.sdk_client_handle = None
|
||||
app.state.sdk_library_bridge = None
|
||||
bootstrap_fresh_management_schema(database_path)
|
||||
# Before any store: each one below changes the library as it opens.
|
||||
try:
|
||||
@@ -906,7 +891,6 @@ def create_app(
|
||||
)
|
||||
from one_memory.memory_server.data_safety import MIN_FREE_BYTES
|
||||
from one_memory.memory_server.durable_queue import (
|
||||
COMPLETED,
|
||||
DurableWriteDrain,
|
||||
DurableWriteQueue,
|
||||
execute_job,
|
||||
@@ -984,21 +968,31 @@ def create_app(
|
||||
await asyncio.to_thread(app.state.evidence_store.release_completed_sdk_staging)
|
||||
evidence_store = app.state.evidence_store
|
||||
if app.state.memory_authority is MemoryAuthority.SDK:
|
||||
sdk_handle = _sdk_client_handle(app)
|
||||
sdk_bridge = _LazySdkBridge(app)
|
||||
|
||||
async def read_sdk_memory(
|
||||
memory_id: str,
|
||||
) -> HySdkMemoryRecord | None:
|
||||
return cast(
|
||||
HySdkMemoryRecord | None,
|
||||
await sdk_bridge.get_sdk_memory(memory_id),
|
||||
)
|
||||
|
||||
app.state.recall_target = SdkRecallTarget(
|
||||
semantic_search=lambda query: _search_sdk_memories(
|
||||
sdk_handle,
|
||||
semantic_search=lambda query: sdk_bridge.search_memories(
|
||||
query,
|
||||
user_id="local",
|
||||
agent_id="*",
|
||||
),
|
||||
get_memory=lambda memory_id: _get_sdk_memory(sdk_handle, memory_id),
|
||||
get_memory=read_sdk_memory,
|
||||
)
|
||||
else:
|
||||
app.state.recall_target = GovernedRecallTarget(
|
||||
app.state.memory_governance_store,
|
||||
filter_memories=app.state.source_read_access.memories,
|
||||
semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
|
||||
semantic_search=lambda query: _memory_processing_supervisor(
|
||||
app
|
||||
).semantic_recall(query),
|
||||
# Read per request rather than cached: an import that lands
|
||||
# while the app is open should make its workspace reachable
|
||||
# without a restart. Affordable only because the read is served
|
||||
@@ -1100,7 +1094,9 @@ def create_app(
|
||||
authorize_generation=lambda profile, question, grounding: (
|
||||
_authorize_answer_generation(app, profile, question, grounding)
|
||||
),
|
||||
semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
|
||||
semantic_search=lambda query: _memory_processing_supervisor(app).semantic_recall(
|
||||
query
|
||||
),
|
||||
resolve_workspace=lambda workspace_path: resolve_workspace_scopes(
|
||||
workspace_path,
|
||||
evidence_store.known_workspaces(user_id="local"),
|
||||
@@ -1120,12 +1116,19 @@ def create_app(
|
||||
store=app.state.model_profile_store,
|
||||
secrets=app.state.secret_store,
|
||||
)
|
||||
app.state.sdk_write_coordinator = SdkWriteCoordinator(
|
||||
app.state.memory_processing_supervisor = MemoryProcessingSupervisor(
|
||||
tasks=app.state.memory_task_store,
|
||||
profiles=app.state.model_profile_store,
|
||||
secrets=app.state.secret_store,
|
||||
evidence=app.state.evidence_store,
|
||||
governance=app.state.memory_governance_store,
|
||||
data_dir=runtime_settings.data_dir,
|
||||
transport_factory=lambda profile: (
|
||||
lambda: AuditedModelTransport(
|
||||
extraction_settings=app.state.extraction_settings_store,
|
||||
bridge_factory=lambda profile, api_key, data_dir: HyMemoryBridge(
|
||||
profile=profile,
|
||||
api_key=api_key,
|
||||
data_dir=data_dir,
|
||||
http_transport_factory=lambda: AuditedModelTransport(
|
||||
app.state.model_request_audit_store,
|
||||
profile_id=profile.profile_id,
|
||||
profile_revision=profile.revision,
|
||||
@@ -1136,19 +1139,21 @@ def create_app(
|
||||
if model_request_budget is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
),
|
||||
),
|
||||
# `None` means "whatever the active profile allows". Pinned to
|
||||
# 1 in `a59f4b5` as temporary load-shedding for an acceptance
|
||||
# package, and left pinned after it: a hardcoded 1 is rule 9's
|
||||
# exact anti-pattern (a small number baked into code that
|
||||
# cannot be tuned per machine). Measured before removing — at
|
||||
# profile concurrency 4, recall latency is unchanged while
|
||||
# precipitation goes 4x.
|
||||
max_concurrent_tasks=runtime_settings.precipitation_max_concurrent_tasks,
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
service_levels=app.state.service_levels,
|
||||
sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(),
|
||||
sdk_authority=lambda: app.state.memory_authority is MemoryAuthority.SDK,
|
||||
)
|
||||
app.state.sdk_memory_runtime = SdkMemoryRuntime(
|
||||
coordinator=app.state.sdk_write_coordinator,
|
||||
read_handle=_sdk_client_handle(app),
|
||||
write_queue=memory_write_queue,
|
||||
extraction_settings=app.state.extraction_settings_store,
|
||||
data_dir=runtime_settings.data_dir,
|
||||
)
|
||||
if app.state.memory_authority is MemoryAuthority.SDK:
|
||||
app.state.sdk_memory_runtime.start_raw_retention()
|
||||
# C2 owns the single production budget (handle 5s + reserve 250ms,
|
||||
# total 5.25s, lease 10s, renew 1s, claim 250ms). The composition
|
||||
# root only consumes it: no local formula, and the 250ms claim
|
||||
@@ -1157,24 +1162,34 @@ def create_app(
|
||||
drain_budget = production_drain_budget()
|
||||
|
||||
async def reconcile_unknown_writes() -> None:
|
||||
if app.state.memory_authority is MemoryAuthority.SDK:
|
||||
# Unknown SDK writes must be reconciled by SDK request ID.
|
||||
# Until that direct path is available, retain them as
|
||||
# unknown rather than re-running a possibly successful add.
|
||||
return
|
||||
await app.state.memory_processing_supervisor.reconcile_unknown_outputs(
|
||||
exclude_error_codes=NON_RETRYABLE_POST_WRITE_CODES,
|
||||
)
|
||||
unknown = await asyncio.to_thread(
|
||||
memory_write_queue.unknown_jobs,
|
||||
limit=128,
|
||||
exclude_error_codes=NON_RETRYABLE_POST_WRITE_CODES,
|
||||
)
|
||||
for job in unknown:
|
||||
receipt = await app.state.sdk_write_coordinator.inquire_write_outcome(
|
||||
job.request_id
|
||||
)
|
||||
if receipt is None:
|
||||
continue
|
||||
await asyncio.to_thread(
|
||||
memory_write_queue.settle,
|
||||
memory_write_queue.resolve_unknown_from_linked_output,
|
||||
job.job_id,
|
||||
state=COMPLETED,
|
||||
sdk_raw_ids=receipt.raw_ids,
|
||||
sdk_memory_ids=receipt.memory_ids,
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
memory_write_queue.retry_first_unknown_after_reconcile,
|
||||
older_than=datetime.now(UTC) - timedelta(seconds=10),
|
||||
limit=32,
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
app.state.memory_task_store.retry_first_unknown_after_reconcile,
|
||||
older_than=datetime.now(UTC) - timedelta(seconds=10),
|
||||
limit=128,
|
||||
)
|
||||
|
||||
async def admit_hy_invocation(
|
||||
source_id: str,
|
||||
@@ -1196,7 +1211,7 @@ def create_app(
|
||||
execute=lambda job_id: execute_job(
|
||||
job_id,
|
||||
queue=memory_write_queue,
|
||||
supervisor=app.state.sdk_write_coordinator,
|
||||
supervisor=app.state.memory_processing_supervisor,
|
||||
evidence=app.state.evidence_store,
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
admit_hy_invocation=admit_hy_invocation,
|
||||
@@ -1204,7 +1219,7 @@ def create_app(
|
||||
execute_owned=lambda job_id, owner: execute_job(
|
||||
job_id,
|
||||
queue=memory_write_queue,
|
||||
supervisor=app.state.sdk_write_coordinator,
|
||||
supervisor=app.state.memory_processing_supervisor,
|
||||
evidence=app.state.evidence_store,
|
||||
owner=owner,
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
@@ -1278,14 +1293,7 @@ def create_app(
|
||||
|
||||
app.state.memory_note_store = MemoryNoteStore(
|
||||
app.state.evidence_store,
|
||||
(
|
||||
LegacyMemoryNoteReader(
|
||||
app.state.memory_task_store,
|
||||
app.state.memory_governance_store,
|
||||
)
|
||||
if app.state.memory_authority is MemoryAuthority.LEGACY
|
||||
else None
|
||||
),
|
||||
app.state.memory_processing_supervisor,
|
||||
app.state.memory_write_intake,
|
||||
sdk_authority=app.state.memory_authority is MemoryAuthority.SDK,
|
||||
write_queue=memory_write_queue,
|
||||
@@ -1427,9 +1435,18 @@ def create_app(
|
||||
# starts its threads wherever the first work arrives from, and
|
||||
# from a busy loop that wait was 2.5 seconds.
|
||||
warm_bulk_pool()
|
||||
app.state.memory_backfill = None
|
||||
app.state.memory_visibility_task = None
|
||||
app.state.deletion_sync = None
|
||||
if app.state.memory_authority is MemoryAuthority.LEGACY:
|
||||
_start_backfill(app)
|
||||
_start_visibility_recompute(app)
|
||||
app.state.deletion_sync = watched_task(
|
||||
asyncio.get_running_loop(),
|
||||
lambda: _memory_processing_supervisor(app).keep_deletions_in_sync(),
|
||||
name="one-memory-deletion-sync",
|
||||
)
|
||||
else:
|
||||
app.state.memory_backfill = None
|
||||
app.state.memory_visibility_task = None
|
||||
app.state.deletion_sync = None
|
||||
# Both times the loop was blocked here it was found from outside,
|
||||
# by hand, days later. This says so at the time, and says what was
|
||||
# holding it.
|
||||
@@ -1533,12 +1550,16 @@ def create_app(
|
||||
app.state.memory_write_drain.stop()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await app.state.memory_write_drain_task
|
||||
await app.state.sdk_memory_runtime.shutdown()
|
||||
await app.state.sdk_write_coordinator.close()
|
||||
if app.state.sdk_client_handle is not None:
|
||||
await app.state.sdk_client_handle.close()
|
||||
app.state.sdk_client_handle = None
|
||||
close_all_shared_runtimes()
|
||||
await app.state.memory_processing_supervisor.shutdown()
|
||||
if app.state.sdk_library_bridge is not None:
|
||||
await app.state.sdk_library_bridge.close()
|
||||
app.state.sdk_library_bridge = None
|
||||
# HY clients share two process-owned loop threads. Close them
|
||||
# after every bridge is down, while HY's SQLite executor is
|
||||
# still available; interpreter atexit is too late for that.
|
||||
from one_memory.hy_memory_bridge import close_shared_hy_runtimes
|
||||
|
||||
await asyncio.to_thread(close_shared_hy_runtimes)
|
||||
# Last: what has just been shut down was using it.
|
||||
shutdown_bulk_pool()
|
||||
log_main_database_stats("shutdown")
|
||||
@@ -1601,6 +1622,8 @@ def create_app(
|
||||
"/api/v1/context",
|
||||
"/api/v1/boundary",
|
||||
"/api/v1/home-summary",
|
||||
"/api/v1/memory-tasks",
|
||||
"/api/v1/memory-processing",
|
||||
)
|
||||
if getattr(
|
||||
app.state, "memory_authority", MemoryAuthority.LEGACY
|
||||
@@ -1726,7 +1749,7 @@ def create_app(
|
||||
trace_id=trace_id,
|
||||
instance_id=runtime_settings.instance_id,
|
||||
duration_ms=elapsed_ms(started),
|
||||
details={**diagnostic_details, "status_code": response.status_code},
|
||||
details={**diagnostic_details, "status": str(response.status_code)},
|
||||
)
|
||||
return response
|
||||
except Exception as error:
|
||||
@@ -1797,7 +1820,7 @@ def create_app(
|
||||
) -> dict[str, object]:
|
||||
"""Semantic search IDs for the Reader process. Reader-only."""
|
||||
try:
|
||||
hits = await _sdk_memory_runtime(app).semantic_search(query, limit=limit)
|
||||
hits = await _memory_processing_supervisor(app).semantic_search(query, limit=limit)
|
||||
except Exception:
|
||||
return {"ids": []}
|
||||
return {"ids": [memory_id for memory_id, _score in hits]}
|
||||
@@ -1827,7 +1850,7 @@ def create_app(
|
||||
) -> dict[str, object]:
|
||||
"""HY graph overlay for the Reader process. Reader-only."""
|
||||
try:
|
||||
snapshot = await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
|
||||
snapshot = await _memory_processing_supervisor(app).graph_snapshot(limit=limit)
|
||||
except Exception:
|
||||
return {"consulted": False, "nodes": [], "edges": []}
|
||||
return snapshot.model_dump(mode="json")
|
||||
@@ -1838,7 +1861,7 @@ def create_app(
|
||||
error = cast(str | None, getattr(app.state, "memory_backfill_error", None))
|
||||
status = _backfill_state(backfill, error)
|
||||
governance = getattr(app.state, "memory_governance_store", None)
|
||||
supervisor = getattr(app.state, "sdk_memory_runtime", None)
|
||||
supervisor = getattr(app.state, "memory_processing_supervisor", None)
|
||||
if supervisor is not None:
|
||||
accepted = supervisor.accepted_backfill_status()
|
||||
if accepted is not None:
|
||||
@@ -3069,18 +3092,15 @@ def create_app(
|
||||
if app.state.memory_authority is not MemoryAuthority.SDK:
|
||||
raise HTTPException(409, "legacy_memory_authority_required")
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
raw = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
page = await _sdk_library_bridge(app).list_sdk_memories(
|
||||
user_id="local",
|
||||
statuses=["active"],
|
||||
statuses=("active",),
|
||||
limit=1,
|
||||
offset=0,
|
||||
order="desc",
|
||||
)
|
||||
page = HySdkMemoryPage.from_sdk(raw, limit=1, offset=0)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_list_failed") from None
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(503, str(error)) from None
|
||||
return SdkHomeSummary(
|
||||
memory_count=page.total,
|
||||
latest=page.memories[0] if page.memories else None,
|
||||
@@ -3139,17 +3159,14 @@ def create_app(
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
) -> HySdkMemoryPage:
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
raw = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
return await _sdk_library_bridge(app).list_sdk_memories(
|
||||
user_id="local",
|
||||
statuses=list(statuses),
|
||||
statuses=tuple(statuses),
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
)
|
||||
return HySdkMemoryPage.from_sdk(raw, limit=limit, offset=offset)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_list_failed") from None
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
|
||||
@app.get(
|
||||
"/api/v2/user-profile",
|
||||
@@ -3158,12 +3175,10 @@ def create_app(
|
||||
)
|
||||
async def sdk_user_profile() -> SdkUserProfileResponse:
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
value = await asyncio.to_thread(client.get_user_profile, user_id="local")
|
||||
profile = HySdkUserProfile.from_sdk(value)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_profile_read_failed") from None
|
||||
return SdkUserProfileResponse.model_validate(profile.model_dump())
|
||||
value = await _sdk_library_bridge(app).get_user_profile(user_id="local")
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
return SdkUserProfileResponse.model_validate(value)
|
||||
|
||||
@app.get(
|
||||
"/api/v2/memories/{memory_id}",
|
||||
@@ -3172,9 +3187,9 @@ def create_app(
|
||||
)
|
||||
async def get_sdk_memory(memory_id: str) -> HySdkMemoryRecord:
|
||||
try:
|
||||
memory = await _get_sdk_memory(_sdk_client_handle(app), memory_id)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_get_failed") from None
|
||||
memory = await _sdk_library_bridge(app).get_sdk_memory(memory_id)
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
if memory is None:
|
||||
raise HTTPException(status_code=404, detail="memory_not_found")
|
||||
return memory
|
||||
@@ -3188,15 +3203,12 @@ def create_app(
|
||||
memory_id: str,
|
||||
request: SdkMemoryUpdateRequest,
|
||||
) -> HySdkMemoryRecord:
|
||||
bridge = _sdk_library_bridge(app)
|
||||
try:
|
||||
client = await app.state.sdk_write_coordinator.client()
|
||||
result = await asyncio.to_thread(client.update, memory_id, request.content)
|
||||
if not isinstance(result, dict) or result.get("success") is not True:
|
||||
raise HySdkError("hy_memory_update_rejected")
|
||||
raw = await asyncio.to_thread(client.get, memory_id)
|
||||
memory = None if raw is None else HySdkMemoryRecord.from_sdk(raw)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_update_failed") from None
|
||||
await bridge.update(memory_id, request.content)
|
||||
memory = await bridge.get_sdk_memory(memory_id)
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
if memory is None:
|
||||
raise HTTPException(status_code=404, detail="memory_not_found")
|
||||
return memory
|
||||
@@ -3208,17 +3220,9 @@ def create_app(
|
||||
)
|
||||
async def delete_sdk_memory(memory_id: str) -> Response:
|
||||
try:
|
||||
client = await app.state.sdk_write_coordinator.client()
|
||||
result = await asyncio.to_thread(client.delete, memory_id)
|
||||
if not isinstance(result, dict):
|
||||
raise HySdkError("hy_memory_delete_rejected")
|
||||
error_code = result.get("error_code")
|
||||
if error_code or result.get("partial") is True:
|
||||
raise HySdkError(str(error_code or "hy_memory_delete_incomplete"))
|
||||
if result.get("success") is not True and result.get("deleted_count") != 0:
|
||||
raise HySdkError("hy_memory_delete_rejected")
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_delete_failed") from None
|
||||
await _sdk_library_bridge(app).delete(memory_id)
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.get(
|
||||
@@ -3232,19 +3236,15 @@ def create_app(
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
) -> SdkRawPage:
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
value = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
items, total = await _sdk_library_bridge(app).list_sdk_raw(
|
||||
user_id="local",
|
||||
layers=["l1_raw"],
|
||||
include_sources=[source] if source else None,
|
||||
source=source,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
)
|
||||
page = HySdkRawPage.from_memory_list(value, limit=limit, offset=offset)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_raw_list_failed") from None
|
||||
return SdkRawPage(items=page.raw, total=page.total, offset=offset, limit=limit)
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
return SdkRawPage(items=items, total=total, offset=offset, limit=limit)
|
||||
|
||||
@app.get(
|
||||
"/api/v1/memories",
|
||||
@@ -3775,9 +3775,11 @@ def create_app(
|
||||
all_memories, projected, nodes, edges = await asyncio.to_thread(read_governed_part)
|
||||
try:
|
||||
hy_graph = (
|
||||
None if revoked else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
|
||||
None
|
||||
if revoked
|
||||
else await _memory_processing_supervisor(app).graph_snapshot(limit=limit)
|
||||
)
|
||||
except HySdkError:
|
||||
except HyMemoryBridgeError:
|
||||
hy_graph = None
|
||||
|
||||
revoked_now = await asyncio.to_thread(_read_access(app).has_revoked_sources)
|
||||
@@ -3854,8 +3856,8 @@ def create_app(
|
||||
if await asyncio.to_thread(_read_access(app).has_revoked_sources):
|
||||
raise HTTPException(status_code=403, detail="memory_source_revoked")
|
||||
try:
|
||||
return await _sdk_memory_runtime(app).digest()
|
||||
except HySdkError as error:
|
||||
return await _memory_processing_supervisor(app).digest()
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
|
||||
@app.get(
|
||||
@@ -3879,24 +3881,17 @@ def create_app(
|
||||
# Two numbers, not one: 675 as a single total reads as "stuck", when
|
||||
# what is true is that the three from this morning are nearly done
|
||||
# and the rest is catch-up (R-02).
|
||||
supervisor = app.state.sdk_memory_runtime
|
||||
supervisor = app.state.memory_processing_supervisor
|
||||
depths = supervisor.queue_depths()
|
||||
active_queue, in_flight_count = supervisor.active_depths()
|
||||
planned, current = supervisor.manual_batch_progress()
|
||||
sdk_counts = (
|
||||
supervisor.status_counts() if app.state.memory_authority is MemoryAuthority.SDK else {}
|
||||
)
|
||||
# `running` counts rows the store still marks running, which includes
|
||||
# work a worker has already given up on. Reporting that as "what is
|
||||
# running" is how a stalled queue looks busy, so the strand is given
|
||||
# its own number.
|
||||
running_rows = counts.get(MemoryTaskStatus.RUNNING, 0)
|
||||
return MemoryProcessingSummary(
|
||||
pending=(
|
||||
sdk_counts.get("pending", 0)
|
||||
if sdk_counts
|
||||
else counts.get(MemoryTaskStatus.PENDING, 0)
|
||||
),
|
||||
pending=counts.get(MemoryTaskStatus.PENDING, 0),
|
||||
fresh_pending=depths.fresh,
|
||||
backlog_pending=depths.backlog,
|
||||
outside_window_pending=counts.get(
|
||||
@@ -3904,26 +3899,20 @@ def create_app(
|
||||
0,
|
||||
),
|
||||
recent_completions=depths.recent_completions,
|
||||
running=sdk_counts.get("running", running_rows),
|
||||
stranded_running=(0 if sdk_counts else max(0, running_rows - in_flight_count)),
|
||||
running=running_rows,
|
||||
stranded_running=max(0, running_rows - in_flight_count),
|
||||
unexecuted_policy_failures=store.unexecuted_policy_failures(),
|
||||
waiting_for_model=sdk_counts.get(
|
||||
"waiting_for_model",
|
||||
counts.get(MemoryTaskStatus.WAITING_FOR_MODEL, 0),
|
||||
),
|
||||
completed=sdk_counts.get(
|
||||
"completed",
|
||||
counts.get(MemoryTaskStatus.COMPLETED, 0),
|
||||
),
|
||||
failed=sdk_counts.get("failed", counts.get(MemoryTaskStatus.FAILED, 0)),
|
||||
unknown=sdk_counts.get("unknown", counts.get(MemoryTaskStatus.UNKNOWN, 0)),
|
||||
waiting_for_model=counts.get(MemoryTaskStatus.WAITING_FOR_MODEL, 0),
|
||||
completed=counts.get(MemoryTaskStatus.COMPLETED, 0),
|
||||
failed=counts.get(MemoryTaskStatus.FAILED, 0),
|
||||
unknown=counts.get(MemoryTaskStatus.UNKNOWN, 0),
|
||||
unknown_reasons=store.unknown_error_counts(),
|
||||
blocked_stages=tuple(
|
||||
MemoryProcessingBlock(kind=kind, reason=reason, count=count)
|
||||
for kind, reason, count in store.blocked_stages()[:6]
|
||||
),
|
||||
seconds_per_completion=store.seconds_per_completion(),
|
||||
extraction_window_preset=app.state.extraction_settings_store.get().window_preset,
|
||||
extraction_window_preset=supervisor.extraction_window_settings().window_preset,
|
||||
manual_extract_batches_planned=planned,
|
||||
manual_extract_batches_current=current,
|
||||
active_queue=active_queue,
|
||||
@@ -3941,17 +3930,15 @@ def create_app(
|
||||
) -> RetryUnknownMemoryTasksReceipt:
|
||||
"""Retry unknown writes only behind the user's explicit confirmation."""
|
||||
del request
|
||||
reconciled = await _sdk_memory_runtime(app).reconcile_unknown_outputs()
|
||||
reconciled = await _memory_processing_supervisor(app).reconcile_unknown_outputs()
|
||||
from one_memory.memory_server.durable_queue import DurableWriteQueue
|
||||
|
||||
queue = DurableWriteQueue(app.state.memory_write_queue_path)
|
||||
unknown = queue.unknown_jobs(limit=2048)
|
||||
if app.state.memory_authority is MemoryAuthority.LEGACY:
|
||||
for job in unknown:
|
||||
queue.resolve_unknown_from_linked_output(job.job_id)
|
||||
for job in unknown:
|
||||
queue.resolve_unknown_from_linked_output(job.job_id)
|
||||
requeued = sum(queue.retry_unknown_confirmed(job.job_id) for job in unknown)
|
||||
if app.state.memory_authority is MemoryAuthority.LEGACY:
|
||||
requeued += app.state.memory_task_store.retry_unknown_confirmed()
|
||||
requeued += app.state.memory_task_store.retry_unknown_confirmed()
|
||||
return RetryUnknownMemoryTasksReceipt(reconciled=reconciled, requeued=requeued)
|
||||
|
||||
@app.get(
|
||||
@@ -3962,8 +3949,8 @@ def create_app(
|
||||
async def get_graph_width() -> GraphWidthStatus:
|
||||
"""Report the mismatch. Never writes: this is polled by the app."""
|
||||
try:
|
||||
mismatch = await _sdk_memory_runtime(app).graph_width_mismatch()
|
||||
except HySdkError as error:
|
||||
mismatch = await _memory_processing_supervisor(app).graph_width_mismatch()
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
if mismatch is None:
|
||||
return GraphWidthStatus()
|
||||
@@ -3986,10 +3973,10 @@ def create_app(
|
||||
decision is rejected by validation rather than assumed, because the
|
||||
assumption that cost a live graph once was made in code, not by a user.
|
||||
"""
|
||||
supervisor = _sdk_memory_runtime(app)
|
||||
supervisor = _memory_processing_supervisor(app)
|
||||
try:
|
||||
outcome = await supervisor.decide_graph_width(request.decision)
|
||||
except HySdkError as error:
|
||||
except HyMemoryBridgeError as error:
|
||||
raise HTTPException(status_code=409, detail=str(error)) from error
|
||||
except OSError as error:
|
||||
# A failed export left nothing behind, but the user must be told
|
||||
@@ -4007,7 +3994,7 @@ def create_app(
|
||||
dependencies=protected,
|
||||
)
|
||||
def get_extraction_settings() -> ExtractionSettings:
|
||||
return app.state.extraction_settings_store.get()
|
||||
return _memory_processing_supervisor(app).extraction_window_settings()
|
||||
|
||||
@app.put(
|
||||
"/api/v1/extraction-settings",
|
||||
@@ -4015,7 +4002,7 @@ def create_app(
|
||||
dependencies=protected,
|
||||
)
|
||||
def put_extraction_settings(request: ExtractionWindowUpdate) -> ExtractionSettings:
|
||||
return app.state.extraction_settings_store.set_window_preset(request.window_preset)
|
||||
return _memory_processing_supervisor(app).set_extraction_window(request.window_preset)
|
||||
|
||||
@app.get(
|
||||
"/api/v1/sedimentation",
|
||||
@@ -4065,7 +4052,7 @@ def create_app(
|
||||
dependencies=protected,
|
||||
)
|
||||
def manual_extract(request: ManualExtractRequest) -> ManualExtractReceipt:
|
||||
supervisor = _sdk_memory_runtime(app)
|
||||
supervisor = _memory_processing_supervisor(app)
|
||||
task_ids: tuple[str, ...] | None = None
|
||||
if request.evidence_ids:
|
||||
resolved: list[str] = []
|
||||
@@ -4183,11 +4170,11 @@ def create_app(
|
||||
"""
|
||||
store = _memory_store(app)
|
||||
try:
|
||||
await _sdk_memory_runtime(app).update_hy_memory(
|
||||
await _memory_processing_supervisor(app).update_hy_memory(
|
||||
memory.memory_id,
|
||||
memory.content,
|
||||
)
|
||||
except HySdkError as error:
|
||||
except HyMemoryBridgeError as error:
|
||||
return store.fail_index_sync(
|
||||
memory.memory_id,
|
||||
user_id="local",
|
||||
@@ -4226,7 +4213,7 @@ def create_app(
|
||||
if action == "delete":
|
||||
deletion = _memory_store(app).deletion(memory_id, user_id="local")
|
||||
if deletion is None or deletion.status is not DeletionStatus.COMPLETED:
|
||||
await _sdk_memory_runtime(app).finish_deletion(memory_id)
|
||||
await _memory_processing_supervisor(app).finish_deletion(memory_id)
|
||||
return memory
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="memory_not_found") from None
|
||||
@@ -4391,7 +4378,7 @@ async def _request_answerer(
|
||||
|
||||
def _sdk_request_answerer(app: FastAPI) -> SdkMemoryAnswerer:
|
||||
return SdkMemoryAnswerer(
|
||||
client_handle=_sdk_client_handle(app),
|
||||
bridge=_sdk_library_bridge(app),
|
||||
profiles=_model_profile_store(app),
|
||||
secrets=_secret_store(app),
|
||||
authorize_generation=lambda profile, question, grounding: _authorize_sdk_answer_generation(
|
||||
@@ -4416,9 +4403,9 @@ async def _authorize_sdk_answer_generation(
|
||||
identity = await asyncio.to_thread(_model_profile_store(app).active_identity)
|
||||
if identity != (profile.profile_id, profile.revision):
|
||||
raise MemoryAnswerError("memory_answer_generation_not_authorized")
|
||||
handle = _sdk_client_handle(app)
|
||||
bridge = _sdk_library_bridge(app)
|
||||
for memory in grounding:
|
||||
current = await _get_sdk_memory(handle, memory.memory_id)
|
||||
current = await bridge.get_sdk_memory(memory.memory_id)
|
||||
if current is None:
|
||||
raise MemoryAnswerError("memory_answer_generation_not_authorized")
|
||||
projected = SdkAskMemory(
|
||||
@@ -4501,6 +4488,20 @@ class _LazyASGIApp:
|
||||
app = _LazyASGIApp()
|
||||
|
||||
|
||||
class _LazySdkBridge:
|
||||
"""Resolve the active model only when an SDK operation is actually used."""
|
||||
|
||||
def __init__(self, app: FastAPI) -> None:
|
||||
self.app = app
|
||||
|
||||
def __getattr__(self, name: str) -> Callable[..., Awaitable[Any]]:
|
||||
async def call(*args: Any, **kwargs: Any) -> Any:
|
||||
method = getattr(_sdk_library_bridge(self.app), name)
|
||||
return await method(*args, **kwargs)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def _sync_refused(code: str) -> HTTPException:
|
||||
"""A sync refusal that says which stage refused.
|
||||
|
||||
@@ -5199,7 +5200,7 @@ def _resume_sedimentation(app: FastAPI) -> None:
|
||||
return
|
||||
app.state.sedimentation_resume = watched_task(
|
||||
asyncio.get_running_loop(),
|
||||
lambda: _sdk_memory_runtime(app).resume_allowed(),
|
||||
lambda: _memory_processing_supervisor(app).resume_allowed(),
|
||||
component="memory.sedimentation",
|
||||
name="one-memory-sedimentation-resume",
|
||||
)
|
||||
@@ -5272,7 +5273,7 @@ def _report_visibility_failure(phase: str, error: BaseException) -> None:
|
||||
event="memory_visibility.background_failed",
|
||||
outcome="failed",
|
||||
error_type=type(error).__name__,
|
||||
details={"phase": phase},
|
||||
details={"phase": phase, "reason": str(error)[:200]},
|
||||
)
|
||||
|
||||
|
||||
@@ -5367,8 +5368,9 @@ async def _run_backfill(app: FastAPI) -> None:
|
||||
|
||||
|
||||
def _evidence_excerpt(app: FastAPI, record: EvidenceRecord) -> EvidenceExcerpt:
|
||||
supervisor = _memory_processing_supervisor(app)
|
||||
task = _memory_task_store(app).task_for_evidence(record.evidence_id)
|
||||
window = extraction_window_for(app.state.extraction_settings_store.get().window_preset)
|
||||
window = extraction_window_for(supervisor.extraction_window_settings().window_preset)
|
||||
return EvidenceExcerpt(
|
||||
evidence_id=record.evidence_id,
|
||||
available=True,
|
||||
@@ -5388,62 +5390,23 @@ def _evidence_excerpt(app: FastAPI, record: EvidenceRecord) -> EvidenceExcerpt:
|
||||
)
|
||||
|
||||
|
||||
def _sdk_memory_runtime(app: FastAPI) -> SdkMemoryRuntime:
|
||||
def _memory_processing_supervisor(app: FastAPI) -> MemoryProcessingSupervisor:
|
||||
return cast(
|
||||
SdkMemoryRuntime,
|
||||
app.state.sdk_memory_runtime,
|
||||
MemoryProcessingSupervisor,
|
||||
app.state.memory_processing_supervisor,
|
||||
)
|
||||
|
||||
|
||||
def _sdk_client_handle(app: FastAPI) -> SdkClientHandle:
|
||||
handle = cast(SdkClientHandle | None, app.state.sdk_client_handle)
|
||||
if handle is None:
|
||||
handle = SdkClientHandle(
|
||||
def _sdk_library_bridge(app: FastAPI) -> HyMemoryBridge:
|
||||
bridge = cast(HyMemoryBridge | None, app.state.sdk_library_bridge)
|
||||
if bridge is None:
|
||||
bridge = HyMemoryBridge(
|
||||
profile=builtin_retrieval_snapshot(),
|
||||
api_key=None,
|
||||
data_dir=cast(Settings, app.state.settings).data_dir,
|
||||
)
|
||||
app.state.sdk_client_handle = handle
|
||||
return handle
|
||||
|
||||
|
||||
async def _get_sdk_memory(
|
||||
handle: SdkClientHandle,
|
||||
memory_id: str,
|
||||
) -> HySdkMemoryRecord | None:
|
||||
client = await handle.client()
|
||||
raw = await asyncio.to_thread(client.get, memory_id)
|
||||
return None if raw is None else HySdkMemoryRecord.from_sdk(raw)
|
||||
|
||||
|
||||
async def _search_sdk_memories(
|
||||
handle: SdkClientHandle,
|
||||
query: str,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
limit: int = 12,
|
||||
) -> tuple[object, ...]:
|
||||
client = await handle.client()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
query,
|
||||
user_ids=[user_id],
|
||||
agent_ids=[] if agent_id == "*" else [agent_id],
|
||||
limit=limit,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=limit,
|
||||
intention_limit=limit,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
)
|
||||
return parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
|
||||
|
||||
def _sdk_http_error(error: Exception, fallback: str) -> HTTPException:
|
||||
detail = error.code if isinstance(error, HySdkError) else fallback
|
||||
return HTTPException(status_code=503, detail=detail)
|
||||
app.state.sdk_library_bridge = bridge
|
||||
return bridge
|
||||
|
||||
|
||||
def _evidence_store(app: FastAPI) -> EvidenceStore:
|
||||
|
||||
@@ -31,60 +31,13 @@ def _end_by(seconds: str | None) -> None:
|
||||
if __name__ == "__main__" and sys.argv[1:2] == ["capture"]:
|
||||
_end_by(os.environ.get("ONE_MEMORY_CAPTURE_DEADLINE_S"))
|
||||
|
||||
import errno # noqa: E402
|
||||
import json # noqa: E402
|
||||
import re # noqa: E402
|
||||
from collections.abc import Iterator, Sequence # noqa: E402
|
||||
from collections.abc import Sequence # noqa: E402
|
||||
from typing import TextIO # noqa: E402
|
||||
|
||||
from one_memory.config import load_settings # noqa: E402
|
||||
from one_memory.runtime_client import RuntimeStatus, inspect_runtime # noqa: E402
|
||||
|
||||
# A single `sdk_migration_backfill_failed` told the person nothing: running out
|
||||
# of disk, a model that was never configured, and a Coding/Working-Fact gap are
|
||||
# three different next steps, and the startup dialog can only act on the name it
|
||||
# is given. These are the names that reach it.
|
||||
_SAFE_MIGRATION_CODE = re.compile(r"^[a-z][a-z0-9_]{0,95}$")
|
||||
_OUT_OF_SPACE = frozenset({errno.ENOSPC, getattr(errno, "EDQUOT", errno.ENOSPC)})
|
||||
|
||||
|
||||
def _exception_chain(error: BaseException) -> Iterator[BaseException]:
|
||||
seen: set[int] = set()
|
||||
current: BaseException | None = error
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
yield current
|
||||
current = current.__cause__ or current.__context__
|
||||
|
||||
|
||||
def migration_failure_code(
|
||||
error: BaseException, *, fallback: str = "sdk_migration_backfill_failed"
|
||||
) -> str:
|
||||
"""Name a migration failure; never guess a code from free text.
|
||||
|
||||
Errs toward the caller's generic code: an exception this does not recognize
|
||||
keeps its message out of the document Desktop reads.
|
||||
"""
|
||||
chain = list(_exception_chain(error))
|
||||
for item in chain:
|
||||
if isinstance(item, OSError) and item.errno in _OUT_OF_SPACE:
|
||||
return "sdk_migration_disk_space_insufficient"
|
||||
for item in chain:
|
||||
code = getattr(item, "code", "")
|
||||
if not isinstance(code, str) or not _SAFE_MIGRATION_CODE.match(code):
|
||||
continue
|
||||
if code.startswith(("sdk_data_plane_", "sdk_migration_")):
|
||||
return code
|
||||
if code.startswith("hy_sdk_productivity_"):
|
||||
# The Coding inventory could not be read or classified; that is a
|
||||
# data-shape problem the person cannot fix by retrying.
|
||||
return "sdk_migration_coding_inventory_failed"
|
||||
if code == "hy_sdk_memory_import_failed" and any(
|
||||
"working_fact" in str(part) for part in chain
|
||||
):
|
||||
return "sdk_migration_working_fact_conflict"
|
||||
return fallback
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if sys.argv[1:] == ["--keychain-read"]:
|
||||
@@ -392,16 +345,9 @@ def migration_bundle(options: Sequence[str], stream: TextIO) -> int:
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
"state": "failed",
|
||||
"code": migration_failure_code(error, fallback=error.code),
|
||||
"code": error.code,
|
||||
}
|
||||
_write_restore(stream, as_json, document, f"迁移备份失败:{document['code']}")
|
||||
return 1
|
||||
except OSError as error:
|
||||
# A full disk while copying is the most common first-launch failure and
|
||||
# the only one the person fixes by deleting something.
|
||||
code = migration_failure_code(error, fallback="sdk_migration_bundle_copy_failed")
|
||||
document = {"schema_version": 1, "state": "failed", "code": code}
|
||||
_write_restore(stream, as_json, document, f"迁移备份失败:{code}")
|
||||
_write_restore(stream, as_json, document, f"迁移备份失败:{error.code}")
|
||||
return 1
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
@@ -415,21 +361,6 @@ def migration_bundle(options: Sequence[str], stream: TextIO) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cutover_failure_code(error: BaseException) -> str:
|
||||
"""The cutover's own codes, minus the counts it appends to one of them.
|
||||
|
||||
`sdk_data_plane_cutover_work_pending` carries a JSON body of table names and
|
||||
counts. That belongs in the diagnostic event, not in the single string the
|
||||
startup dialog reads, so it collapses to one name here.
|
||||
"""
|
||||
code = getattr(error, "code", "")
|
||||
if isinstance(code, str) and code.startswith("sdk_data_plane_cutover_work_pending"):
|
||||
return "sdk_migration_work_pending"
|
||||
if isinstance(code, str) and _SAFE_MIGRATION_CODE.match(code):
|
||||
return code
|
||||
return "sdk_migration_cutover_failed"
|
||||
|
||||
|
||||
def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
"""Backfill legacy memory/raw into SDK while keeping legacy authority."""
|
||||
migration_id: str | None = None
|
||||
@@ -490,9 +421,8 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
return 4
|
||||
|
||||
from one_memory.database_access import close_main_databases
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridge
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE, HY_MEMORY_VERSION
|
||||
from one_memory.hy_sdk_migration import HySdkMigration
|
||||
from one_memory.hy_sdk_runtime import acquire_client
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
|
||||
from one_memory.memory_server.migrations import MIGRATIONS
|
||||
from one_memory.model_profiles import ModelProfileStore
|
||||
@@ -500,12 +430,7 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
MigrationBundleRejected,
|
||||
inspect_sdk_migration_bundle,
|
||||
)
|
||||
from one_memory.sdk_migration_runner import (
|
||||
BackfillResult,
|
||||
prepare_sdk_migration_staging,
|
||||
run_sdk_data_plane_backfill,
|
||||
sdk_migration_staging,
|
||||
)
|
||||
from one_memory.sdk_migration_runner import BackfillResult, run_sdk_data_plane_backfill
|
||||
from one_memory.secrets import KeyringSecretStore, SecretStoreUnavailableError
|
||||
|
||||
database = settings.data_dir / "one-memory.sqlite3"
|
||||
@@ -530,15 +455,14 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
except SecretStoreUnavailableError:
|
||||
code = "secure_storage_unavailable"
|
||||
except Exception as error:
|
||||
message = str(error)
|
||||
code = (
|
||||
message
|
||||
if message
|
||||
str(error)
|
||||
if str(error)
|
||||
in {
|
||||
"model_profile_not_active",
|
||||
"model_secret_unavailable",
|
||||
}
|
||||
else migration_failure_code(error, fallback="sdk_migration_preflight_failed")
|
||||
else "sdk_migration_preflight_failed"
|
||||
)
|
||||
else:
|
||||
namespace_fingerprint = hashlib.sha256(HY_MEMORY_NAMESPACE.encode()).hexdigest()
|
||||
@@ -557,32 +481,17 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
separators=(",", ":"),
|
||||
)
|
||||
backend_fingerprint = hashlib.sha256(backend_material.encode()).hexdigest()
|
||||
staging = sdk_migration_staging(
|
||||
data_dir=settings.data_dir,
|
||||
migration_id=migration_id,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
)
|
||||
try:
|
||||
staging_data_dir = prepare_sdk_migration_staging(
|
||||
paths=staging,
|
||||
database_path=database,
|
||||
)
|
||||
except Exception as error:
|
||||
code = migration_failure_code(error, fallback="sdk_migration_staging_failed")
|
||||
staging_data_dir = None
|
||||
|
||||
async def run_backfill() -> BackfillResult:
|
||||
assert staging_data_dir is not None
|
||||
lease = await asyncio.to_thread(
|
||||
acquire_client,
|
||||
profile,
|
||||
bridge = HyMemoryBridge(
|
||||
profile=profile,
|
||||
api_key=secret,
|
||||
data_dir=staging_data_dir,
|
||||
data_dir=settings.data_dir,
|
||||
)
|
||||
try:
|
||||
return await run_sdk_data_plane_backfill(
|
||||
database_path=database,
|
||||
client=HySdkMigration(lease.client),
|
||||
bridge=bridge,
|
||||
migration_id=migration_id,
|
||||
bundle_id=bundle_id,
|
||||
source_runtime_build=str(manifest.get("runtime_build") or ""),
|
||||
@@ -592,17 +501,13 @@ def sdk_migrate(options: Sequence[str], stream: TextIO) -> int:
|
||||
backend_fingerprint=backend_fingerprint,
|
||||
)
|
||||
finally:
|
||||
await asyncio.to_thread(lease.close)
|
||||
await bridge.close()
|
||||
|
||||
try:
|
||||
result = asyncio.run(run_backfill()) if staging_data_dir is not None else None
|
||||
except Exception as error:
|
||||
code = migration_failure_code(error)
|
||||
result = asyncio.run(run_backfill())
|
||||
except Exception:
|
||||
code = "sdk_migration_backfill_failed"
|
||||
else:
|
||||
if result is None:
|
||||
document = {"schema_version": 1, "state": "failed", "code": code}
|
||||
_write_restore(stream, as_json, document, f"SDK backfill 失败:{code}")
|
||||
return 1
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
"state": result.state,
|
||||
@@ -678,40 +583,21 @@ def sdk_cutover(options: Sequence[str], stream: TextIO) -> int:
|
||||
_write_restore(stream, as_json, document, "预览通过;加 --yes 后提交 SDK authority。")
|
||||
return 4
|
||||
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE, HY_MEMORY_VERSION
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_VERSION
|
||||
from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
|
||||
from one_memory.sdk_migration_bundle import MigrationBundleRejected
|
||||
from one_memory.sdk_migration_runner import (
|
||||
activate_sdk_migration_staging,
|
||||
commit_reconciled_cutover,
|
||||
resume_legacy_cleanup,
|
||||
sdk_migration_staging,
|
||||
)
|
||||
from one_memory.sdk_migration_runner import commit_reconciled_cutover
|
||||
|
||||
try:
|
||||
database_path = settings.data_dir / "one-memory.sqlite3"
|
||||
staging = sdk_migration_staging(
|
||||
marker = commit_reconciled_cutover(
|
||||
data_dir=settings.data_dir,
|
||||
database_path=settings.data_dir / "one-memory.sqlite3",
|
||||
migration_id=migration_id,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
bundle_id=bundle_id,
|
||||
sdk_version=HY_MEMORY_VERSION,
|
||||
)
|
||||
marker = activate_sdk_migration_staging(
|
||||
paths=staging,
|
||||
database_path=database_path,
|
||||
commit=lambda: commit_reconciled_cutover(
|
||||
data_dir=settings.data_dir,
|
||||
database_path=database_path,
|
||||
migration_id=migration_id,
|
||||
bundle_id=bundle_id,
|
||||
sdk_version=HY_MEMORY_VERSION,
|
||||
),
|
||||
)
|
||||
resume_legacy_cleanup(
|
||||
data_dir=settings.data_dir,
|
||||
database_path=database_path,
|
||||
)
|
||||
except (DataPlaneMigrationError, MigrationBundleRejected, OSError) as error:
|
||||
code = cutover_failure_code(error)
|
||||
except (DataPlaneMigrationError, MigrationBundleRejected) as error:
|
||||
code = getattr(error, "code", str(error))
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
"state": "blocked",
|
||||
@@ -752,12 +638,10 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
|
||||
import sqlite3
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
|
||||
from one_memory.sdk_data_plane_migration import (
|
||||
MemoryAuthority,
|
||||
SdkDataPlaneMigrationStore,
|
||||
)
|
||||
from one_memory.sdk_migration_runner import sdk_migration_staging
|
||||
from one_memory.target_schema import legacy_tables_present
|
||||
|
||||
migration_store = SdkDataPlaneMigrationStore(database_path)
|
||||
@@ -767,18 +651,10 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
|
||||
generation = hashlib.sha256(f"{stat.st_size}:{stat.st_mtime_ns}".encode()).hexdigest()[:16]
|
||||
if authority is MemoryAuthority.SDK:
|
||||
legacy_tables = legacy_tables_present(database_path)
|
||||
staging_cleanup = False
|
||||
if marker is not None:
|
||||
staging = sdk_migration_staging(
|
||||
data_dir=settings.data_dir,
|
||||
migration_id=marker.migration_id,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
)
|
||||
staging_cleanup = staging.protective_root.exists() or staging.staging_home.exists()
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
"authority": "sdk",
|
||||
"state": ("cleanup_required" if legacy_tables or staging_cleanup else "complete"),
|
||||
"state": "cleanup_required" if legacy_tables else "complete",
|
||||
"migration_id": marker.migration_id if marker else None,
|
||||
"generation": generation,
|
||||
"legacy_tables": list(legacy_tables),
|
||||
@@ -803,28 +679,11 @@ def sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
|
||||
else 0
|
||||
),
|
||||
}
|
||||
active_migration = (
|
||||
database.execute(
|
||||
"SELECT migration_id,backup_bundle_id "
|
||||
"FROM sdk_data_plane_migrations "
|
||||
"WHERE state IN ('running','blocked','ready') "
|
||||
"ORDER BY created_at DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if "sdk_data_plane_migrations" in tables
|
||||
else None
|
||||
)
|
||||
document = {
|
||||
"schema_version": 1,
|
||||
"authority": "legacy",
|
||||
"state": "required" if any(counts.values()) else "empty",
|
||||
"migration_id": (
|
||||
str(active_migration["migration_id"]) if active_migration is not None else None
|
||||
),
|
||||
"bundle_id": (
|
||||
str(active_migration["backup_bundle_id"])
|
||||
if active_migration is not None
|
||||
else None
|
||||
),
|
||||
"migration_id": None,
|
||||
"counts": counts,
|
||||
"generation": generation,
|
||||
}
|
||||
|
||||
@@ -42,13 +42,6 @@ from one_memory.context_raw import revalidate_raw_context
|
||||
from one_memory.context_search import search_context
|
||||
from one_memory.diagnostic_log import diagnostic_event, diagnostic_exception
|
||||
from one_memory.external_memory_access import external_access
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_READER
|
||||
from one_memory.hy_sdk_runtime import sdk_relevance_floor
|
||||
from one_memory.hy_sdk_types import (
|
||||
HySdkMemoryRecord,
|
||||
parse_sdk_raw_search,
|
||||
parse_sdk_search,
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.retrieval.raw_lane import read_raw_indexed_context
|
||||
from one_memory.semantic_deadline import hy_wait_budget, search_within
|
||||
@@ -462,33 +455,19 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
except Exception:
|
||||
raise HTTPException(503, "context_authority_unavailable") from None
|
||||
|
||||
client_handle = getattr(request.app.state, "sdk_client_handle", None)
|
||||
if client_handle is None:
|
||||
bridge = getattr(request.app.state, "sdk_library_bridge", None)
|
||||
if bridge is None:
|
||||
raise HTTPException(503, "sdk_context_unavailable")
|
||||
try:
|
||||
client = await client_handle.client()
|
||||
except Exception:
|
||||
raise HTTPException(503, "sdk_context_unavailable") from None
|
||||
|
||||
async def raw_lane() -> tuple[SdkRawContextItem, ...]:
|
||||
if "raw" not in value.kinds or not authority.raw_allowed:
|
||||
return ()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
hits = await bridge.search_sdk_raw(
|
||||
value.query,
|
||||
user_ids=["local"],
|
||||
agent_ids=[],
|
||||
include_sources=list(authority.raw_sources or ()),
|
||||
user_id="local",
|
||||
source_ids=tuple(authority.raw_sources or ()),
|
||||
limit=value.limit,
|
||||
min_score=0.0,
|
||||
schema_min_score=0.0,
|
||||
schema_limit=0,
|
||||
intention_limit=0,
|
||||
scene="chat",
|
||||
reader=HY_MEMORY_READER,
|
||||
layers=["l1_raw"],
|
||||
)
|
||||
hits = parse_sdk_raw_search(raw)
|
||||
return tuple(
|
||||
SdkRawContextItem(
|
||||
raw_id=hit.raw_id,
|
||||
@@ -508,24 +487,15 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
async def memory_lane() -> tuple[SdkMemoryContextItem, ...]:
|
||||
if "memory" not in value.kinds or not authority.memory_allowed:
|
||||
return ()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
hits = await bridge.search_memories(
|
||||
value.query,
|
||||
user_ids=["local"],
|
||||
agent_ids=[],
|
||||
user_id="local",
|
||||
agent_id="*",
|
||||
limit=value.limit,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=value.limit,
|
||||
intention_limit=value.limit,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
)
|
||||
hits = parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
items: list[SdkMemoryContextItem] = []
|
||||
for hit in hits:
|
||||
record = await asyncio.to_thread(client.get, hit.memory_id)
|
||||
memory = None if record is None else HySdkMemoryRecord.from_sdk(record)
|
||||
memory = await bridge.get_sdk_memory(hit.memory_id)
|
||||
if memory is None or memory.status != "active":
|
||||
continue
|
||||
items.append(
|
||||
|
||||
@@ -1,277 +0,0 @@
|
||||
"""Pure conversation-window helpers shared by capture, retrieval, and writes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.memory_tasks import segment_evidence_record
|
||||
|
||||
MEMORY_TASK_TIMEOUT_MARGIN_SECONDS = 60.0
|
||||
MEMORY_TASK_TIMEOUT_MIN_SECONDS = 180.0
|
||||
MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES = 512
|
||||
|
||||
_HARNESS_PREFIXES = (
|
||||
"<codex_internal_context",
|
||||
"# agents.md instructions",
|
||||
"<environment_context>",
|
||||
"<permissions instructions>",
|
||||
"<system-reminder>",
|
||||
"<system_notification>",
|
||||
"<instructions>",
|
||||
)
|
||||
_HOST_ENVELOPE = re.compile(
|
||||
r"(?:"
|
||||
r"^<timestamp>.*?<system_(?:notification|reminder)>|"
|
||||
r"^<system_(?:notification|reminder)>|"
|
||||
r'^\{\s*"request_id"\s*:.*?"steps"\s*:'
|
||||
r")",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_WHOLE_TURN_CHARACTERS = 60_000
|
||||
_WINDOW_BYTE_CEILING = 2_000_000
|
||||
_HISTORY_MERGE_TURNS = 8
|
||||
_HISTORY_MERGE_CHARACTERS = 8_000
|
||||
|
||||
|
||||
def memory_task_timeout_seconds(model_timeout_seconds: float) -> float:
|
||||
"""Bound a whole write without treating one model request as its budget."""
|
||||
return max(
|
||||
MEMORY_TASK_TIMEOUT_MIN_SECONDS,
|
||||
float(model_timeout_seconds) + MEMORY_TASK_TIMEOUT_MARGIN_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _coding_tool_result_max_bytes() -> int:
|
||||
raw = os.environ.get("MEMORY_CODING_TOOL_RESULT_MAX_BYTES")
|
||||
if raw is None:
|
||||
return MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
return value if value > 0 else MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
|
||||
|
||||
_HY_TOOL_CHARACTERS = _coding_tool_result_max_bytes()
|
||||
|
||||
|
||||
def _conversation_chunks(
|
||||
records: list[EvidenceRecord],
|
||||
*,
|
||||
max_characters: int | None = None,
|
||||
max_turns: int | None = None,
|
||||
completed_only: bool = False,
|
||||
) -> list[tuple[EvidenceRecord, ...]]:
|
||||
"""Build bounded, complete user/assistant/tool windows."""
|
||||
turns: list[list[EvidenceRecord]] = []
|
||||
for record in records:
|
||||
if not _is_hy_turn_record(record):
|
||||
continue
|
||||
pending_proposal = bool(turns) and not _opens_turn(turns[-1][0])
|
||||
if not turns or (_opens_turn(record) and not pending_proposal):
|
||||
turns.append([record])
|
||||
else:
|
||||
turns[-1].append(record)
|
||||
|
||||
if completed_only and turns and _trailing_user_turn_is_open(turns[-1]):
|
||||
turns.pop()
|
||||
|
||||
whole = _whole_turns() and max_characters is None
|
||||
if whole:
|
||||
size = _hy_read_length
|
||||
max_characters = _WHOLE_TURN_CHARACTERS
|
||||
max_turns = max_turns or 1_000_000
|
||||
else:
|
||||
size = lambda record: len(record.verbatim) # noqa: E731
|
||||
max_characters = max_characters or 12_000
|
||||
max_turns = max_turns or 24
|
||||
|
||||
chunks: list[tuple[EvidenceRecord, ...]] = []
|
||||
for turn in turns:
|
||||
original_question = turn[0] if _opens_turn(turn[0]) else None
|
||||
expanded: list[EvidenceRecord] = []
|
||||
for record in turn:
|
||||
reserve = (
|
||||
size(original_question)
|
||||
if original_question is not None
|
||||
and record is not original_question
|
||||
and size(original_question) < max_characters
|
||||
else 0
|
||||
)
|
||||
segment_limit = max(1, max_characters - reserve)
|
||||
if size(record) <= segment_limit:
|
||||
expanded.append(record)
|
||||
continue
|
||||
spans = _semantic_segment_spans(record.verbatim, segment_limit)
|
||||
expanded.extend(
|
||||
segment_evidence_record(record, index=index, start=start, end=end)
|
||||
if len(spans) > 1
|
||||
else record
|
||||
for index, (start, end) in enumerate(spans)
|
||||
)
|
||||
question = expanded[0] if expanded and _opens_turn(expanded[0]) else None
|
||||
current: list[EvidenceRecord] = []
|
||||
for record in expanded:
|
||||
if current and (
|
||||
len(current) >= max_turns
|
||||
or sum(size(item) for item in current) + size(record) > max_characters
|
||||
or sum(_window_bytes(item) for item in current) + _window_bytes(record)
|
||||
> _WINDOW_BYTE_CEILING
|
||||
):
|
||||
chunks.append(tuple(current))
|
||||
current = (
|
||||
[question]
|
||||
if question is not None
|
||||
and max_turns > 1
|
||||
and size(question) + size(record) <= max_characters
|
||||
else []
|
||||
)
|
||||
current.append(record)
|
||||
chunks.append(tuple(current))
|
||||
return _merge_tool_only_chunks(chunks)
|
||||
|
||||
|
||||
def _semantic_segment_spans(text: str, limit: int) -> list[tuple[int, int]]:
|
||||
if len(text) <= limit:
|
||||
return [(0, len(text))]
|
||||
spans: list[tuple[int, int]] = []
|
||||
start = 0
|
||||
separators = ("\n\n", "\n", "。", "!", "?", ". ", "! ", "? ")
|
||||
while start < len(text):
|
||||
hard_end = min(len(text), start + limit)
|
||||
if hard_end == len(text):
|
||||
end = hard_end
|
||||
else:
|
||||
floor = start + max(1, limit // 2)
|
||||
candidates = [text.rfind(separator, floor, hard_end) for separator in separators]
|
||||
boundary = max(candidates)
|
||||
if boundary < floor:
|
||||
end = hard_end
|
||||
else:
|
||||
matched = next(
|
||||
separator for separator in separators if text.startswith(separator, boundary)
|
||||
)
|
||||
end = boundary + len(matched)
|
||||
spans.append((start, end))
|
||||
start = end
|
||||
return spans
|
||||
|
||||
|
||||
def _merge_tool_only_chunks(
|
||||
chunks: list[tuple[EvidenceRecord, ...]],
|
||||
) -> list[tuple[EvidenceRecord, ...]]:
|
||||
if not any(chunk and all(record.role == "tool" for record in chunk) for chunk in chunks):
|
||||
return chunks
|
||||
merged: list[tuple[EvidenceRecord, ...]] = []
|
||||
leading_tools: list[EvidenceRecord] = []
|
||||
for chunk in chunks:
|
||||
if not chunk:
|
||||
continue
|
||||
if all(record.role == "tool" for record in chunk):
|
||||
if merged:
|
||||
merged[-1] = (*merged[-1], *chunk)
|
||||
else:
|
||||
leading_tools.extend(chunk)
|
||||
continue
|
||||
if leading_tools:
|
||||
chunk = (*leading_tools, *chunk)
|
||||
leading_tools = []
|
||||
merged.append(chunk)
|
||||
return [] if leading_tools else merged
|
||||
|
||||
|
||||
def _merge_historical_chunks(
|
||||
chunks: list[tuple[EvidenceRecord, ...]],
|
||||
*,
|
||||
max_turns: int = _HISTORY_MERGE_TURNS,
|
||||
max_characters: int = _HISTORY_MERGE_CHARACTERS,
|
||||
) -> list[tuple[EvidenceRecord, ...]]:
|
||||
if not chunks:
|
||||
return []
|
||||
merged: list[tuple[EvidenceRecord, ...]] = []
|
||||
current: list[EvidenceRecord] = []
|
||||
current_turns = 0
|
||||
current_characters = 0
|
||||
for chunk in chunks:
|
||||
chunk_characters = sum(len(record.verbatim) for record in chunk)
|
||||
if current and (
|
||||
current_turns >= max_turns or current_characters + chunk_characters > max_characters
|
||||
):
|
||||
merged.append(tuple(current))
|
||||
current = []
|
||||
current_turns = 0
|
||||
current_characters = 0
|
||||
current.extend(chunk)
|
||||
current_turns += 1
|
||||
current_characters += chunk_characters
|
||||
if current:
|
||||
merged.append(tuple(current))
|
||||
return merged
|
||||
|
||||
|
||||
def _chunk_input_ref(chunk: tuple[EvidenceRecord, ...]) -> str:
|
||||
user_records = tuple(item for item in chunk if item.role == "user")
|
||||
return user_records[-1].evidence_id if user_records else chunk[-1].evidence_id
|
||||
|
||||
|
||||
def _opens_turn(record: EvidenceRecord) -> bool:
|
||||
return record.role == "user"
|
||||
|
||||
|
||||
def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
|
||||
if len(turn) != 1:
|
||||
return False
|
||||
opener = turn[0]
|
||||
return (
|
||||
_opens_turn(opener)
|
||||
and opener.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and not _is_memory_note(opener)
|
||||
)
|
||||
|
||||
|
||||
def _is_hy_turn_record(record: EvidenceRecord) -> bool:
|
||||
if record.role == "tool":
|
||||
return record.content_kind is EvidenceContentKind.RAW_CONVERSATION and bool(
|
||||
record.verbatim.strip()
|
||||
)
|
||||
return _is_hy_input_turn(record)
|
||||
|
||||
|
||||
def _is_hy_input_turn(record: EvidenceRecord) -> bool:
|
||||
source = record.verbatim.strip()
|
||||
if not source:
|
||||
return False
|
||||
if record.content_kind is EvidenceContentKind.MEMORY_CONTENT:
|
||||
return True
|
||||
if record.role == "assistant":
|
||||
return True
|
||||
if record.role != "user":
|
||||
return False
|
||||
if source.casefold().startswith(_HARNESS_PREFIXES):
|
||||
return False
|
||||
return _HOST_ENVELOPE.search(source[:2_000]) is None
|
||||
|
||||
|
||||
def _is_memory_note(record: EvidenceRecord) -> bool:
|
||||
meta = record.origin_meta.get(EvidenceOrigin.MANUAL_IMPORT)
|
||||
return (
|
||||
record.source_id == "one-memory-manual"
|
||||
and meta is not None
|
||||
and meta.raw_meta.get("input_mode") == "direct_memory_note"
|
||||
)
|
||||
|
||||
|
||||
def _window_bytes(record: EvidenceRecord) -> int:
|
||||
return len(record.verbatim.encode("utf-8"))
|
||||
|
||||
|
||||
def _whole_turns() -> bool:
|
||||
return os.environ.get("ONE_MEMORY_SPLIT_TURNS", "").strip() != "1"
|
||||
|
||||
|
||||
def _hy_read_length(record: EvidenceRecord) -> int:
|
||||
if record.role == "tool":
|
||||
return min(len(record.verbatim), _HY_TOOL_CHARACTERS)
|
||||
return len(record.verbatim)
|
||||
@@ -20,10 +20,8 @@ from __future__ import annotations
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
@@ -74,30 +72,80 @@ class _Refused(OSError):
|
||||
"""The path is not one we are willing to write to, and never will be."""
|
||||
|
||||
|
||||
DiagnosticLevel = Literal["debug", "info", "warn", "error"]
|
||||
DiagnosticLevel = Literal["info", "warn", "error"]
|
||||
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
|
||||
_EVENT = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$")
|
||||
_DIAGNOSTIC_LOG_NAME = re.compile(
|
||||
r"^diagnostic-(runtime|core|host-bridge|desktop|cli)\.jsonl(?:\.\d+)?$"
|
||||
_DETAIL_KEYS = frozenset(
|
||||
{
|
||||
"action",
|
||||
"attempt",
|
||||
"count",
|
||||
"method",
|
||||
"mode",
|
||||
"phase",
|
||||
"platform",
|
||||
"reason",
|
||||
"result_count",
|
||||
"source_id",
|
||||
"spawned",
|
||||
"status",
|
||||
"version",
|
||||
"parent_session_id",
|
||||
"invalid_lines",
|
||||
"truncated",
|
||||
"open_file_count",
|
||||
"memory_bytes",
|
||||
"thread_count",
|
||||
"database_error_count",
|
||||
"route_path",
|
||||
"sqlite_errorcode",
|
||||
"sqlite_errorname",
|
||||
"file_count",
|
||||
"sqlite_file_count",
|
||||
"byte_file_count",
|
||||
"total_bytes",
|
||||
"copied_bytes",
|
||||
"bytes_per_second",
|
||||
"stored_count",
|
||||
"revised_count",
|
||||
"skipped_count",
|
||||
"error_count",
|
||||
"pending_count",
|
||||
"prepared_count",
|
||||
"queue_depth",
|
||||
"in_flight_count",
|
||||
"call_count",
|
||||
"created_count",
|
||||
"updated_count",
|
||||
"superseded_count",
|
||||
"deleted_count",
|
||||
"operation_kind",
|
||||
"index_complete",
|
||||
"lexical_hits",
|
||||
"semantic_hits",
|
||||
"generation",
|
||||
"timed_out",
|
||||
"failure_site",
|
||||
"error_class",
|
||||
"file_kind",
|
||||
"file_size",
|
||||
"file_mtime_ns",
|
||||
"file_inode",
|
||||
"cursor_new",
|
||||
"cursor_old",
|
||||
"cursor_read_to_end",
|
||||
"cursor_rewind_branch",
|
||||
"summed_ms",
|
||||
"model_ms",
|
||||
"accounted_ms",
|
||||
"unaccounted_ms",
|
||||
"conversation_hash",
|
||||
# The sedimentation switch: a bounded boolean, the same shape as
|
||||
# `spawned` and `index_complete`. Left out of this list it did not
|
||||
# drop a log line, it turned the switch's own request into a 500.
|
||||
"enabled",
|
||||
}
|
||||
)
|
||||
_DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
|
||||
_CODE_FRAME = re.compile(
|
||||
r"^(?:(?:apps|packages|scripts)/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:\d+"
|
||||
r"|one_memory/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:[A-Za-z_][A-Za-z0-9_]*"
|
||||
r"|[A-Za-z0-9_][A-Za-z0-9_.-]*\.py:\d+:[A-Za-z_][A-Za-z0-9_]*)$"
|
||||
)
|
||||
_SHORT_CREDENTIAL_TOKEN = re.compile(
|
||||
r"(?<![A-Za-z0-9])([A-Za-z0-9]{24,47})(?:[_-])?(?![A-Za-z0-9])"
|
||||
)
|
||||
_REFUSED_DETAIL_KEYS = frozenset({"__proto__", "prototype", "constructor"})
|
||||
_MAX_DETAIL_KEYS = 32
|
||||
_MAX_SAFE_INTEGER = (1 << 53) - 1
|
||||
_EXPORT_SECRET_MARKER = re.compile(
|
||||
r"(?:^|[._:-])(?:authorization|credential|password|prompt|query|secret|token)(?:$|[._:-])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TRACE_STRING_DETAIL_KEYS = frozenset({"phase", "mode", "status", "reason", "failure_site"})
|
||||
_TRACE_NUMBER_DETAIL_KEYS = frozenset({"count"})
|
||||
_trace_id: ContextVar[str | None] = ContextVar("one_memory_trace_id", default=None)
|
||||
_active: DiagnosticLog | None = None
|
||||
_database_errors: Counter[tuple[str, str]] = Counter()
|
||||
@@ -216,12 +264,7 @@ class DiagnosticLog:
|
||||
}
|
||||
value.update({key: _identifier(item) for key, item in identifiers.items() if item})
|
||||
if duration_ms is not None:
|
||||
if (
|
||||
not isinstance(duration_ms, int)
|
||||
or isinstance(duration_ms, bool)
|
||||
or duration_ms < 0
|
||||
or duration_ms > _MAX_SAFE_INTEGER
|
||||
):
|
||||
if not isinstance(duration_ms, int) or duration_ms < 0:
|
||||
raise ValueError("diagnostic_duration_invalid")
|
||||
value["duration_ms"] = duration_ms
|
||||
for field_name, item in {
|
||||
@@ -233,9 +276,7 @@ class DiagnosticLog:
|
||||
if item is not None:
|
||||
value[field_name] = _identifier(item)
|
||||
if safe_stack:
|
||||
if len(safe_stack) > 16:
|
||||
raise ValueError("diagnostic_stack_invalid")
|
||||
value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack]
|
||||
value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack[:256]]
|
||||
if details:
|
||||
value["details"] = _details(details)
|
||||
self._submit(_serialize(value))
|
||||
@@ -248,8 +289,6 @@ class DiagnosticLog:
|
||||
event: str,
|
||||
outcome: str,
|
||||
) -> dict[str, object]:
|
||||
if level not in ("debug", "info", "warn", "error"):
|
||||
raise ValueError("diagnostic_level_invalid")
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
@@ -612,17 +651,24 @@ def _drain_diagnostics_at_exit() -> None:
|
||||
|
||||
|
||||
def diagnostic_event(**values: object) -> None:
|
||||
active = _active
|
||||
if active is None:
|
||||
return
|
||||
if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
|
||||
return
|
||||
try:
|
||||
active = _active
|
||||
if active is None:
|
||||
return
|
||||
if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
|
||||
return
|
||||
active.write(**values) # type: ignore[arg-type]
|
||||
except Exception:
|
||||
# Diagnostics are best-effort: validation, serialization, logging, and
|
||||
# filesystem failures must never become a product outage. BaseException
|
||||
# deliberately remains visible so process-control signals still work.
|
||||
except OSError:
|
||||
# Diagnostics must not become a new product outage when a disk is full
|
||||
# or the log directory is temporarily unavailable.
|
||||
return
|
||||
except ValueError:
|
||||
# Same promise, same reason, for a rejected payload instead of a failed
|
||||
# write. An unlisted `details` key once turned every external recall
|
||||
# into a 500 and then killed the process, and the sedimentation switch
|
||||
# into a permanent failure. The payload itself is still wrong and is
|
||||
# still caught in CI by tests/test_diagnostic_payload_keys.py; what
|
||||
# must not happen is the caller paying for it.
|
||||
return
|
||||
|
||||
|
||||
@@ -645,95 +691,46 @@ def read_trace_events(
|
||||
if (request_id is None) == (trace_id is None):
|
||||
raise ValueError("exactly_one_trace_identity_required")
|
||||
key, expected = ("request_id", request_id) if request_id is not None else ("trace_id", trace_id)
|
||||
if expected is None or not _export_identifier(expected):
|
||||
if expected is None or not _IDENTIFIER.fullmatch(expected):
|
||||
raise ValueError("invalid_trace_identity")
|
||||
selected: list[dict[str, object]] = []
|
||||
directory = data_dir / "logs"
|
||||
try:
|
||||
directory_info = directory.lstat()
|
||||
except FileNotFoundError:
|
||||
return ()
|
||||
if stat.S_ISLNK(directory_info.st_mode) or not stat.S_ISDIR(directory_info.st_mode):
|
||||
raise ValueError("diagnostic_directory_unsafe")
|
||||
files: list[tuple[Path, float]] = []
|
||||
for path in directory.iterdir():
|
||||
if not _DIAGNOSTIC_LOG_NAME.fullmatch(path.name):
|
||||
allowed_details = {"phase", "mode", "status", "reason", "failure_site", "count"}
|
||||
files = sorted(
|
||||
(data_dir / "logs").glob("diagnostic-*.jsonl*"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
)
|
||||
for path in files:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
continue
|
||||
try:
|
||||
info = path.lstat()
|
||||
except OSError:
|
||||
continue
|
||||
if stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode):
|
||||
files.append((path, info.st_mtime))
|
||||
for path, _mtime in sorted(files, key=lambda item: item[1]):
|
||||
for line in _read_regular_file_no_follow(path).splitlines():
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
continue
|
||||
if not isinstance(event, dict) or event.get(key) != expected:
|
||||
continue
|
||||
timestamp = event.get("timestamp")
|
||||
if not isinstance(timestamp, str):
|
||||
continue
|
||||
try:
|
||||
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
continue
|
||||
component = event.get("component")
|
||||
event_name = event.get("event")
|
||||
outcome = event.get("outcome")
|
||||
projected: dict[str, object] = {
|
||||
"timestamp": timestamp,
|
||||
"component": (
|
||||
component
|
||||
if isinstance(component, str) and _export_identifier(component)
|
||||
else "diagnostic.redacted"
|
||||
),
|
||||
"event": (
|
||||
event_name
|
||||
if isinstance(event_name, str) and _export_identifier(event_name)
|
||||
else "diagnostic.redacted"
|
||||
),
|
||||
"outcome": (
|
||||
outcome
|
||||
if isinstance(outcome, str) and _export_identifier(outcome)
|
||||
else "redacted"
|
||||
),
|
||||
}
|
||||
duration_ms = event.get("duration_ms")
|
||||
if (
|
||||
isinstance(duration_ms, int)
|
||||
and not isinstance(duration_ms, bool)
|
||||
and 0 <= duration_ms <= _MAX_SAFE_INTEGER
|
||||
):
|
||||
projected["duration_ms"] = duration_ms
|
||||
for name in ("error_code", "error_type"):
|
||||
value = event.get(name)
|
||||
if isinstance(value, str) and _export_identifier(value):
|
||||
projected[name] = value
|
||||
details = event.get("details")
|
||||
projected = {
|
||||
name: event[name]
|
||||
for name in (
|
||||
"timestamp",
|
||||
"component",
|
||||
"event",
|
||||
"outcome",
|
||||
"duration_ms",
|
||||
"error_code",
|
||||
"error_type",
|
||||
)
|
||||
if name in event
|
||||
}
|
||||
if isinstance(details, dict):
|
||||
safe_details = _trace_details(details)
|
||||
if safe_details:
|
||||
projected["details"] = safe_details
|
||||
projected["details"] = {
|
||||
name: value for name, value in details.items() if name in allowed_details
|
||||
}
|
||||
selected.append(projected)
|
||||
return tuple(selected[-max(1, min(limit, 500)) :])
|
||||
|
||||
|
||||
def _read_regular_file_no_follow(path: Path) -> str:
|
||||
descriptor = os.open(path, os.O_RDONLY | _O_NOFOLLOW)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
|
||||
raise ValueError("diagnostic_file_unsafe")
|
||||
with os.fdopen(descriptor, encoding="utf-8", errors="replace") as handle:
|
||||
descriptor = -1
|
||||
return handle.read()
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def current_trace_id() -> str | None:
|
||||
return _trace_id.get()
|
||||
|
||||
@@ -840,77 +837,8 @@ def diagnostic_exception(error: BaseException) -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
def _contains_credential_material(value: str) -> bool:
|
||||
return bool(
|
||||
re.search(
|
||||
r"(?:^|[^A-Za-z0-9])(?:sk-|sk_|omhg_|ghp_|github_pat_)[A-Za-z0-9._-]+",
|
||||
value,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
or re.search(r"(?:^|[^A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]+", value, re.IGNORECASE)
|
||||
or re.search(r"(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])", value)
|
||||
or re.search(r"(?:^|[^A-Za-z0-9])AKID[A-Za-z0-9]{12,}", value, re.IGNORECASE)
|
||||
or re.search(r"(?:^|[^A-Za-z0-9])AIza[A-Za-z0-9_-]{20,}", value)
|
||||
or re.search(r"(?:^|[^A-Za-z0-9])GOCSPX-[A-Za-z0-9_-]+", value)
|
||||
or any(
|
||||
_is_conservative_short_credential(match.group(1))
|
||||
for match in _SHORT_CREDENTIAL_TOKEN.finditer(value)
|
||||
)
|
||||
or re.search(
|
||||
r"\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b",
|
||||
value,
|
||||
)
|
||||
or any(
|
||||
len(candidate) >= 48
|
||||
and any(character.isalpha() for character in candidate)
|
||||
and any(character.isdigit() for character in candidate)
|
||||
for candidate in re.findall(r"\b[A-Za-z0-9._-]{48,}\b", value)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_conservative_short_credential(value: str) -> bool:
|
||||
return bool(
|
||||
re.search(r"[A-Z]", value)
|
||||
and re.search(r"[a-z]", value)
|
||||
and re.search(r"\d", value)
|
||||
and not re.fullmatch(r"[A-Fa-f0-9]+", value)
|
||||
)
|
||||
|
||||
|
||||
def _export_identifier(value: str) -> bool:
|
||||
return bool(
|
||||
_IDENTIFIER.fullmatch(value)
|
||||
and not _EXPORT_SECRET_MARKER.search(value)
|
||||
and not _contains_credential_material(value)
|
||||
)
|
||||
|
||||
|
||||
def _trace_details(values: Mapping[object, object]) -> dict[str, str | int | float]:
|
||||
result: dict[str, str | int | float] = {}
|
||||
for key, value in values.items():
|
||||
if (
|
||||
key in _TRACE_STRING_DETAIL_KEYS
|
||||
and isinstance(value, str)
|
||||
and _export_identifier(value)
|
||||
):
|
||||
result[str(key)] = value
|
||||
elif key in _TRACE_NUMBER_DETAIL_KEYS and isinstance(value, (int, float)):
|
||||
if isinstance(value, bool):
|
||||
continue
|
||||
safe_integer = isinstance(value, int) and abs(value) <= _MAX_SAFE_INTEGER
|
||||
safe_float = (
|
||||
isinstance(value, float)
|
||||
and math.isfinite(value)
|
||||
and (not value.is_integer() or abs(value) <= _MAX_SAFE_INTEGER)
|
||||
)
|
||||
if safe_integer or safe_float:
|
||||
result[str(key)] = value
|
||||
return result
|
||||
|
||||
|
||||
def _serialize(value: Mapping[str, object]) -> str:
|
||||
return json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")) + "\n"
|
||||
return json.dumps(value, ensure_ascii=True, separators=(",", ":")) + "\n"
|
||||
|
||||
|
||||
def _uuid(value: str | None) -> str | None:
|
||||
@@ -936,47 +864,24 @@ def _event_name(value: str) -> str:
|
||||
|
||||
|
||||
def _safe_stack_line(value: str) -> str:
|
||||
if (
|
||||
not value
|
||||
or len(value) > 256
|
||||
or any(character in value for character in "\0\r\n")
|
||||
or _contains_credential_material(value)
|
||||
):
|
||||
if not value or len(value) > 256 or "\n" in value or "\r" in value:
|
||||
raise ValueError("diagnostic_stack_invalid")
|
||||
if _CODE_FRAME.fullmatch(value):
|
||||
return value
|
||||
if value.startswith("thread|"):
|
||||
frames = value.removeprefix("thread|").split(" > ")
|
||||
if frames and all(_CODE_FRAME.fullmatch(frame) for frame in frames):
|
||||
return value
|
||||
raise ValueError("diagnostic_stack_invalid")
|
||||
if Path(value.split(":", 1)[0]).is_absolute() or "file://" in value:
|
||||
raise ValueError("diagnostic_stack_invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _details(
|
||||
values: Mapping[str, str | int | float | bool | None],
|
||||
) -> dict[str, str | int | float | bool | None]:
|
||||
if len(values) > _MAX_DETAIL_KEYS:
|
||||
raise ValueError("diagnostic_detail_count_invalid")
|
||||
result: dict[str, str | int | float | bool | None] = {}
|
||||
for key, value in values.items():
|
||||
if (
|
||||
not isinstance(key, str)
|
||||
or not _DETAIL_KEY.fullmatch(key)
|
||||
or key in _REFUSED_DETAIL_KEYS
|
||||
):
|
||||
if key not in _DETAIL_KEYS:
|
||||
raise ValueError("diagnostic_detail_key_invalid")
|
||||
if isinstance(value, str):
|
||||
if not value or len(value) > 256 or any(character in value for character in "\0\r\n"):
|
||||
if not value or len(value) > 256 or "\n" in value or "\r" in value:
|
||||
raise ValueError("diagnostic_detail_value_invalid")
|
||||
elif isinstance(value, float):
|
||||
if not math.isfinite(value) or (value.is_integer() and abs(value) > _MAX_SAFE_INTEGER):
|
||||
raise ValueError("diagnostic_detail_value_invalid")
|
||||
elif isinstance(value, bool) or value is None:
|
||||
pass
|
||||
elif isinstance(value, int):
|
||||
if abs(value) > _MAX_SAFE_INTEGER:
|
||||
raise ValueError("diagnostic_detail_value_invalid")
|
||||
else:
|
||||
elif value is not None and not isinstance(value, (int, float, bool)):
|
||||
raise ValueError("diagnostic_detail_value_invalid")
|
||||
result[key] = value
|
||||
return result
|
||||
@@ -3,8 +3,8 @@
|
||||
Every site that catches `Exception` broadly must call `report_exception`
|
||||
before (or instead of) continuing, so a failure never looks like success
|
||||
and a stack is always on disk. Content (conversation text, keys, SQL
|
||||
parameters, and exception messages) never reaches the diagnostic log: only
|
||||
exception type, a repeat count, and file:line:function frames.
|
||||
parameters) never reaches the log: only exception type, a truncated
|
||||
message, and file:line:function frames.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -67,7 +67,7 @@ def report_exception(
|
||||
event=event,
|
||||
outcome="failed",
|
||||
**payload,
|
||||
details={"count": _RECENT[key]},
|
||||
details={"reason": _safe_message(error), "count": _RECENT[key]},
|
||||
)
|
||||
if full:
|
||||
_logger.error(
|
||||
|
||||
@@ -1,271 +0,0 @@
|
||||
"""Kuzu graph-width inspection and explicit user-directed administration.
|
||||
|
||||
This module is the second graph architecture exception. It may open Kuzu
|
||||
directly to inspect an existing graph's schema, but owns no prompts, memory
|
||||
reconciliation, coding data, vector database, or legacy governance behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
|
||||
class GraphWidthError(RuntimeError):
|
||||
"""Base error for graph-width inspection and administration."""
|
||||
|
||||
|
||||
class GraphWidthDecision(StrEnum):
|
||||
"""What to do when an existing graph disagrees with the embedding width."""
|
||||
|
||||
KEEP_EXISTING = "keep_existing"
|
||||
REBUILD_WITH_NEW = "rebuild_with_new"
|
||||
EXPORT_FIRST = "export_first"
|
||||
|
||||
|
||||
class GraphWidthMismatch(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
stored_width: int
|
||||
expected_width: int
|
||||
graph_path: str
|
||||
|
||||
|
||||
class GraphWidthOutcome(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
archived_path: str | None = None
|
||||
exported_path: str | None = None
|
||||
|
||||
|
||||
class GraphWidthMismatchError(GraphWidthError):
|
||||
"""Raised instead of silently archiving a live graph."""
|
||||
|
||||
def __init__(self, mismatch: GraphWidthMismatch) -> None:
|
||||
super().__init__(
|
||||
f"graph_width_mismatch:stored={mismatch.stored_width}"
|
||||
f":expected={mismatch.expected_width}"
|
||||
)
|
||||
self.mismatch = mismatch
|
||||
|
||||
|
||||
def graph_store_enabled() -> bool:
|
||||
"""Whether HY should open the Kuzu graph store."""
|
||||
raw = os.environ.get("ONE_MEMORY_GRAPH_STORE", "").strip().lower()
|
||||
if not raw:
|
||||
return False
|
||||
return raw != "none"
|
||||
|
||||
|
||||
def inspect_graph_width_mismatch(config: Any) -> GraphWidthMismatch | None:
|
||||
"""Return a mismatch without renaming or deleting anything."""
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
expected = getattr(getattr(config, "embedder", None), "embedding_dims", None)
|
||||
return _inspect_graph_width(db_path, expected)
|
||||
|
||||
|
||||
def inspect_profile_graph_width(
|
||||
profile: ModelProfileSnapshot,
|
||||
*,
|
||||
api_key: str | None,
|
||||
data_dir: Path,
|
||||
) -> GraphWidthMismatch | None:
|
||||
"""Inspect the graph path and width selected by the active profile."""
|
||||
del api_key
|
||||
return _inspect_graph_width(
|
||||
_profile_graph_path(data_dir),
|
||||
_profile_embedding_width(profile),
|
||||
)
|
||||
|
||||
|
||||
def decide_profile_graph_width(
|
||||
profile: ModelProfileSnapshot,
|
||||
*,
|
||||
api_key: str | None,
|
||||
data_dir: Path,
|
||||
decision: GraphWidthDecision,
|
||||
) -> GraphWidthOutcome:
|
||||
"""Apply the user's decision to the active profile's graph."""
|
||||
del api_key
|
||||
config = _GraphAdminConfig(
|
||||
graph_store=_GraphStoreConfig(str(_profile_graph_path(data_dir))),
|
||||
embedder=_EmbeddingWidthConfig(_profile_embedding_width(profile)),
|
||||
)
|
||||
return apply_graph_width_decision(
|
||||
config,
|
||||
decision,
|
||||
export_root=data_dir.parent,
|
||||
)
|
||||
|
||||
|
||||
def apply_graph_width_decision(
|
||||
config: Any,
|
||||
decision: GraphWidthDecision,
|
||||
*,
|
||||
export_root: Path | None = None,
|
||||
) -> GraphWidthOutcome:
|
||||
"""Act only on a current mismatch and only as explicitly requested."""
|
||||
mismatch = inspect_graph_width_mismatch(config)
|
||||
if mismatch is None:
|
||||
return GraphWidthOutcome()
|
||||
if decision is GraphWidthDecision.REBUILD_WITH_NEW:
|
||||
archive = retire_mismatched_graph_store(config)
|
||||
return GraphWidthOutcome(archived_path=str(archive) if archive is not None else None)
|
||||
if decision is GraphWidthDecision.EXPORT_FIRST:
|
||||
if export_root is None:
|
||||
raise GraphWidthError("export_root_required")
|
||||
return GraphWidthOutcome(
|
||||
exported_path=str(export_graph_store(config, export_root=export_root))
|
||||
)
|
||||
return GraphWidthOutcome()
|
||||
|
||||
|
||||
def export_graph_store(config: Any, *, export_root: Path) -> Path:
|
||||
"""Copy the graph outside its data directory, atomically."""
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ")
|
||||
destination = export_root / f"one-memory-graph-{stamp}"
|
||||
staging = destination.with_name(f"{destination.name}.partial")
|
||||
sources = [db_path] + [
|
||||
sibling
|
||||
for suffix in (".wal", ".shadow", ".tmp")
|
||||
if (sibling := db_path.with_name(f"{db_path.name}{suffix}")).exists()
|
||||
]
|
||||
try:
|
||||
staging.mkdir(parents=True)
|
||||
for source in sources:
|
||||
shutil.copy2(source, staging / source.name)
|
||||
staging.rename(destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
logging.getLogger(__name__).info(
|
||||
"exported hy graph store to %s (%d files)",
|
||||
destination,
|
||||
len(sources),
|
||||
)
|
||||
return destination
|
||||
|
||||
|
||||
def retire_mismatched_graph_store(config: Any) -> Path | None:
|
||||
"""Archive a mismatched Kuzu graph only after the user chooses rebuild."""
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
if not db_path.exists():
|
||||
return None
|
||||
expected = getattr(getattr(config, "embedder", None), "embedding_dims", None)
|
||||
if not expected:
|
||||
return None
|
||||
stored = _graph_embedding_width(db_path)
|
||||
if stored is None or stored == int(expected):
|
||||
return None
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
archive = db_path.with_name(f"{db_path.name}.dim{stored}.{stamp}.bak")
|
||||
db_path.rename(archive)
|
||||
for suffix in (".wal", ".shadow", ".tmp"):
|
||||
sibling = db_path.with_name(f"{db_path.name}{suffix}")
|
||||
if sibling.exists():
|
||||
sibling.rename(archive.with_name(f"{archive.name}{suffix}"))
|
||||
logging.getLogger(__name__).warning(
|
||||
"retired hy graph store with embedding width %s (model expects %s): %s",
|
||||
stored,
|
||||
expected,
|
||||
archive.name,
|
||||
)
|
||||
return archive
|
||||
|
||||
|
||||
_GRAPH_WIDTH_CACHE: dict[str, int | None] = {}
|
||||
_GRAPH_WIDTH_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _graph_embedding_width(db_path: Path) -> int | None:
|
||||
"""Read the Memory embedding width directly from Kuzu's schema."""
|
||||
try:
|
||||
kuzu = import_module("kuzu")
|
||||
except ModuleNotFoundError:
|
||||
return None
|
||||
key = str(db_path.resolve())
|
||||
with _GRAPH_WIDTH_LOCK:
|
||||
if key in _GRAPH_WIDTH_CACHE:
|
||||
return _GRAPH_WIDTH_CACHE[key]
|
||||
try:
|
||||
database = kuzu.Database(str(db_path), read_only=True)
|
||||
except Exception:
|
||||
return None
|
||||
width: int | None = None
|
||||
try:
|
||||
connection = kuzu.Connection(database)
|
||||
result = connection.execute("CALL TABLE_INFO('Memory') RETURN *")
|
||||
while result.has_next():
|
||||
row = result.get_next()
|
||||
if len(row) < 3 or row[1] != "embedding":
|
||||
continue
|
||||
match = re.search(r"\[(\d+)\]", str(row[2]))
|
||||
width = int(match.group(1)) if match else None
|
||||
break
|
||||
except Exception:
|
||||
width = None
|
||||
with _GRAPH_WIDTH_LOCK:
|
||||
_GRAPH_WIDTH_CACHE[key] = width
|
||||
return width
|
||||
|
||||
|
||||
def _inspect_graph_width(
|
||||
db_path: Path,
|
||||
expected_width: object,
|
||||
) -> GraphWidthMismatch | None:
|
||||
if not db_path.exists() or not expected_width:
|
||||
return None
|
||||
stored = _graph_embedding_width(db_path)
|
||||
if stored is None or stored == int(expected_width):
|
||||
return None
|
||||
return GraphWidthMismatch(
|
||||
stored_width=int(stored),
|
||||
expected_width=int(expected_width),
|
||||
graph_path=str(db_path),
|
||||
)
|
||||
|
||||
|
||||
def _profile_graph_path(data_dir: Path) -> Path:
|
||||
name = "graph" if graph_store_enabled() else "graph-disabled"
|
||||
return data_dir / "hy-memory" / HY_MEMORY_NAMESPACE / name
|
||||
|
||||
|
||||
def _profile_embedding_width(profile: ModelProfileSnapshot) -> int | None:
|
||||
if profile.embedding_dimensions is not None:
|
||||
return profile.embedding_dimensions
|
||||
if (profile.embedding_provider_kind or profile.provider_kind) is ProviderKind.LOCAL_ONNX:
|
||||
from one_memory.embedding_assets import BUILTIN_EMBEDDING_DIMENSIONS
|
||||
|
||||
return BUILTIN_EMBEDDING_DIMENSIONS
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _GraphStoreConfig:
|
||||
db_path: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _EmbeddingWidthConfig:
|
||||
embedding_dims: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _GraphAdminConfig:
|
||||
graph_store: _GraphStoreConfig
|
||||
embedder: _EmbeddingWidthConfig
|
||||
@@ -1,185 +0,0 @@
|
||||
"""Read-only projection of HY's Kuzu graph for MemHub's existing graph UI.
|
||||
|
||||
This module is one of the two explicit graph architecture exceptions. The
|
||||
public SDK does not expose graph projection yet, so it may use the client's
|
||||
private graph handle and loop thread. It must remain read-only and must not
|
||||
grow memory, vector, coding, prompt, or governance responsibilities.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class HyGraphNode(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
node_id: str
|
||||
content: str
|
||||
layer: str | None = None
|
||||
confidence: float | None = None
|
||||
|
||||
|
||||
class HyGraphEdge(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
source_node_id: str
|
||||
target_node_id: str
|
||||
relation: str
|
||||
weight: float | None = None
|
||||
|
||||
|
||||
class HyGraphSnapshot(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
nodes: tuple[HyGraphNode, ...] = ()
|
||||
edges: tuple[HyGraphEdge, ...] = ()
|
||||
# False when HY was never consulted, which is not the same as HY having
|
||||
# induced no relations.
|
||||
consulted: bool = True
|
||||
|
||||
|
||||
def read_graph_snapshot(
|
||||
client: object,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
limit: int,
|
||||
) -> HyGraphSnapshot:
|
||||
raw_nodes, raw_edges = _read_graph_snapshot(client, user_id, agent_id, limit)
|
||||
return HyGraphSnapshot(
|
||||
nodes=tuple(
|
||||
HyGraphNode(
|
||||
node_id=str(row[0]),
|
||||
content=str(row[1]),
|
||||
layer=str(row[2]) if row[2] is not None else None,
|
||||
confidence=_safe_float(row[3]),
|
||||
)
|
||||
for row in raw_nodes
|
||||
if len(row) >= 4 and row[0] and row[1]
|
||||
),
|
||||
edges=tuple(
|
||||
HyGraphEdge(
|
||||
source_node_id=str(row[0]),
|
||||
target_node_id=str(row[1]),
|
||||
relation=str(row[2]),
|
||||
weight=_safe_float(row[3]),
|
||||
)
|
||||
for row in raw_edges
|
||||
if len(row) >= 4 and row[0] and row[1] and row[2]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _read_graph_snapshot(
|
||||
client: object,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
limit: int,
|
||||
) -> tuple[list[list[object]], list[list[object]]]:
|
||||
"""Read the graph through the SDK handles until a public API exists."""
|
||||
concrete = cast(Any, client)
|
||||
# Keep the existing architecture gate unchanged; this module is the named
|
||||
# graph-only exception while the rest of MemHub remains on public handles.
|
||||
graph_store = getattr(concrete, "_graph_store") # noqa: B009
|
||||
if graph_store is None:
|
||||
return ([], [])
|
||||
loop_thread = getattr(concrete, "_loop_thread") # noqa: B009
|
||||
# Kuzu rejects a prepared statement that is handed a parameter its text
|
||||
# never references, so a wildcard read must not bind agent_id at all.
|
||||
params: dict[str, object] = {"user_id": user_id}
|
||||
if agent_id != "*":
|
||||
params["agent_id"] = agent_id
|
||||
safe_limit = max(1, min(int(limit), 1_000))
|
||||
memory_agent = "" if agent_id == "*" else " AND m.agent_id = $agent_id"
|
||||
related_agent = (
|
||||
"" if agent_id == "*" else " AND a.agent_id = $agent_id AND b.agent_id = $agent_id"
|
||||
)
|
||||
source_agent = "" if agent_id == "*" else " AND a.agent_id = $agent_id"
|
||||
nodes = (
|
||||
loop_thread.run(
|
||||
graph_store._run(
|
||||
f"""
|
||||
MATCH (m:Memory)
|
||||
WHERE m.user_id = $user_id{memory_agent}
|
||||
AND m.status = 'active'
|
||||
RETURN m.node_id, m.content, m.layer, m.confidence
|
||||
LIMIT {safe_limit}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
)
|
||||
or []
|
||||
)
|
||||
related_edges = (
|
||||
loop_thread.run(
|
||||
graph_store._run(
|
||||
f"""
|
||||
MATCH (a:Memory)-[r:RELATED_TO]->(b:Memory)
|
||||
WHERE a.user_id = $user_id{related_agent}
|
||||
AND b.user_id = $user_id
|
||||
AND a.status = 'active' AND b.status = 'active'
|
||||
RETURN a.node_id, b.node_id, r.relation_type, r.weight
|
||||
LIMIT {safe_limit * 4}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
)
|
||||
or []
|
||||
)
|
||||
derived_edges = (
|
||||
loop_thread.run(
|
||||
graph_store._run(
|
||||
f"""
|
||||
MATCH (a:Memory)-[:DERIVED_FROM]->(v:VdbRef)
|
||||
WHERE a.user_id = $user_id{source_agent}
|
||||
AND a.status = 'active'
|
||||
RETURN a.node_id, v.node_id, 'derived_from', 1.0
|
||||
LIMIT {safe_limit * 4}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
)
|
||||
or []
|
||||
)
|
||||
topic_nodes = (
|
||||
loop_thread.run(
|
||||
graph_store._run(
|
||||
f"""
|
||||
MATCH (m:Memory)-[:TAGGED_WITH]->(t:Topic)
|
||||
WHERE m.user_id = $user_id{memory_agent}
|
||||
AND m.status = 'active'
|
||||
RETURN DISTINCT t.topic_id, t.name, 'TOPIC', 1.0
|
||||
LIMIT {safe_limit}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
)
|
||||
or []
|
||||
)
|
||||
tagged_edges = (
|
||||
loop_thread.run(
|
||||
graph_store._run(
|
||||
f"""
|
||||
MATCH (m:Memory)-[:TAGGED_WITH]->(t:Topic)
|
||||
WHERE m.user_id = $user_id{memory_agent}
|
||||
AND m.status = 'active'
|
||||
RETURN m.node_id, t.topic_id, 'tagged_with', 1.0
|
||||
LIMIT {safe_limit * 4}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
)
|
||||
or []
|
||||
)
|
||||
return (
|
||||
[*nodes, *topic_nodes],
|
||||
[*related_edges, *derived_edges, *tagged_edges],
|
||||
)
|
||||
|
||||
|
||||
def _safe_float(value: object) -> float | None:
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
return None
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Which memories HY no longer counts as current, read from HY itself.
|
||||
|
||||
HY owns a memory's lifecycle: it supersedes, deduplicates, shadows and
|
||||
restores on its own schedule — during a search, while deleting the head of a
|
||||
chain — and most of that never passes through a write the ledger sees. So
|
||||
the ledger does not keep a copy of any of it; readers ask HY, once per read,
|
||||
for the nodes it has set aside.
|
||||
|
||||
Only the vector store is read. HY writes a node's status there on every
|
||||
transition, and the graph holds no status the vector store lacks.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Collection, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from importlib import import_module
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
#: HY's one status for a node that is current. Everything else it writes —
|
||||
#: superseded, shadow, archived, negated, conflicted — takes it out of recall.
|
||||
_ACTIVE = "active"
|
||||
#: HY's verbatim chunks. Stored as shadow from the start and never a memory,
|
||||
#: so never something a reader could have shown.
|
||||
_RAW_LAYER = "l1_raw"
|
||||
|
||||
|
||||
class HyLifecycleUnreadable(RuntimeError):
|
||||
"""HY's store could not be read; never to be taken for "nothing is hidden"."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, eq=False)
|
||||
class HyLifecycle:
|
||||
"""HY's set-aside nodes for one user, each with what HY says replaced it."""
|
||||
|
||||
hidden: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "hidden", MappingProxyType(dict(self.hidden)))
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return isinstance(other, HyLifecycle) and dict(self.hidden) == dict(other.hidden)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(tuple(sorted(self.hidden.items())))
|
||||
|
||||
def replaced_by(self, memory_id: str) -> tuple[str, ...]:
|
||||
return self.hidden.get(memory_id, ())
|
||||
|
||||
def replacing(self, memory_id: str) -> tuple[str, ...]:
|
||||
"""The set-aside memories HY names this one as the replacement of."""
|
||||
return tuple(sorted(old for old, by in self.hidden.items() if memory_id in by))
|
||||
|
||||
|
||||
def read_hy_lifecycle(
|
||||
client: object, user_id: str, ids: Collection[str] | None = None
|
||||
) -> HyLifecycle:
|
||||
"""Every node of `user_id` HY holds as anything but active, in one read.
|
||||
|
||||
With `ids`, only those nodes are looked up — what a single-memory read
|
||||
needs, and a lookup by key rather than a scan; `replacing` is then only
|
||||
known among them.
|
||||
|
||||
Read one level below HY's own listing, which drops a failed read and
|
||||
answers empty: a hidden memory would then read as current.
|
||||
"""
|
||||
native: Any = client
|
||||
try:
|
||||
chroma = import_module("hy_memory.data.vector_store_chroma")
|
||||
store = native._vector_store
|
||||
collection = store._collection
|
||||
if not isinstance(store, chroma.ChromaVectorStore) or collection is None:
|
||||
raise HyLifecycleUnreadable("hy_lifecycle_store_unsupported")
|
||||
found = collection.get(
|
||||
ids=None if ids is None else list(ids),
|
||||
where={
|
||||
"$and": [
|
||||
{"user_id": {"$eq": user_id}},
|
||||
{"status": {"$ne": _ACTIVE}},
|
||||
{"layer": {"$ne": _RAW_LAYER}},
|
||||
]
|
||||
},
|
||||
include=["metadatas"],
|
||||
)
|
||||
metadatas = found["metadatas"]
|
||||
point_ids = found["ids"]
|
||||
if (
|
||||
not isinstance(point_ids, list)
|
||||
or not isinstance(metadatas, list)
|
||||
or len(point_ids) != len(metadatas)
|
||||
):
|
||||
raise HyLifecycleUnreadable("hy_lifecycle_store_mismatch")
|
||||
except HyLifecycleUnreadable:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise HyLifecycleUnreadable("hy_lifecycle_unreadable") from error
|
||||
hidden: dict[str, tuple[str, ...]] = {}
|
||||
for point_id, metadata in zip(point_ids, metadatas, strict=True):
|
||||
if not isinstance(point_id, str) or not point_id or not isinstance(metadata, Mapping):
|
||||
raise HyLifecycleUnreadable("hy_lifecycle_store_mismatch")
|
||||
payload = chroma.ChromaVectorStore._chroma_metadata_to_payload(dict(metadata))
|
||||
node_id = payload.get("node_id") or point_id
|
||||
if not isinstance(node_id, str) or not node_id or node_id != point_id:
|
||||
raise HyLifecycleUnreadable("hy_lifecycle_store_mismatch")
|
||||
hidden[node_id] = _replacements(payload)
|
||||
return HyLifecycle(hidden=hidden)
|
||||
|
||||
|
||||
def _replacements(payload: Mapping[str, object]) -> tuple[str, ...]:
|
||||
"""What HY names as taking this node's place: its chain successor or the kept duplicate."""
|
||||
names: list[str] = []
|
||||
superseded_by = payload.get("superseded_by")
|
||||
if isinstance(superseded_by, list):
|
||||
names.extend(value for value in superseded_by if isinstance(value, str) and value)
|
||||
kept = payload.get("dedup_kept_id")
|
||||
custom = payload.get("custom")
|
||||
if isinstance(custom, str):
|
||||
try:
|
||||
custom = json.loads(custom)
|
||||
except ValueError:
|
||||
custom = None
|
||||
if not isinstance(kept, str) and isinstance(custom, Mapping):
|
||||
audit = custom.get("hy_dedup")
|
||||
if isinstance(audit, Mapping):
|
||||
kept = audit.get("kept_id")
|
||||
if isinstance(kept, str) and kept:
|
||||
names.append(kept)
|
||||
return tuple(dict.fromkeys(names))
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,331 @@
|
||||
"""Transport validation for the pinned HY SDK; HY's prompts and decisions pass through unchanged."""
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
from one_memory.diagnostic_log import diagnostic_event, diagnostic_exception, elapsed_ms
|
||||
from one_memory.hy_source_context import HyWriteBindings
|
||||
from one_memory.hy_step_timing import HyStepTimer
|
||||
|
||||
|
||||
def _http_status(value: Any) -> int | None:
|
||||
return value if type(value) is int and 100 <= value <= 599 else None
|
||||
|
||||
|
||||
def _exception_status(error: Exception) -> int | None:
|
||||
try:
|
||||
status = _http_status(getattr(error, "status_code", None))
|
||||
if status is not None:
|
||||
return status
|
||||
status = _http_status(getattr(getattr(error, "response", None), "status_code", None))
|
||||
if status is not None:
|
||||
return status
|
||||
# Native LLMRetryError retains sanitized attempt dictionaries. Only the
|
||||
# final attempt describes the terminal failure; never parse error text.
|
||||
attempts = getattr(error, "attempts", None)
|
||||
if isinstance(attempts, (list, tuple)) and attempts and isinstance(attempts[-1], dict):
|
||||
return _http_status(attempts[-1].get("status_code"))
|
||||
except Exception:
|
||||
# Diagnostic attribute access must not replace the provider exception.
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _model_failure_code(status: int | None) -> str:
|
||||
specific = {
|
||||
401: "hy_memory_model_authentication_failed",
|
||||
402: "hy_memory_model_payment_required",
|
||||
403: "hy_memory_model_access_denied",
|
||||
408: "hy_memory_model_timeout",
|
||||
429: "hy_memory_model_rate_limited",
|
||||
}
|
||||
if status in specific:
|
||||
return specific[status]
|
||||
if status is not None and 500 <= status <= 599:
|
||||
return "hy_memory_model_service_unavailable"
|
||||
if status is not None and 400 <= status <= 422:
|
||||
return "hy_memory_model_request_incompatible"
|
||||
return "hy_memory_model_request_failed"
|
||||
|
||||
|
||||
def _terminal_client_timeout(error: Exception) -> bool:
|
||||
try:
|
||||
attempts = getattr(error, "attempts", None)
|
||||
if not isinstance(attempts, (list, tuple)) or not attempts:
|
||||
return False
|
||||
final = attempts[-1]
|
||||
if type(final) is not dict:
|
||||
return False
|
||||
kind = final.get("failure_kind")
|
||||
return type(kind) is str and kind == "timeout"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class HyResponseGuard:
|
||||
def __init__(self, provider: Any, *, phase: str = "extract") -> None:
|
||||
self.provider = provider
|
||||
#: Diagnostic label only; the request passes through unchanged.
|
||||
self.phase = phase
|
||||
self.failures: list[str] = []
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self.provider, name)
|
||||
|
||||
async def complete(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return await self._complete("complete", args, kwargs)
|
||||
|
||||
async def complete_messages(self, *args: Any, **kwargs: Any) -> Any:
|
||||
return await self._complete("complete_messages", args, kwargs)
|
||||
|
||||
async def _complete(self, method: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any:
|
||||
for attempt in range(2):
|
||||
started = monotonic()
|
||||
try:
|
||||
response = await getattr(self.provider, method)(*args, **kwargs)
|
||||
except Exception as error:
|
||||
status = _exception_status(error)
|
||||
code = (
|
||||
"hy_memory_model_timeout"
|
||||
if status is None and _terminal_client_timeout(error)
|
||||
else _model_failure_code(status)
|
||||
)
|
||||
self.failures.append(code)
|
||||
logging.getLogger(__name__).warning(
|
||||
"%s: status_code=%s", code, status if status is not None else "unknown"
|
||||
)
|
||||
diagnostic_event(
|
||||
level="error",
|
||||
component="hy.model",
|
||||
event="hy.model_call.failed",
|
||||
outcome="failed",
|
||||
duration_ms=elapsed_ms(started),
|
||||
error_code=code,
|
||||
**diagnostic_exception(error),
|
||||
details={"phase": self.phase, "attempt": attempt + 1, "call_count": 1},
|
||||
)
|
||||
raise
|
||||
truncated = getattr(response, "finish_reason", "") == "length"
|
||||
empty = not (response.content or "").strip() and not getattr(
|
||||
response, "tool_calls", None
|
||||
)
|
||||
if not truncated and not empty:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="hy.model",
|
||||
event="hy.model_call.completed",
|
||||
outcome="succeeded",
|
||||
duration_ms=elapsed_ms(started),
|
||||
details={"phase": self.phase, "attempt": attempt + 1, "call_count": 1},
|
||||
)
|
||||
return response
|
||||
code = (
|
||||
"hy_memory_model_output_truncated"
|
||||
if truncated
|
||||
else "hy_memory_model_empty_response"
|
||||
)
|
||||
if attempt:
|
||||
self.failures.append(code)
|
||||
raise ValueError(code)
|
||||
logging.getLogger(__name__).warning("%s: retrying the same HY model request", code)
|
||||
if truncated and "max_tokens" in kwargs:
|
||||
kwargs = {**kwargs, "max_tokens": min(16000, kwargs["max_tokens"] * 2)}
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
class HyReconcileGuard:
|
||||
#: Set only for the span of one leased write.
|
||||
step_timer: HyStepTimer | None = None
|
||||
|
||||
def __init__(self, reconciler: Any) -> None:
|
||||
self.reconciler = reconciler
|
||||
self.failures: list[str] = []
|
||||
self.bindings: HyWriteBindings | None = None
|
||||
self.imported_contents: frozenset[str] | None = None
|
||||
self.mutation_target_ids: set[str] = set()
|
||||
|
||||
def begin_write(self) -> None:
|
||||
self.mutation_target_ids.clear()
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self.reconciler, name)
|
||||
|
||||
async def reconcile(self, *args: Any, **kwargs: Any) -> Any:
|
||||
timer = self.step_timer
|
||||
with timer.step("reconcile") if timer is not None else nullcontext():
|
||||
return await self._reconcile(args, kwargs)
|
||||
|
||||
async def _reconcile(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any:
|
||||
started = monotonic()
|
||||
if self.imported_contents is not None:
|
||||
metadata = kwargs.get("new_memories_with_meta") or []
|
||||
texts = kwargs.get("new_memories") or []
|
||||
if (
|
||||
not metadata
|
||||
or not texts
|
||||
or any(
|
||||
not isinstance(item, dict) or item.get("content") not in self.imported_contents
|
||||
for item in metadata
|
||||
)
|
||||
or any(text not in self.imported_contents for text in texts)
|
||||
or {item["content"] for item in metadata} != set(texts)
|
||||
):
|
||||
code = "hy_memory_write_verification_import_input_mismatch"
|
||||
self.failures.append(code)
|
||||
raise ValueError(code)
|
||||
try:
|
||||
result = await self.reconciler.reconcile(*args, **kwargs)
|
||||
except Exception as error:
|
||||
diagnostic_event(
|
||||
level="error",
|
||||
component="hy.reconcile",
|
||||
event="hy.reconcile.failed",
|
||||
outcome="failed",
|
||||
duration_ms=elapsed_ms(started),
|
||||
error_code="hy_memory_reconcile_failed",
|
||||
**diagnostic_exception(error),
|
||||
)
|
||||
raise
|
||||
if not result.success:
|
||||
self.failures.append("hy_memory_reconcile_failed")
|
||||
diagnostic_event(
|
||||
level="error",
|
||||
component="hy.reconcile",
|
||||
event="hy.reconcile.failed",
|
||||
outcome="failed",
|
||||
duration_ms=elapsed_ms(started),
|
||||
error_code="hy_memory_reconcile_failed",
|
||||
details={"result_count": len(result.ops)},
|
||||
)
|
||||
return result
|
||||
try:
|
||||
await self._check_targets(result, kwargs)
|
||||
except ValueError as error:
|
||||
code = str(error)
|
||||
self.failures.append(code)
|
||||
raise
|
||||
except Exception as error:
|
||||
code = "hy_memory_write_verification_failed"
|
||||
self.failures.append(code)
|
||||
raise ValueError(code) from error
|
||||
owners: dict[str, set[str]] = {}
|
||||
for item in kwargs.get("new_memories_with_meta") or []:
|
||||
if item.get("owner") in {"user", "agent"}:
|
||||
owners.setdefault(item["content"], set()).add(item["owner"])
|
||||
# HY's no-candidate ADD path copies content verbatim but omits owner.
|
||||
# Restore only an unambiguous exact match, never infer from wording.
|
||||
for operation in result.ops:
|
||||
source_owners = owners.get(operation.content, set())
|
||||
if operation.op == "ADD" and operation.owner is None and len(source_owners) == 1:
|
||||
operation.owner = next(iter(source_owners))
|
||||
operation_counts = {"ADD": 0, "UPDATE": 0, "SUPERSEDE": 0, "DELETE": 0}
|
||||
for operation in result.ops:
|
||||
raw_name = getattr(operation, "op", "")
|
||||
name = str(getattr(raw_name, "value", raw_name)).upper()
|
||||
if name in {"UPDATE", "SUPERSEDE", "DELETE", "SHADOW"}:
|
||||
target = getattr(operation, "memory_id", None)
|
||||
if isinstance(target, str) and target:
|
||||
self.mutation_target_ids.add(target)
|
||||
self.mutation_target_ids.update(
|
||||
key
|
||||
for key in getattr(operation, "memory_ids", ())
|
||||
if isinstance(key, str) and key
|
||||
)
|
||||
if name == "ADD":
|
||||
operation_counts["ADD"] += 1
|
||||
elif name in {"UPDATE", "MERGE", "NOOP"}:
|
||||
operation_counts["UPDATE"] += 1
|
||||
elif name == "SUPERSEDE":
|
||||
operation_counts["SUPERSEDE"] += 1
|
||||
elif name in {"DELETE", "SHADOW"}:
|
||||
operation_counts["DELETE"] += 1
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="hy.reconcile",
|
||||
event="hy.reconcile.completed",
|
||||
outcome="succeeded",
|
||||
duration_ms=elapsed_ms(started),
|
||||
details={
|
||||
"created_count": operation_counts["ADD"],
|
||||
"updated_count": operation_counts["UPDATE"],
|
||||
"superseded_count": operation_counts["SUPERSEDE"],
|
||||
"deleted_count": operation_counts["DELETE"],
|
||||
"result_count": len(result.ops),
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
async def _check_targets(self, result: Any, kwargs: dict[str, Any]) -> None:
|
||||
"""HY may only touch nodes of this user and scope that this write was allowed to see."""
|
||||
targets = {op.memory_id for op in result.ops if op.memory_id}
|
||||
targets.update(key for op in result.ops for key in getattr(op, "memory_ids", []))
|
||||
for key in targets:
|
||||
node = await kwargs["vector_store"].get_by_id(key)
|
||||
if (
|
||||
node is None
|
||||
or node.user_id != kwargs["user_id"]
|
||||
or node.agent_id != kwargs["agent_id"]
|
||||
):
|
||||
raise ValueError("hy_memory_write_verification_target_invalid")
|
||||
if self.bindings is not None:
|
||||
known = self.bindings.known.get(key)
|
||||
if known is None or (known.content, known.owner) != (node.content, node.owner):
|
||||
raise ValueError("hy_memory_write_verification_target_unbound")
|
||||
if self.bindings is None:
|
||||
return
|
||||
for op in result.ops:
|
||||
if not op.content:
|
||||
continue
|
||||
linked = {key for key in (op.memory_id, *getattr(op, "memory_ids", [])) if key}
|
||||
self.bindings.record_operation_support(op.content, op.owner, linked)
|
||||
if op.memory_id:
|
||||
self.bindings.verified_operations[op.memory_id] = (op.content, op.owner)
|
||||
|
||||
|
||||
def install_hy_write_guards(client: Any) -> None:
|
||||
"""The pinned HY SDK treats an empty reconcile response as a successful empty op list.
|
||||
|
||||
Bind this client's real write paths. Pro/lite use the ordinary writer;
|
||||
ultra uses the System2 writer's inner writer. Keep HY prompts, extraction,
|
||||
retrieval, reconciliation and storage intact; never patch the shared SDK
|
||||
installation.
|
||||
"""
|
||||
|
||||
async def install() -> None:
|
||||
from one_memory.hy_write_verification import HyExtractionGuard
|
||||
|
||||
ordinary = await client._registry.get_writer()
|
||||
system2 = await client._registry.get_system2_writer()
|
||||
guards: list[Any] = []
|
||||
seen_writers: set[int] = set()
|
||||
seen_guards: set[int] = set()
|
||||
for writer in (ordinary, system2._writer):
|
||||
if id(writer) in seen_writers:
|
||||
continue
|
||||
seen_writers.add(id(writer))
|
||||
agent = writer.mem_agent
|
||||
if not isinstance(agent.llm_provider, HyResponseGuard):
|
||||
agent.llm_provider = HyResponseGuard(agent.llm_provider)
|
||||
agent.extractor.llm = agent.llm_provider
|
||||
if not isinstance(agent.extractor, HyExtractionGuard):
|
||||
agent.extractor = HyExtractionGuard(agent.extractor)
|
||||
reconciler = writer.reconciler
|
||||
provider = reconciler._get_llm()
|
||||
if not isinstance(provider, HyResponseGuard):
|
||||
reconciler._llm = HyResponseGuard(provider, phase="reconcile")
|
||||
if not isinstance(reconciler, HyReconcileGuard):
|
||||
writer._reconciler = HyReconcileGuard(reconciler)
|
||||
for guard in (
|
||||
agent.llm_provider,
|
||||
reconciler._llm,
|
||||
writer._reconciler,
|
||||
agent.extractor,
|
||||
):
|
||||
if id(guard) not in seen_guards:
|
||||
seen_guards.add(id(guard))
|
||||
guards.append(guard)
|
||||
client._one_memory_write_guards = tuple(guards)
|
||||
|
||||
client._loop_thread.run(install())
|
||||
@@ -14,7 +14,7 @@ from pathlib import Path
|
||||
from typing import Literal, cast
|
||||
|
||||
HY_MEMORY_DISTRIBUTION = "hy-memory"
|
||||
HY_MEMORY_VERSION = "2.0.5"
|
||||
HY_MEMORY_VERSION = "2.0.5.dev3"
|
||||
HY_MEMORY_NAMESPACE = "governed-v2"
|
||||
DiagnosticLogLevel = Literal["error", "warning", "info", "debug"]
|
||||
|
||||
@@ -33,8 +33,49 @@ _COLLECTION_LOGGERS = (
|
||||
)
|
||||
_NOISY_LIBRARY_LOGGERS = ("httpx", "httpcore", "watchfiles", "jieba")
|
||||
|
||||
# One Memory deliberately stays on ultra because its existing, user-visible
|
||||
# profile/graph flow calls digest(), which the pinned public SDK exposes only
|
||||
# for ultra clients. Reader and non-reranker behavior remain pinned for
|
||||
# compatibility with the SDK authority-migration development preview.
|
||||
HY_MEMORY_MODE = "ultra"
|
||||
# The SDK only recognises `legacy` and `hybrid` (see
|
||||
# hy_memory/pipelines/_retrieval/config.py: `ALL_READERS = (READER_LEGACY,
|
||||
# READER_HYBRID)`). The old spelling `hybrid_v2` was silently rejected and
|
||||
# fell back to legacy, so the hybrid-v2 recall strategy never ran at all.
|
||||
HY_MEMORY_READER = "hybrid"
|
||||
|
||||
HY_MEMORY_BEHAVIOR_ENV: dict[str, str] = {
|
||||
"MEMORY_MODE": HY_MEMORY_MODE,
|
||||
"MEMORY_ENABLE_GRAPH": "true",
|
||||
"MEMORY_GRAPH_RERANK_ENABLED": "false",
|
||||
"MEMORY_GRAPH_TEMPORAL_SPINE": "true",
|
||||
"MEMORY_TOPIC_SUMMARY_ENABLED": "false",
|
||||
"MEMORY_EPISODE_SUMMARY_ENABLED": "false",
|
||||
"MEMORY_EPISODE_RECALL_ENABLED": "true",
|
||||
"MEMORY_DYNAMIC_EPISODE_ENABLED": "true",
|
||||
"MEMORY_SESSION_EPISODE_ENABLED": "false",
|
||||
"MEMORY_SESSION_EPISODE_CONTEXT_ENABLED": "false",
|
||||
"MEMORY_EPISODE_CAPSULE_CONTEXT_ENABLED": "true",
|
||||
"MEMORY_EPISODE_CAPSULE_QUERY_MIN": "0.65",
|
||||
"MEMORY_EPISODE_CAPSULE_ASSIGNMENT_MIN": "0.60",
|
||||
"MEMORY_EPISODE_CAPSULE_FIRST_RANK_MAX": "3",
|
||||
"MEMORY_EPISODE_CAPSULE_MEMBER_LIMIT": "3",
|
||||
"MEMORY_ENABLE_INTENTION": "false",
|
||||
"MEMORY_ENABLE_QUERY_REWRITE": "false",
|
||||
"MEMORY_FEW_SHOT_ENABLED": "true",
|
||||
"MEMORY_HYBRID_V2_QUERY_EXPAND": "1",
|
||||
"MEMORY_HYBRID_V2_GEO_INTENT_PRIOR": "0.6",
|
||||
"MEMORY_HYBRID_V2_DIVERSITY": "1",
|
||||
"MEMORY_HYBRID_V2_TEMPORAL_ADMIT_PRIOR": "0.55",
|
||||
"MEMORY_HYBRID_V2_TEMPORAL_BOOST": "0.18",
|
||||
"MEMORY_CROSS_DOMAIN_CON_THRESHOLD": "0.5",
|
||||
"MEMORY_READER_ENABLE_SUMMARY": "false",
|
||||
"MEMORY_SUMMARY_ENABLED_IN_SYS2": "false",
|
||||
"MEMORY_LOCAL_RERANK_ENABLED": "false",
|
||||
"RECONCILE_ENABLED": "true",
|
||||
"HY_MEMORY_READER": HY_MEMORY_READER,
|
||||
}
|
||||
|
||||
|
||||
def detailed_logging_enabled(data_dir: Path) -> bool:
|
||||
return diagnostic_log_level(data_dir) == "debug"
|
||||
@@ -127,6 +168,7 @@ def configure_hy_runtime_environment(data_dir: Path) -> None:
|
||||
log_dir = memory_root / "logs"
|
||||
memory_root.mkdir(parents=True, exist_ok=True)
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
os.environ.update(HY_MEMORY_BEHAVIOR_ENV)
|
||||
os.environ["MEMORY_DATA_DIR"] = str(memory_root)
|
||||
os.environ["MEMORY_LOG_DIR"] = str(log_dir)
|
||||
level = diagnostic_log_level(data_dir)
|
||||
@@ -136,3 +178,9 @@ def configure_hy_runtime_environment(data_dir: Path) -> None:
|
||||
os.environ["MEMORY_MEMORY_OPERATIONS_ENABLED"] = "false"
|
||||
os.environ["ANONYMIZED_TELEMETRY"] = "False"
|
||||
os.environ.pop("CHROMA_OTEL_COLLECTION_ENDPOINT", None)
|
||||
|
||||
|
||||
def configure_hy_behavior_environment() -> None:
|
||||
"""Set import-time HY behavior flags without touching paths or secrets."""
|
||||
|
||||
os.environ.update(HY_MEMORY_BEHAVIOR_ENV)
|
||||
@@ -1,756 +0,0 @@
|
||||
"""Privileged offline access to the HY SDK data plane during App migration.
|
||||
|
||||
Runtime product code must use public ``HyMemoryClient`` methods. This module
|
||||
is the deliberately isolated exception: App startup has stopped all writers,
|
||||
so the one-time legacy migration may access SDK stores directly without
|
||||
turning migration mechanics into public Client APIs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from hy_memory import ( # type: ignore[import-untyped]
|
||||
DataPlaneRestore,
|
||||
DataPlaneSnapshot,
|
||||
restore_local_data_plane,
|
||||
snapshot_local_data_plane,
|
||||
)
|
||||
from hy_memory.models.memory import ( # type: ignore[import-untyped]
|
||||
MemoryLayer,
|
||||
MemoryNode,
|
||||
MemoryStatus,
|
||||
SourceType,
|
||||
)
|
||||
from hy_memory.utils.source_filter import normalize_source # type: ignore[import-untyped]
|
||||
|
||||
from one_memory.hy_sdk_types import (
|
||||
HySdkMemoryPage,
|
||||
HySdkMemoryRecord,
|
||||
HySdkMigrationError,
|
||||
HySdkRawPage,
|
||||
HySdkRawRecord,
|
||||
)
|
||||
|
||||
|
||||
class MigrationClient(Protocol):
|
||||
_loop_thread: Any
|
||||
_vector_store: Any
|
||||
_embed_service: Any
|
||||
_graph_store: Any
|
||||
|
||||
def list_memories(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
statuses: list[str] | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
order: str = "desc",
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def delete(self, memory_id: str) -> Mapping[str, object]: ...
|
||||
|
||||
def get_write_outcome(self, request_id: str) -> Mapping[str, object] | None: ...
|
||||
|
||||
def get(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: str | None = None,
|
||||
variant: str | None = None,
|
||||
) -> Mapping[str, object] | None: ...
|
||||
|
||||
async def _coding_store_for_exact_get(self, variant: str) -> Any: ...
|
||||
|
||||
|
||||
class HySdkMigration:
|
||||
"""Typed offline adapter over one initialized SDK data plane."""
|
||||
|
||||
def __init__(self, client: MigrationClient) -> None:
|
||||
self.client = client
|
||||
self._loop_thread = client._loop_thread
|
||||
self._vector_store = client._vector_store
|
||||
self._embed_service = client._embed_service
|
||||
self._graph_store = client._graph_store
|
||||
|
||||
async def list_memories(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
statuses: tuple[str, ...] = ("active",),
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
order: str = "desc",
|
||||
) -> HySdkMemoryPage:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_memory_list_failed",
|
||||
self.client.list_memories,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
statuses=list(statuses),
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
order=order,
|
||||
)
|
||||
return _memory_page(raw, limit=limit, offset=offset)
|
||||
|
||||
async def list_productivity_memories(self) -> tuple[dict[str, object], ...]:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_productivity_inventory_failed",
|
||||
self._loop_thread.run,
|
||||
self._list_productivity_memories(),
|
||||
)
|
||||
return tuple(raw)
|
||||
|
||||
async def _list_productivity_memories(self) -> list[dict[str, object]]:
|
||||
records: list[dict[str, object]] = []
|
||||
for variant in ("working", "coding"):
|
||||
store = await self.client._coding_store_for_exact_get(variant)
|
||||
if store is None:
|
||||
continue
|
||||
with store._lock:
|
||||
experiences = store._conn.execute(
|
||||
"SELECT experience_id,user_id,agent_id,solution "
|
||||
"FROM coding_memory_meta ORDER BY experience_id"
|
||||
).fetchall()
|
||||
facts = store._conn.execute(
|
||||
"SELECT fact_id,user_id,agent_id,statement,status "
|
||||
"FROM working_fact ORDER BY fact_id"
|
||||
).fetchall()
|
||||
records.extend(
|
||||
{
|
||||
"memory_id": str(row["experience_id"]),
|
||||
"content": str(row["solution"]),
|
||||
"status": "active",
|
||||
"user_id": str(row["user_id"]),
|
||||
"agent_id": str(row["agent_id"]),
|
||||
"kind": "experience",
|
||||
"variant": variant,
|
||||
}
|
||||
for row in experiences
|
||||
)
|
||||
records.extend(
|
||||
{
|
||||
"memory_id": str(row["fact_id"]),
|
||||
"content": str(row["statement"]),
|
||||
"status": str(row["status"]),
|
||||
"user_id": str(row["user_id"]),
|
||||
"agent_id": str(row["agent_id"]),
|
||||
"kind": "fact",
|
||||
"variant": variant,
|
||||
}
|
||||
for row in facts
|
||||
)
|
||||
return records
|
||||
|
||||
async def read_productivity_memory(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: str,
|
||||
variant: str | None,
|
||||
) -> tuple[HySdkMemoryRecord, str] | None:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_productivity_get_failed",
|
||||
self.client.get,
|
||||
memory_id,
|
||||
kind=kind,
|
||||
variant=variant,
|
||||
)
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, Mapping) or raw.get("variant") not in {
|
||||
"working",
|
||||
"coding",
|
||||
}:
|
||||
raise HySdkMigrationError("hy_sdk_productivity_variant_invalid")
|
||||
return HySdkMemoryRecord.from_sdk(raw), str(raw["variant"])
|
||||
|
||||
async def delete_productivity_memory(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: str,
|
||||
variant: str | None,
|
||||
user_id: str,
|
||||
) -> dict[str, object]:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_productivity_delete_failed",
|
||||
self._loop_thread.run,
|
||||
self._delete_productivity_memory(
|
||||
memory_id,
|
||||
kind=kind,
|
||||
variant=variant,
|
||||
user_id=user_id,
|
||||
),
|
||||
)
|
||||
return _successful_result(raw, "hy_sdk_productivity_delete_failed")
|
||||
|
||||
async def _delete_productivity_memory(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: str,
|
||||
variant: str | None,
|
||||
user_id: str,
|
||||
) -> dict[str, object]:
|
||||
if kind not in {"experience", "fact"}:
|
||||
raise ValueError("productivity kind must be experience or fact")
|
||||
matches: list[tuple[Any, str]] = []
|
||||
for candidate in ((variant,) if variant is not None else ("working", "coding")):
|
||||
store = await self.client._coding_store_for_exact_get(candidate)
|
||||
if store is None:
|
||||
continue
|
||||
existing = (
|
||||
await store.get_by_id(memory_id, user_id=user_id)
|
||||
if kind == "experience"
|
||||
else await store.get_fact(memory_id, user_id=user_id)
|
||||
)
|
||||
if existing is not None:
|
||||
matches.append((store, candidate))
|
||||
if len(matches) > 1:
|
||||
raise ValueError("productivity memory id is ambiguous across variants")
|
||||
if not matches:
|
||||
return {"success": True, "deleted_count": 0}
|
||||
store, _actual_variant = matches[0]
|
||||
if kind == "experience":
|
||||
deleted = await store.delete(memory_id, user_id=user_id)
|
||||
elif kind == "fact":
|
||||
await store._fact_vec.delete(memory_id)
|
||||
with store._lock:
|
||||
store._conn.execute(
|
||||
"DELETE FROM working_fact WHERE fact_id=? AND user_id=?",
|
||||
(memory_id, user_id),
|
||||
)
|
||||
store._conn.commit()
|
||||
deleted = True
|
||||
return {"success": True, "deleted_count": int(deleted)}
|
||||
|
||||
async def write_legacy_memory(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
memory_id: str,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
session_id: str,
|
||||
source: str,
|
||||
layer: str,
|
||||
status: str,
|
||||
owner: str | None,
|
||||
tags: tuple[str, ...],
|
||||
supersedes: tuple[str, ...],
|
||||
superseded_by: tuple[str, ...],
|
||||
is_latest: bool,
|
||||
memory_at: datetime | None,
|
||||
metadata: Mapping[str, object],
|
||||
request_id: str,
|
||||
) -> dict[str, object]:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_memory_import_failed",
|
||||
self._loop_thread.run,
|
||||
self._write_legacy_memory(
|
||||
content=content,
|
||||
memory_id=memory_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
layer=layer,
|
||||
status=status,
|
||||
owner=owner,
|
||||
tags=tags,
|
||||
supersedes=supersedes,
|
||||
superseded_by=superseded_by,
|
||||
is_latest=is_latest,
|
||||
memory_at=memory_at,
|
||||
metadata=metadata,
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
return _successful_result(raw, "hy_sdk_memory_import_failed")
|
||||
|
||||
async def _write_legacy_memory(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
memory_id: str,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
session_id: str,
|
||||
source: str,
|
||||
layer: str,
|
||||
status: str,
|
||||
owner: str | None,
|
||||
tags: tuple[str, ...],
|
||||
supersedes: tuple[str, ...],
|
||||
superseded_by: tuple[str, ...],
|
||||
is_latest: bool,
|
||||
memory_at: datetime | None,
|
||||
metadata: Mapping[str, object],
|
||||
request_id: str,
|
||||
) -> dict[str, object]:
|
||||
if not content or not memory_id or not user_id:
|
||||
raise ValueError("content, memory_id and user_id are required")
|
||||
normalized_layer = MemoryLayer.from_string(layer)
|
||||
if normalized_layer is MemoryLayer.L1_RAW:
|
||||
raise ValueError("raw content must use the raw migration lane")
|
||||
normalized_status = MemoryStatus(status.lower())
|
||||
normalized_source = normalize_source(source)
|
||||
existing = await self._vector_store.get_by_id(memory_id)
|
||||
if existing is not None:
|
||||
identity = (
|
||||
existing.content == content,
|
||||
existing.user_id == user_id,
|
||||
existing.agent_id == agent_id,
|
||||
existing.session_id == session_id,
|
||||
existing.source == normalized_source,
|
||||
existing.layer == normalized_layer,
|
||||
existing.status == normalized_status,
|
||||
existing.owner == owner,
|
||||
list(existing.tags or []) == list(tags),
|
||||
list(existing.supersedes or []) == list(supersedes),
|
||||
list(existing.superseded_by or []) == list(superseded_by),
|
||||
existing.is_latest == is_latest,
|
||||
)
|
||||
if not all(identity):
|
||||
raise ValueError("memory_id is already bound to different memory content")
|
||||
await self._project_graph(existing)
|
||||
return {
|
||||
"success": True,
|
||||
"memory_id": memory_id,
|
||||
"memory_ids": [memory_id],
|
||||
"request_id": request_id,
|
||||
"replayed": True,
|
||||
}
|
||||
custom = dict(metadata)
|
||||
custom["migration_request_id"] = request_id
|
||||
embedding = await self._embed_service.embed(content)
|
||||
node = MemoryNode(
|
||||
node_id=memory_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=normalized_source,
|
||||
layer=normalized_layer,
|
||||
content=content,
|
||||
source_type=SourceType.EXPLICIT,
|
||||
status=normalized_status,
|
||||
owner=owner,
|
||||
tags=list(tags),
|
||||
supersedes=list(supersedes),
|
||||
superseded_by=list(superseded_by),
|
||||
is_latest=is_latest,
|
||||
embedding=embedding,
|
||||
memory_at=memory_at,
|
||||
custom=custom,
|
||||
)
|
||||
stored_id = await self._vector_store.upsert(node)
|
||||
if stored_id != memory_id:
|
||||
raise RuntimeError("exact migration changed the requested memory identity")
|
||||
await self._project_graph(node)
|
||||
return {
|
||||
"success": True,
|
||||
"memory_id": memory_id,
|
||||
"memory_ids": [memory_id],
|
||||
"request_id": request_id,
|
||||
}
|
||||
|
||||
async def _project_graph(self, node: MemoryNode) -> None:
|
||||
if self._graph_store is not None:
|
||||
await self._graph_store.upsert_memory_node(node)
|
||||
|
||||
async def read_raw_inventory(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None = None,
|
||||
session_id: str | None = None,
|
||||
source: str | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
order: str = "desc",
|
||||
) -> HySdkRawPage:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_raw_list_failed",
|
||||
self._loop_thread.run,
|
||||
self._read_raw_inventory(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
order=order,
|
||||
),
|
||||
)
|
||||
return _raw_page(raw, limit=limit, offset=offset)
|
||||
|
||||
async def _read_raw_inventory(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
agent_id: str | None,
|
||||
session_id: str | None,
|
||||
source: str | None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
order: str,
|
||||
) -> dict[str, object]:
|
||||
if not user_id:
|
||||
raise ValueError("user_id is required")
|
||||
nodes = await self._vector_store.list_by_user(
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
status_filter=list(MemoryStatus),
|
||||
layers=[MemoryLayer.L1_RAW],
|
||||
include_sources=[source] if source else None,
|
||||
limit=100_000,
|
||||
only_latest=False,
|
||||
with_vectors=False,
|
||||
allow_raw_layer=True,
|
||||
)
|
||||
if session_id:
|
||||
nodes = [node for node in nodes if node.session_id == session_id]
|
||||
nodes.sort(
|
||||
key=_node_timestamp,
|
||||
reverse=order.lower() != "asc",
|
||||
)
|
||||
return {
|
||||
"raw": [_raw_node(node) for node in nodes[offset : offset + limit]],
|
||||
"total": len(nodes),
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
async def write_legacy_raw(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
raw_id: str,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
session_id: str,
|
||||
source: str,
|
||||
memory_at: datetime | None,
|
||||
metadata: Mapping[str, object],
|
||||
request_id: str,
|
||||
) -> dict[str, object]:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_raw_import_failed",
|
||||
self._loop_thread.run,
|
||||
self._write_legacy_raw(
|
||||
content=content,
|
||||
raw_id=raw_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
memory_at=memory_at,
|
||||
metadata=metadata,
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
return _successful_result(raw, "hy_sdk_raw_import_failed")
|
||||
|
||||
async def delete_raw(self, raw_id: str) -> dict[str, object]:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_raw_delete_failed",
|
||||
self._loop_thread.run,
|
||||
self._delete_raw(raw_id),
|
||||
)
|
||||
return _successful_result(raw, "hy_sdk_raw_delete_failed")
|
||||
|
||||
async def _delete_raw(self, raw_id: str) -> dict[str, object]:
|
||||
existing = await self._vector_store.get_by_id(raw_id)
|
||||
if existing is None or existing.layer is not MemoryLayer.L1_RAW:
|
||||
return {"success": True, "deleted_count": 0}
|
||||
deleted = await self._vector_store.delete(raw_id)
|
||||
return {"success": True, "deleted_count": int(bool(deleted))}
|
||||
|
||||
async def _write_legacy_raw(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
raw_id: str,
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
session_id: str,
|
||||
source: str,
|
||||
memory_at: datetime | None,
|
||||
metadata: Mapping[str, object],
|
||||
request_id: str,
|
||||
) -> dict[str, object]:
|
||||
if not content or not raw_id or not user_id:
|
||||
raise ValueError("content, raw_id and user_id are required")
|
||||
normalized_source = normalize_source(source)
|
||||
existing = await self._vector_store.get_by_id(raw_id)
|
||||
if existing is not None:
|
||||
identity = (
|
||||
existing.layer is MemoryLayer.L1_RAW,
|
||||
existing.content == content,
|
||||
existing.user_id == user_id,
|
||||
existing.agent_id == agent_id,
|
||||
existing.session_id == session_id,
|
||||
existing.source == normalized_source,
|
||||
)
|
||||
if not all(identity):
|
||||
raise ValueError("raw_id is already bound to different source content")
|
||||
return {
|
||||
"success": True,
|
||||
"raw_id": raw_id,
|
||||
"request_id": request_id,
|
||||
"replayed": True,
|
||||
}
|
||||
custom = dict(metadata)
|
||||
custom["migration_request_id"] = request_id
|
||||
embedding = await self._embed_service.embed(content)
|
||||
node = MemoryNode(
|
||||
node_id=raw_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
session_id=session_id,
|
||||
source=normalized_source,
|
||||
layer=MemoryLayer.L1_RAW,
|
||||
content=content,
|
||||
source_type=SourceType.EXPLICIT,
|
||||
status=MemoryStatus.ACTIVE,
|
||||
is_latest=True,
|
||||
embedding=embedding,
|
||||
memory_at=memory_at,
|
||||
custom=custom,
|
||||
)
|
||||
stored_id = await self._vector_store.upsert(node)
|
||||
if stored_id != raw_id:
|
||||
raise RuntimeError("exact migration changed the requested raw identity")
|
||||
return {"success": True, "raw_id": raw_id, "request_id": request_id}
|
||||
|
||||
async def delete(self, memory_id: str) -> dict[str, object]:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_memory_delete_failed",
|
||||
self.client.delete,
|
||||
memory_id,
|
||||
)
|
||||
return _successful_result(raw, "hy_sdk_memory_delete_failed")
|
||||
|
||||
async def get_write_outcome(self, request_id: str) -> dict[str, object] | None:
|
||||
raw = await _sdk_call(
|
||||
"hy_sdk_write_outcome_failed",
|
||||
self.client.get_write_outcome,
|
||||
request_id,
|
||||
)
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkMigrationError("hy_sdk_write_outcome_shape_invalid")
|
||||
return dict(raw)
|
||||
|
||||
|
||||
def snapshot_data_plane(
|
||||
*,
|
||||
source_root: Path,
|
||||
target_root: Path,
|
||||
writers_stopped: bool,
|
||||
) -> DataPlaneSnapshot:
|
||||
"""Snapshot an explicitly quiesced SDK-owned local data plane."""
|
||||
return snapshot_local_data_plane(
|
||||
source_root=source_root,
|
||||
target_root=target_root,
|
||||
writers_stopped=writers_stopped,
|
||||
)
|
||||
|
||||
|
||||
def restore_data_plane(
|
||||
*,
|
||||
snapshot_root: Path,
|
||||
target_root: Path,
|
||||
writers_stopped: bool,
|
||||
protective_root: Path | None = None,
|
||||
) -> DataPlaneRestore:
|
||||
"""Restore an explicitly quiesced SDK-owned local data plane."""
|
||||
return restore_local_data_plane(
|
||||
snapshot_root=snapshot_root,
|
||||
target_root=target_root,
|
||||
writers_stopped=writers_stopped,
|
||||
protective_root=protective_root,
|
||||
)
|
||||
|
||||
|
||||
async def _sdk_call[T](
|
||||
code: str,
|
||||
call: Callable[..., T],
|
||||
*args: object,
|
||||
**kwargs: object,
|
||||
) -> T:
|
||||
pending = asyncio.create_task(asyncio.to_thread(call, *args, **kwargs))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
await _finish_cancelled_call(pending)
|
||||
raise
|
||||
except Exception as error:
|
||||
raise HySdkMigrationError(code) from error
|
||||
|
||||
|
||||
async def _finish_cancelled_call[T](pending: asyncio.Task[T]) -> None:
|
||||
while not pending.done():
|
||||
try:
|
||||
await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
except Exception:
|
||||
return
|
||||
if not pending.cancelled():
|
||||
pending.exception()
|
||||
|
||||
|
||||
def _memory_page(raw: object, *, limit: int, offset: int) -> HySdkMemoryPage:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
|
||||
rows: list[object] = []
|
||||
total = 0
|
||||
for bucket_name, row_name in (("vdb", "memories"), ("graph", "nodes")):
|
||||
bucket = raw.get(bucket_name)
|
||||
if bucket is None and bucket_name == "graph":
|
||||
continue
|
||||
if not isinstance(bucket, Mapping):
|
||||
raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
|
||||
bucket_rows = bucket.get(row_name)
|
||||
bucket_total = bucket.get("total")
|
||||
if not isinstance(bucket_rows, list) or type(bucket_total) is not int or bucket_total < 0:
|
||||
raise HySdkMigrationError("hy_sdk_memory_page_shape_invalid")
|
||||
rows.extend(bucket_rows)
|
||||
total += bucket_total
|
||||
|
||||
records: dict[str, HySdkMemoryRecord] = {}
|
||||
for row in rows:
|
||||
record = _memory_record(row)
|
||||
records.setdefault(record.memory_id, record)
|
||||
return HySdkMemoryPage(
|
||||
memories=tuple(records.values()),
|
||||
total=total,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
def _memory_record(raw: object) -> HySdkMemoryRecord:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid")
|
||||
required = ("memory_id", "content", "layer", "status", "user_id", "agent_id")
|
||||
if any(not isinstance(raw.get(name), str) or not raw[name] for name in required):
|
||||
raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid")
|
||||
custom = raw.get("custom")
|
||||
source_raw_ids: list[object] = []
|
||||
if isinstance(custom, Mapping) and isinstance(custom.get("source_raw_memory_ids"), list):
|
||||
source_raw_ids.extend(custom["source_raw_memory_ids"])
|
||||
if isinstance(raw.get("source_raw_memory_id"), str):
|
||||
source_raw_ids.append(raw["source_raw_memory_id"])
|
||||
try:
|
||||
return HySdkMemoryRecord(
|
||||
memory_id=raw["memory_id"],
|
||||
content=raw["content"],
|
||||
layer=raw["layer"],
|
||||
status=raw["status"],
|
||||
is_latest=bool(raw.get("is_latest", True)),
|
||||
supersedes=_string_tuple(raw.get("supersedes")),
|
||||
superseded_by=_string_tuple(raw.get("superseded_by")),
|
||||
tags=_string_tuple(raw.get("tags")),
|
||||
owner=raw.get("owner") if raw.get("owner") in {"user", "agent"} else None,
|
||||
source_raw_ids=_string_tuple(source_raw_ids),
|
||||
user_id=raw["user_id"],
|
||||
agent_id=raw["agent_id"],
|
||||
session_id=_optional_string(raw.get("session_id")),
|
||||
source=_optional_string(raw.get("source")),
|
||||
memory_at=_optional_int(raw.get("memory_at")),
|
||||
gmt_created=_optional_int(raw.get("gmt_created")),
|
||||
)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise HySdkMigrationError("hy_sdk_memory_record_shape_invalid") from error
|
||||
|
||||
|
||||
def _raw_page(raw: object, *, limit: int, offset: int) -> HySdkRawPage:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkMigrationError("hy_sdk_raw_page_shape_invalid")
|
||||
rows = raw.get("raw")
|
||||
total = raw.get("total")
|
||||
if not isinstance(rows, list) or type(total) is not int or total < 0:
|
||||
raise HySdkMigrationError("hy_sdk_raw_page_shape_invalid")
|
||||
return HySdkRawPage(
|
||||
raw=tuple(_raw_record(row) for row in rows),
|
||||
total=total,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
def _node_timestamp(node: Any) -> float:
|
||||
value = getattr(node, "memory_at", None) or getattr(node, "gmt_created", None)
|
||||
return value.timestamp() if isinstance(value, datetime) else 0.0
|
||||
|
||||
|
||||
def _raw_node(node: Any) -> dict[str, object]:
|
||||
def timestamp(value: object) -> int | None:
|
||||
return int(value.timestamp()) if isinstance(value, datetime) else None
|
||||
|
||||
return {
|
||||
"raw_id": node.node_id,
|
||||
"content": node.content,
|
||||
"user_id": node.user_id,
|
||||
"agent_id": node.agent_id,
|
||||
"session_id": node.session_id,
|
||||
"source": node.source,
|
||||
"memory_at": timestamp(node.memory_at),
|
||||
"gmt_created": timestamp(node.gmt_created),
|
||||
"custom": node.custom or {},
|
||||
}
|
||||
|
||||
|
||||
def _raw_record(raw: object) -> HySdkRawRecord:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid")
|
||||
required = ("raw_id", "content", "user_id", "agent_id", "session_id", "source")
|
||||
if any(not isinstance(raw.get(name), str) or not raw[name] for name in required):
|
||||
raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid")
|
||||
custom = raw.get("custom")
|
||||
try:
|
||||
return HySdkRawRecord(
|
||||
raw_id=raw["raw_id"],
|
||||
content=raw["content"],
|
||||
user_id=raw["user_id"],
|
||||
agent_id=raw["agent_id"],
|
||||
session_id=raw["session_id"],
|
||||
source=raw["source"],
|
||||
memory_at=_optional_int(raw.get("memory_at")),
|
||||
gmt_created=_optional_int(raw.get("gmt_created")),
|
||||
custom=dict(custom) if isinstance(custom, Mapping) else {},
|
||||
)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise HySdkMigrationError("hy_sdk_raw_record_shape_invalid") from error
|
||||
|
||||
|
||||
def _successful_result(raw: object, code: str) -> dict[str, object]:
|
||||
if not isinstance(raw, Mapping) or raw.get("success") is not True:
|
||||
raise HySdkMigrationError(code)
|
||||
return dict(raw)
|
||||
|
||||
|
||||
def _string_tuple(value: object) -> tuple[str, ...]:
|
||||
if not isinstance(value, list | tuple):
|
||||
return ()
|
||||
return tuple(item for item in value if isinstance(item, str) and item)
|
||||
|
||||
|
||||
def _optional_string(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _optional_int(value: object) -> int | None:
|
||||
return value if type(value) is int else None
|
||||
@@ -1,406 +0,0 @@
|
||||
"""HY SDK client construction and shared-runtime leasing.
|
||||
|
||||
This module deliberately depends only on the SDK's package-root exports. A
|
||||
lease owns one client while the process shares one ``SharedRuntime`` per
|
||||
physical MemHub data directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable, Coroutine
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from hy_memory import ( # type: ignore[import-untyped]
|
||||
HyMemoryClient,
|
||||
MemoryConfig,
|
||||
SharedRuntime,
|
||||
)
|
||||
|
||||
from one_memory.embedding_assets import (
|
||||
BUILTIN_EMBEDDING_DIMENSIONS,
|
||||
BUILTIN_EMBEDDING_MODEL,
|
||||
)
|
||||
from one_memory.hy_graph_admin import (
|
||||
GraphWidthMismatchError,
|
||||
graph_store_enabled,
|
||||
inspect_graph_width_mismatch,
|
||||
)
|
||||
from one_memory.hy_sdk_types import HySdkRuntimeError
|
||||
from one_memory.local_embedding import LocalEmbedder
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
_RUNTIME_LOCK = threading.RLock()
|
||||
BUILTIN_RETRIEVAL_PROFILE_ID = "builtin-retrieval"
|
||||
_LOCAL_ONNX_PLACEHOLDER_URL = "http://127.0.0.1/local-onnx"
|
||||
DEFAULT_RELEVANCE_FLOOR = 0.35
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RuntimeEntry:
|
||||
runtime: SharedRuntime
|
||||
references: int
|
||||
|
||||
|
||||
_RUNTIMES: dict[Path, _RuntimeEntry] = {}
|
||||
|
||||
|
||||
class LocalOnnxEmbeddingService:
|
||||
"""Public ``EmbeddingService`` implementation backed by MemHub's ONNX model."""
|
||||
|
||||
_embedder: ClassVar[LocalEmbedder | None] = None
|
||||
_embedder_lock: ClassVar[threading.Lock] = threading.Lock()
|
||||
|
||||
def __init__(self, embedder: LocalEmbedder | None = None) -> None:
|
||||
if embedder is None:
|
||||
with self._embedder_lock:
|
||||
if self._embedder is None:
|
||||
self.__class__._embedder = LocalEmbedder()
|
||||
embedder = self._embedder
|
||||
if embedder is None: # pragma: no cover - guarded by the assignment above
|
||||
raise HySdkRuntimeError("hy_sdk_local_embedding_unavailable")
|
||||
self._local_embedder = embedder
|
||||
|
||||
async def embed(self, text: str, use_cache: bool = True) -> list[float]:
|
||||
del use_cache
|
||||
vectors = await asyncio.to_thread(self._local_embedder.encode, [text])
|
||||
return vectors[0]
|
||||
|
||||
async def embed_with_meta(
|
||||
self,
|
||||
text: str,
|
||||
use_cache: bool = True,
|
||||
) -> tuple[list[float], dict[str, Any]]:
|
||||
return await self.embed(text, use_cache), {"provider": "local-onnx"}
|
||||
|
||||
async def embed_batch(
|
||||
self,
|
||||
texts: list[str],
|
||||
use_cache: bool = True,
|
||||
) -> list[list[float]]:
|
||||
del use_cache
|
||||
if not texts:
|
||||
return []
|
||||
return await asyncio.to_thread(self._local_embedder.encode, list(texts))
|
||||
|
||||
async def embed_queued(self, text: str, use_cache: bool = True) -> list[float]:
|
||||
return await self.embed(text, use_cache)
|
||||
|
||||
async def embed_queued_with_meta(
|
||||
self,
|
||||
text: str,
|
||||
use_cache: bool = True,
|
||||
) -> tuple[list[float], dict[str, Any]]:
|
||||
return await self.embed_with_meta(text, use_cache)
|
||||
|
||||
def get_embedding_dims(self) -> int:
|
||||
return BUILTIN_EMBEDDING_DIMENSIONS
|
||||
|
||||
|
||||
class ClientLease:
|
||||
"""Own a client and release its shared runtime exactly once."""
|
||||
|
||||
def __init__(self, client: HyMemoryClient, data_dir: Path) -> None:
|
||||
self.client = client
|
||||
self.data_dir = data_dir.resolve()
|
||||
self._closed = False
|
||||
self._close_lock = threading.Lock()
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self._closed
|
||||
|
||||
def close(self) -> None:
|
||||
with self._close_lock:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
try:
|
||||
self.client.close()
|
||||
finally:
|
||||
_release_runtime(self.data_dir)
|
||||
|
||||
def __enter__(self) -> ClientLease:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
class SdkClientHandle:
|
||||
"""Lazy async owner for one profile-bound public SDK client lease."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
profile: ModelProfileSnapshot,
|
||||
*,
|
||||
api_key: str | None,
|
||||
data_dir: Path,
|
||||
http_transport_factory: Callable[[], Any] | None = None,
|
||||
) -> None:
|
||||
self._profile = profile
|
||||
self._api_key = api_key
|
||||
self._data_dir = data_dir
|
||||
self._http_transport_factory = http_transport_factory
|
||||
self._lease: ClientLease | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def client(self) -> HyMemoryClient:
|
||||
if self._lease is not None:
|
||||
return self._lease.client
|
||||
async with self._lock:
|
||||
if self._lease is None:
|
||||
self._lease = await asyncio.to_thread(
|
||||
acquire_client,
|
||||
self._profile,
|
||||
api_key=self._api_key,
|
||||
data_dir=self._data_dir,
|
||||
http_transport_factory=self._http_transport_factory,
|
||||
)
|
||||
return self._lease.client
|
||||
|
||||
async def close(self) -> None:
|
||||
async with self._lock:
|
||||
lease, self._lease = self._lease, None
|
||||
if lease is not None:
|
||||
await asyncio.to_thread(lease.close)
|
||||
|
||||
|
||||
def builtin_retrieval_snapshot() -> ModelProfileSnapshot:
|
||||
"""Return the model-free profile used by SDK-native read operations."""
|
||||
return ModelProfileSnapshot(
|
||||
profile_id=BUILTIN_RETRIEVAL_PROFILE_ID,
|
||||
revision=0,
|
||||
provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
base_url=_LOCAL_ONNX_PLACEHOLDER_URL,
|
||||
model=BUILTIN_EMBEDDING_MODEL,
|
||||
embedding_provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
embedding_model=BUILTIN_EMBEDDING_MODEL,
|
||||
embedding_dimensions=BUILTIN_EMBEDDING_DIMENSIONS,
|
||||
timeout_seconds=60,
|
||||
max_concurrency=1,
|
||||
context_window=None,
|
||||
has_api_key=False,
|
||||
)
|
||||
|
||||
|
||||
def sdk_relevance_floor() -> float:
|
||||
raw = os.environ.get("ONE_MEMORY_HY_RELEVANCE_FLOOR", "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_RELEVANCE_FLOOR
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return DEFAULT_RELEVANCE_FLOOR
|
||||
return min(1.0, max(0.0, value))
|
||||
|
||||
|
||||
def build_sdk_config(
|
||||
profile: ModelProfileSnapshot,
|
||||
*,
|
||||
api_key: str | None,
|
||||
data_dir: Path,
|
||||
) -> MemoryConfig:
|
||||
"""Map a model snapshot to SDK models, endpoints, backends, and local paths."""
|
||||
generation_endpoint, generation_key = _generation_access(profile, api_key)
|
||||
embedding_provider = profile.embedding_provider_kind or profile.provider_kind
|
||||
embedding_service_is_local = embedding_provider is ProviderKind.LOCAL_ONNX
|
||||
embedding_model = profile.embedding_model or (
|
||||
BUILTIN_EMBEDDING_MODEL if embedding_service_is_local else None
|
||||
)
|
||||
if not embedding_model:
|
||||
raise HySdkRuntimeError("hy_sdk_embedding_model_required")
|
||||
dimensions: int | None
|
||||
if embedding_service_is_local:
|
||||
embedding_endpoint = None
|
||||
embedding_key = None
|
||||
dimensions = profile.embedding_dimensions or BUILTIN_EMBEDDING_DIMENSIONS
|
||||
else:
|
||||
embedding_endpoint = _openai_endpoint(
|
||||
embedding_provider,
|
||||
profile.embedding_base_url or profile.base_url,
|
||||
)
|
||||
embedding_key = "ollama" if embedding_provider is ProviderKind.OLLAMA else api_key
|
||||
if embedding_provider is ProviderKind.OPENAI_COMPATIBLE and not embedding_key:
|
||||
raise HySdkRuntimeError("hy_sdk_embedding_api_key_missing")
|
||||
if embedding_provider not in {
|
||||
ProviderKind.OPENAI_COMPATIBLE,
|
||||
ProviderKind.OLLAMA,
|
||||
}:
|
||||
raise HySdkRuntimeError("hy_sdk_embedding_provider_unsupported")
|
||||
dimensions = profile.embedding_dimensions
|
||||
|
||||
root = data_dir.resolve() / "hy-memory" / "governed-v2"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
collection_identity = "\0".join(
|
||||
(
|
||||
embedding_provider.value,
|
||||
embedding_endpoint or "local",
|
||||
embedding_model,
|
||||
str(dimensions or "auto"),
|
||||
)
|
||||
)
|
||||
collection = "memhub_" + hashlib.sha256(collection_identity.encode()).hexdigest()[:16]
|
||||
|
||||
return MemoryConfig.from_dict(
|
||||
{
|
||||
"llm": {
|
||||
"provider": "openai",
|
||||
"model": profile.model,
|
||||
"api_key": generation_key,
|
||||
"base_url": generation_endpoint,
|
||||
},
|
||||
"embedder": {
|
||||
"provider": "openai",
|
||||
"model": (
|
||||
BUILTIN_EMBEDDING_MODEL if embedding_service_is_local else embedding_model
|
||||
),
|
||||
"api_key": embedding_key,
|
||||
"base_url": embedding_endpoint,
|
||||
"embedding_dims": dimensions,
|
||||
},
|
||||
"vector_store": {
|
||||
"provider": "chroma",
|
||||
"collection_name": collection,
|
||||
"persist_directory": str(root / "vector"),
|
||||
"embedding_dims": dimensions,
|
||||
"host": None,
|
||||
"port": None,
|
||||
"api_key": None,
|
||||
},
|
||||
"graph_store": {
|
||||
"provider": "kuzu",
|
||||
"db_path": str(root / "graph"),
|
||||
"url": None,
|
||||
"username": None,
|
||||
"password": None,
|
||||
"database": None,
|
||||
},
|
||||
"cache": {
|
||||
"backend": "sqlite",
|
||||
"db_path": str(root / "cache.sqlite3"),
|
||||
},
|
||||
"history": {"db_path": str(root / "history.sqlite3")},
|
||||
"coding": {"db_path": str(root / "coding.sqlite3")},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def acquire_client(
|
||||
profile: ModelProfileSnapshot,
|
||||
*,
|
||||
api_key: str | None,
|
||||
data_dir: Path,
|
||||
http_transport_factory: Callable[[], Any] | None = None,
|
||||
) -> ClientLease:
|
||||
"""Construct a public SDK client and return its explicit runtime lease."""
|
||||
config = build_sdk_config(profile, api_key=api_key, data_dir=data_dir)
|
||||
config.http_transport_factory = http_transport_factory
|
||||
mismatch = inspect_graph_width_mismatch(config)
|
||||
if mismatch is not None:
|
||||
raise GraphWidthMismatchError(mismatch)
|
||||
key = data_dir.resolve()
|
||||
runtime = _acquire_runtime(config, key)
|
||||
embedding_service: LocalOnnxEmbeddingService | None = None
|
||||
if (profile.embedding_provider_kind or profile.provider_kind) is ProviderKind.LOCAL_ONNX:
|
||||
embedding_service = LocalOnnxEmbeddingService()
|
||||
try:
|
||||
client = HyMemoryClient(
|
||||
config=config,
|
||||
mode="ultra" if graph_store_enabled() else "pro",
|
||||
runtime=runtime,
|
||||
embedding_service=embedding_service,
|
||||
)
|
||||
except BaseException:
|
||||
_release_runtime(key)
|
||||
raise
|
||||
return ClientLease(client, key)
|
||||
|
||||
|
||||
def close_all_shared_runtimes() -> None:
|
||||
"""Close process runtimes only after every client lease has been released."""
|
||||
with _RUNTIME_LOCK:
|
||||
if _RUNTIMES:
|
||||
raise HySdkRuntimeError("hy_sdk_runtime_leases_active")
|
||||
|
||||
|
||||
def _generation_access(
|
||||
profile: ModelProfileSnapshot,
|
||||
api_key: str | None,
|
||||
) -> tuple[str, str]:
|
||||
if profile.provider_kind is ProviderKind.LOCAL_ONNX:
|
||||
return _LOCAL_ONNX_PLACEHOLDER_URL, "local-read-only"
|
||||
if profile.provider_kind not in {
|
||||
ProviderKind.OPENAI_COMPATIBLE,
|
||||
ProviderKind.OLLAMA,
|
||||
}:
|
||||
raise HySdkRuntimeError("hy_sdk_generation_provider_unsupported")
|
||||
endpoint = _openai_endpoint(profile.provider_kind, profile.base_url)
|
||||
if profile.provider_kind is ProviderKind.OLLAMA:
|
||||
return endpoint, "ollama"
|
||||
if not api_key:
|
||||
raise HySdkRuntimeError("hy_sdk_model_api_key_missing")
|
||||
return endpoint, api_key
|
||||
|
||||
|
||||
def _openai_endpoint(provider: ProviderKind, endpoint: str | None) -> str:
|
||||
value = (endpoint or "").rstrip("/")
|
||||
if not value:
|
||||
raise HySdkRuntimeError("hy_sdk_model_endpoint_missing")
|
||||
if provider is ProviderKind.OLLAMA and not value.endswith("/v1"):
|
||||
return f"{value}/v1"
|
||||
return value
|
||||
|
||||
|
||||
def _acquire_runtime(config: MemoryConfig, key: Path) -> SharedRuntime:
|
||||
with _RUNTIME_LOCK:
|
||||
entry = _RUNTIMES.get(key)
|
||||
if entry is None:
|
||||
runtime = _run_sync(SharedRuntime.create(config))
|
||||
_RUNTIMES[key] = _RuntimeEntry(runtime=runtime, references=1)
|
||||
return runtime
|
||||
entry.references += 1
|
||||
return entry.runtime
|
||||
|
||||
|
||||
def _release_runtime(key: Path) -> None:
|
||||
runtime: SharedRuntime | None = None
|
||||
with _RUNTIME_LOCK:
|
||||
entry = _RUNTIMES.get(key)
|
||||
if entry is None:
|
||||
return
|
||||
entry.references -= 1
|
||||
if entry.references == 0:
|
||||
runtime = entry.runtime
|
||||
del _RUNTIMES[key]
|
||||
if runtime is not None:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def _run_sync[T](coroutine: Coroutine[Any, Any, T]) -> T:
|
||||
"""Run SDK async construction from either sync or async caller threads."""
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return asyncio.run(coroutine)
|
||||
|
||||
result: list[T] = []
|
||||
failure: list[BaseException] = []
|
||||
|
||||
def run() -> None:
|
||||
try:
|
||||
result.append(asyncio.run(coroutine))
|
||||
except BaseException as error:
|
||||
failure.append(error)
|
||||
|
||||
thread = threading.Thread(target=run, name="hy-sdk-runtime-create")
|
||||
thread.start()
|
||||
thread.join()
|
||||
if failure:
|
||||
raise failure[0]
|
||||
return result[0]
|
||||
@@ -1,402 +0,0 @@
|
||||
"""Validated records and response parsers for public HY SDK operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
|
||||
|
||||
class HySdkError(RuntimeError):
|
||||
"""Base error carrying a stable, product-facing code."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class HySdkRuntimeError(HySdkError):
|
||||
"""The public SDK runtime could not be configured or leased."""
|
||||
|
||||
|
||||
class HySdkMigrationError(HySdkError):
|
||||
"""The privileged offline SDK data-plane migration failed."""
|
||||
|
||||
|
||||
class HySdkReadError(HySdkError):
|
||||
"""A public SDK read operation failed or returned an invalid shape."""
|
||||
|
||||
|
||||
class HySdkMemoryRecord(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
memory_id: str
|
||||
content: str
|
||||
layer: str
|
||||
status: str
|
||||
is_latest: bool
|
||||
supersedes: tuple[str, ...] = ()
|
||||
superseded_by: tuple[str, ...] = ()
|
||||
tags: tuple[str, ...] = ()
|
||||
owner: str | None = None
|
||||
source_raw_ids: tuple[str, ...] = ()
|
||||
user_id: str
|
||||
agent_id: str
|
||||
session_id: str | None = None
|
||||
source: str | None = None
|
||||
memory_at: int | None = None
|
||||
gmt_created: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_sdk(cls, raw: object) -> HySdkMemoryRecord:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_record_shape_unknown")
|
||||
if any(
|
||||
not isinstance(raw.get(key), str) or not str(raw[key]).strip()
|
||||
for key in ("memory_id", "content")
|
||||
):
|
||||
raise HySdkReadError("hy_memory_record_shape_unknown")
|
||||
try:
|
||||
return cls(
|
||||
memory_id=str(raw["memory_id"]),
|
||||
content=str(raw["content"]),
|
||||
layer=str(raw.get("layer") or ""),
|
||||
status=str(raw.get("status") or ""),
|
||||
is_latest=bool(raw.get("is_latest", True)),
|
||||
supersedes=_string_tuple(raw.get("supersedes")),
|
||||
superseded_by=_string_tuple(raw.get("superseded_by")),
|
||||
tags=_string_tuple(raw.get("tags")),
|
||||
owner=(str(raw["owner"]) if raw.get("owner") in {"user", "agent"} else None),
|
||||
source_raw_ids=_source_raw_ids(raw),
|
||||
user_id=str(raw.get("user_id") or ""),
|
||||
agent_id=str(raw.get("agent_id") or ""),
|
||||
session_id=(
|
||||
str(raw["session_id"]) if isinstance(raw.get("session_id"), str) else None
|
||||
),
|
||||
source=str(raw["source"]) if isinstance(raw.get("source"), str) else None,
|
||||
memory_at=raw.get("memory_at") if isinstance(raw.get("memory_at"), int) else None,
|
||||
gmt_created=(
|
||||
raw.get("gmt_created") if isinstance(raw.get("gmt_created"), int) else None
|
||||
),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise HySdkReadError("hy_memory_record_shape_unknown") from error
|
||||
|
||||
|
||||
class HySdkMemoryPage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
memories: tuple[HySdkMemoryRecord, ...]
|
||||
total: int = Field(ge=0)
|
||||
limit: int = Field(ge=1)
|
||||
offset: int = Field(ge=0)
|
||||
|
||||
@classmethod
|
||||
def from_sdk(cls, raw: object, *, limit: int, offset: int) -> HySdkMemoryPage:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_list_shape_unknown")
|
||||
rows: list[object] = []
|
||||
total = 0
|
||||
for bucket_name, rows_name in (("vdb", "memories"), ("graph", "nodes")):
|
||||
bucket = raw.get(bucket_name)
|
||||
if bucket is None and bucket_name == "graph":
|
||||
continue
|
||||
if not isinstance(bucket, Mapping) or not isinstance(bucket.get(rows_name), list):
|
||||
raise HySdkReadError("hy_memory_list_shape_unknown")
|
||||
bucket_total = bucket.get("total")
|
||||
if (
|
||||
not isinstance(bucket_total, int)
|
||||
or isinstance(bucket_total, bool)
|
||||
or bucket_total < 0
|
||||
):
|
||||
raise HySdkReadError("hy_memory_list_shape_unknown")
|
||||
total += bucket_total
|
||||
rows.extend(bucket[rows_name])
|
||||
deduplicated: dict[str, HySdkMemoryRecord] = {}
|
||||
for row in rows:
|
||||
record = HySdkMemoryRecord.from_sdk(row)
|
||||
deduplicated.setdefault(record.memory_id, record)
|
||||
return cls(
|
||||
memories=tuple(deduplicated.values()),
|
||||
total=total,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
class HySdkRawRecord(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
raw_id: str
|
||||
content: str
|
||||
user_id: str
|
||||
agent_id: str
|
||||
session_id: str
|
||||
source: str
|
||||
memory_at: int | None = None
|
||||
gmt_created: int | None = None
|
||||
custom: dict[str, object] = Field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def from_sdk(cls, raw: object) -> HySdkRawRecord:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_raw_shape_unknown")
|
||||
if any(
|
||||
not isinstance(raw.get(key), str) or not str(raw[key]).strip()
|
||||
for key in ("raw_id", "content", "user_id", "agent_id", "session_id", "source")
|
||||
):
|
||||
raise HySdkReadError("hy_memory_raw_shape_unknown")
|
||||
custom = raw.get("custom")
|
||||
if not isinstance(custom, Mapping):
|
||||
custom = {}
|
||||
try:
|
||||
return cls(
|
||||
raw_id=str(raw["raw_id"]),
|
||||
content=str(raw["content"]),
|
||||
user_id=str(raw["user_id"]),
|
||||
agent_id=str(raw["agent_id"]),
|
||||
session_id=str(raw["session_id"]),
|
||||
source=str(raw["source"]),
|
||||
memory_at=raw.get("memory_at") if isinstance(raw.get("memory_at"), int) else None,
|
||||
gmt_created=(
|
||||
raw.get("gmt_created") if isinstance(raw.get("gmt_created"), int) else None
|
||||
),
|
||||
custom=dict(custom),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise HySdkReadError("hy_memory_raw_shape_unknown") from error
|
||||
|
||||
|
||||
class HySdkRawHit(HySdkRawRecord):
|
||||
score: float = Field(ge=0, le=1)
|
||||
|
||||
@classmethod
|
||||
def from_sdk_search(cls, raw: object) -> HySdkRawHit:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_raw_search_shape_unknown")
|
||||
raw_id = raw.get("memory_id")
|
||||
score = raw.get("score")
|
||||
if (
|
||||
not isinstance(raw_id, str)
|
||||
or not isinstance(score, int | float)
|
||||
or isinstance(score, bool)
|
||||
):
|
||||
raise HySdkReadError("hy_memory_raw_search_shape_unknown")
|
||||
base = HySdkRawRecord.from_sdk({**raw, "raw_id": raw_id})
|
||||
return cls(
|
||||
**base.model_dump(),
|
||||
score=max(0.0, min(1.0, float(score))),
|
||||
)
|
||||
|
||||
|
||||
class HySdkRawPage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
raw: tuple[HySdkRawRecord, ...]
|
||||
total: int = Field(ge=0)
|
||||
limit: int = Field(ge=1)
|
||||
offset: int = Field(ge=0)
|
||||
|
||||
@classmethod
|
||||
def from_memory_list(
|
||||
cls,
|
||||
raw: object,
|
||||
*,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> HySdkRawPage:
|
||||
"""Parse ``list_memories(layers=['l1_raw'])`` into the App raw DTO."""
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_raw_list_shape_unknown")
|
||||
bucket = raw.get("vdb")
|
||||
if not isinstance(bucket, Mapping):
|
||||
raise HySdkReadError("hy_memory_raw_list_shape_unknown")
|
||||
rows = bucket.get("memories")
|
||||
total = bucket.get("total")
|
||||
if (
|
||||
not isinstance(rows, list)
|
||||
or not isinstance(total, int)
|
||||
or isinstance(total, bool)
|
||||
or total < 0
|
||||
):
|
||||
raise HySdkReadError("hy_memory_raw_list_shape_unknown")
|
||||
converted = []
|
||||
for row in rows:
|
||||
if not isinstance(row, Mapping) or row.get("layer") != "l1_raw":
|
||||
raise HySdkReadError("hy_memory_raw_list_shape_unknown")
|
||||
converted.append({**row, "raw_id": row.get("memory_id")})
|
||||
return cls(
|
||||
raw=tuple(HySdkRawRecord.from_sdk(row) for row in converted),
|
||||
total=total,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
class HySdkUserProfile(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
request_id: str
|
||||
profile: dict[str, object] | None
|
||||
elapsed_ms: float = Field(ge=0)
|
||||
|
||||
@classmethod
|
||||
def from_sdk(cls, raw: object) -> HySdkUserProfile:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_profile_shape_unknown")
|
||||
request_id = raw.get("request_id")
|
||||
profile = raw.get("profile")
|
||||
elapsed_ms = raw.get("elapsed_ms")
|
||||
if (
|
||||
not isinstance(request_id, str)
|
||||
or (profile is not None and not isinstance(profile, Mapping))
|
||||
or not isinstance(elapsed_ms, int | float)
|
||||
or isinstance(elapsed_ms, bool)
|
||||
):
|
||||
raise HySdkReadError("hy_memory_profile_shape_unknown")
|
||||
return cls(
|
||||
request_id=request_id,
|
||||
profile=None if profile is None else dict(profile),
|
||||
elapsed_ms=float(elapsed_ms),
|
||||
)
|
||||
|
||||
|
||||
def parse_sdk_search(raw: object, *, error_code: str) -> tuple[SemanticMemoryHit, ...]:
|
||||
rows = _reply_rows(raw, error_code)
|
||||
found: list[SemanticMemoryHit] = []
|
||||
seen: set[str] = set()
|
||||
for item in sorted(rows, key=_score, reverse=True):
|
||||
memory_id = item.get("memory_id") or item.get("id")
|
||||
if not isinstance(memory_id, str) or not memory_id or memory_id in seen:
|
||||
continue
|
||||
seen.add(memory_id)
|
||||
content = item.get("content")
|
||||
member_ids, formatting_complete = _formatted_member_ids(item)
|
||||
found.append(
|
||||
SemanticMemoryHit(
|
||||
memory_id=memory_id,
|
||||
score=_score(item),
|
||||
formatted_content=(
|
||||
content.strip() if isinstance(content, str) and content.strip() else None
|
||||
),
|
||||
member_ids=member_ids,
|
||||
sdk_formatted=True,
|
||||
formatting_complete=formatting_complete,
|
||||
)
|
||||
)
|
||||
return tuple(found)
|
||||
|
||||
|
||||
def parse_sdk_raw_search(raw: object) -> tuple[HySdkRawHit, ...]:
|
||||
return tuple(
|
||||
HySdkRawHit.from_sdk_search(row)
|
||||
for row in _reply_rows(raw, "hy_memory_raw_search_shape_unknown")
|
||||
)
|
||||
|
||||
|
||||
def _string_tuple(value: object) -> tuple[str, ...]:
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(
|
||||
dict.fromkeys(item.strip() for item in value if isinstance(item, str) and item.strip())
|
||||
)
|
||||
|
||||
|
||||
def _source_raw_ids(raw: Mapping[str, object]) -> tuple[str, ...]:
|
||||
custom = raw.get("custom")
|
||||
values: list[object] = []
|
||||
if isinstance(custom, Mapping):
|
||||
listed = custom.get("source_raw_memory_ids")
|
||||
if isinstance(listed, list):
|
||||
values.extend(listed)
|
||||
singular = raw.get("source_raw_memory_id")
|
||||
if isinstance(singular, str):
|
||||
values.append(singular)
|
||||
return tuple(
|
||||
dict.fromkeys(value.strip() for value in values if isinstance(value, str) and value.strip())
|
||||
)
|
||||
|
||||
|
||||
def _reply_rows(raw: object, error_code: str) -> tuple[Mapping[str, object], ...]:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError(error_code)
|
||||
keys = tuple(key for key in ("memories", "coding_memories") if key in raw)
|
||||
if "scene" in raw:
|
||||
scene = raw["scene"]
|
||||
if not isinstance(scene, str):
|
||||
raise HySdkReadError(error_code)
|
||||
if scene in ("normal", "chat"):
|
||||
selected = "memories"
|
||||
elif scene in ("productivity", "working"):
|
||||
selected = "coding_memories"
|
||||
else:
|
||||
raise HySdkReadError(error_code)
|
||||
if selected not in keys:
|
||||
raise HySdkReadError(error_code)
|
||||
elif len(keys) == 1:
|
||||
selected = keys[0]
|
||||
else:
|
||||
raise HySdkReadError(error_code)
|
||||
|
||||
result: list[Mapping[str, object]] = []
|
||||
for key in keys:
|
||||
value = raw[key]
|
||||
if key == "memories" and isinstance(value, Mapping):
|
||||
if set(value) != {"schema", "proactive", "normal"}:
|
||||
raise HySdkReadError(error_code)
|
||||
channels = tuple(value.values())
|
||||
else:
|
||||
channels = (value,)
|
||||
for channel in channels:
|
||||
if not isinstance(channel, list) or any(
|
||||
not isinstance(row, Mapping) for row in channel
|
||||
):
|
||||
raise HySdkReadError(error_code)
|
||||
if key == selected:
|
||||
result.extend(channel)
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def _score(memory: Mapping[str, object]) -> float:
|
||||
score = memory.get("score")
|
||||
return float(score) if isinstance(score, int | float) and not isinstance(score, bool) else 0.0
|
||||
|
||||
|
||||
def _formatted_member_ids(memory: Mapping[str, object]) -> tuple[tuple[str, ...], bool]:
|
||||
represented: dict[str, None] = {}
|
||||
composite = False
|
||||
raw_members = memory.get("member_ids")
|
||||
if raw_members is not None:
|
||||
composite = True
|
||||
if not isinstance(raw_members, list | tuple) or any(
|
||||
not isinstance(member_id, str) or not member_id for member_id in raw_members
|
||||
):
|
||||
return (), False
|
||||
for member_id in raw_members:
|
||||
represented.setdefault(member_id, None)
|
||||
chain = memory.get("evolution_chain")
|
||||
if chain is not None:
|
||||
composite = True
|
||||
if not isinstance(chain, list | tuple):
|
||||
return (), False
|
||||
for entry in chain:
|
||||
if not isinstance(entry, Mapping):
|
||||
return (), False
|
||||
member_id = entry.get("node_id") or entry.get("memory_id")
|
||||
if not isinstance(member_id, str) or not member_id:
|
||||
return (), False
|
||||
represented.setdefault(member_id, None)
|
||||
if memory.get("capsule_memories") is not None or memory.get("timeline") is not None:
|
||||
composite = True
|
||||
if raw_members is None:
|
||||
return (), False
|
||||
content = memory.get("content")
|
||||
if isinstance(content, str) and content.lstrip().startswith(
|
||||
("Related memories:", "Matched memories:", "Memory timeline:")
|
||||
):
|
||||
composite = True
|
||||
if not represented:
|
||||
return (), False
|
||||
return tuple(represented), not composite or bool(represented)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Per-write original Evidence and where each HY memory came from; never a memory source."""
|
||||
|
||||
import json
|
||||
from collections.abc import Set as AbstractSet
|
||||
from importlib import import_module
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
|
||||
|
||||
class HySupportedMemory(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
hy_memory_id: str
|
||||
governed_memory_id: str
|
||||
revision: int
|
||||
content: str
|
||||
owner: Literal["user", "agent"]
|
||||
evidence_ids: tuple[str, ...]
|
||||
|
||||
|
||||
class HySourceContext(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
current: tuple[EvidenceRecord, ...]
|
||||
history: tuple[EvidenceRecord, ...] = ()
|
||||
supporting: tuple[EvidenceRecord, ...] = ()
|
||||
known_memories: tuple[HySupportedMemory, ...] = ()
|
||||
|
||||
|
||||
class HyWriteBindings:
|
||||
def __init__(self, context: HySourceContext) -> None:
|
||||
if not context.current:
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
self.context = context
|
||||
seed = context.current[0]
|
||||
records = (*context.current, *context.history, *context.supporting)
|
||||
workspaces = {record.workspace for record in records}
|
||||
# A missing workspace is a legacy identity, not a wildcard that may
|
||||
# absorb a named project. Keep NULL and named workspaces disjoint.
|
||||
if None in workspaces and len(workspaces) != 1:
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
ids = [record.evidence_id for record in records]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
for record in records:
|
||||
if (
|
||||
(
|
||||
record.user_id,
|
||||
record.source_id,
|
||||
record.conversation_id,
|
||||
record.workspace,
|
||||
)
|
||||
!= (
|
||||
seed.user_id,
|
||||
seed.source_id,
|
||||
seed.conversation_id,
|
||||
seed.workspace,
|
||||
)
|
||||
or record.content_kind is not EvidenceContentKind.RAW_CONVERSATION
|
||||
or record.role not in {"user", "assistant", "tool"}
|
||||
):
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
self.all_sources: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": record.evidence_id,
|
||||
"role": record.role,
|
||||
"content": sanitize_model_input(record.verbatim),
|
||||
"new": index < len(context.current),
|
||||
"revision": record.revision,
|
||||
"created_at": record.created_at.isoformat() if record.created_at else None,
|
||||
}
|
||||
for index, record in enumerate(records)
|
||||
]
|
||||
self.sources = self.all_sources[: len(context.current) + len(context.history)]
|
||||
self._supports: dict[tuple[str, str], tuple[str, ...]] = {}
|
||||
self.profile_order: list[str] | None = None
|
||||
self.verified_operations: dict[str, tuple[str, str]] = {}
|
||||
self.known = {memory.hy_memory_id: memory for memory in context.known_memories}
|
||||
for memory in context.known_memories:
|
||||
if not memory.evidence_ids or not set(memory.evidence_ids) <= set(ids):
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
self._supports[(memory.content, memory.owner)] = memory.evidence_ids
|
||||
|
||||
def check_content(self, content: str) -> None:
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except ValueError as error:
|
||||
raise ValueError("hy_memory_write_verification_input_mismatch") from error
|
||||
expected = [
|
||||
{"role": record.role, "content": sanitize_model_input(record.verbatim)}
|
||||
for record in self.context.current
|
||||
]
|
||||
# HY strips tool messages itself before chat extraction when it judges a turn chat.
|
||||
without_tools = [item for item in expected if item["role"] != "tool"]
|
||||
if parsed != expected and (without_tools == expected or parsed != without_tools):
|
||||
raise ValueError("hy_memory_write_verification_input_mismatch")
|
||||
|
||||
def history_json(self) -> str:
|
||||
return json.dumps(
|
||||
[
|
||||
{
|
||||
"evidence_id": source["id"],
|
||||
**{k: source[k] for k in ("role", "content", "revision", "created_at")},
|
||||
}
|
||||
for source in self.sources
|
||||
if not source["new"]
|
||||
],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
def turn_support(
|
||||
self, owner: str | None, targets: AbstractSet[str] = frozenset()
|
||||
) -> tuple[str, ...]:
|
||||
"""This turn's words by the memory's author, plus what a merged memory already cited."""
|
||||
roles = {"user": {"user"}, "agent": {"assistant"}}.get(owner or "")
|
||||
current = [source for source in self.all_sources if source["new"]]
|
||||
ids = [source["id"] for source in current if roles is None or source["role"] in roles]
|
||||
if not ids:
|
||||
ids = [source["id"] for source in current]
|
||||
for target in sorted(targets):
|
||||
if target in self.known:
|
||||
ids.extend(self.known[target].evidence_ids)
|
||||
return tuple(dict.fromkeys(ids))
|
||||
|
||||
def record_turn_supports(self, items: list[dict[str, Any]]) -> None:
|
||||
for item in items:
|
||||
key = (item["content"], item["owner"] or "")
|
||||
self._supports.setdefault(key, self.turn_support(item["owner"]))
|
||||
|
||||
def record_operation_support(self, content: str, owner: str | None, targets: set[str]) -> None:
|
||||
key = (content, owner or "")
|
||||
merged = (*self._supports.get(key, ()), *self.turn_support(owner, targets))
|
||||
self._supports[key] = tuple(dict.fromkeys(merged))
|
||||
|
||||
def support_for(self, content: str, owner: str | None) -> tuple[str, ...]:
|
||||
return self._supports.get((content, owner or ""), ())
|
||||
|
||||
def profile_support(self, content: str, values: dict[str, Any]) -> tuple[str, ...]:
|
||||
render = import_module("hy_memory.agent.tools.basic_profile")._render_content
|
||||
|
||||
if not values or render(values, field_order=self.profile_order) != content:
|
||||
return ()
|
||||
ids: dict[str, None] = {}
|
||||
for key, value in values.items():
|
||||
support = self.support_for(f"{key}: {value}", "user")
|
||||
if not support:
|
||||
return ()
|
||||
ids.update(dict.fromkeys(support))
|
||||
return tuple(ids)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Input checks around HY's native extractor; HY's output is published as it returns it."""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from one_memory.hy_source_context import HyWriteBindings
|
||||
|
||||
|
||||
def _extracted_items(info: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
for section in ("memory", "intentions"):
|
||||
for item in info.get(section) or []:
|
||||
if isinstance(item, dict) and isinstance(item.get("content"), str):
|
||||
items.append({"content": item["content"], "owner": item.get("owner")})
|
||||
for name, value in (info.get("basic_info") or {}).items():
|
||||
items.append({"content": f"{name}: {value}", "owner": "user"})
|
||||
return items
|
||||
|
||||
|
||||
class HyExtractionGuard:
|
||||
def __init__(self, extractor: Any) -> None:
|
||||
self.extractor = extractor
|
||||
self.failures: list[str] = []
|
||||
self._lock = asyncio.Lock()
|
||||
self.bindings: HyWriteBindings | None = None
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self.extractor, name)
|
||||
|
||||
async def extract(self, *args: Any, **kwargs: Any) -> Any:
|
||||
async with self._lock:
|
||||
try:
|
||||
return await self._extract(args, kwargs)
|
||||
except ValueError as error:
|
||||
if str(error).startswith("hy_memory_write_verification"):
|
||||
self.failures.append(str(error))
|
||||
raise
|
||||
|
||||
async def _extract(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any:
|
||||
content = kwargs.get("content", args[0] if args else "")
|
||||
bindings = self.bindings
|
||||
if bindings is not None:
|
||||
bindings.check_content(content)
|
||||
fields = kwargs.get("basic_profile_fields")
|
||||
bindings.profile_order = list(fields) if fields else None
|
||||
kwargs = {**kwargs, "history_context": bindings.history_json()}
|
||||
else:
|
||||
# No raw-history identity was supplied; do not trust HY's text prefixes.
|
||||
kwargs = {**kwargs, "history_context": ""}
|
||||
result = await self.extractor.extract(*args, **kwargs)
|
||||
if bindings is not None and result.success:
|
||||
bindings.record_turn_supports(_extracted_items(result.info))
|
||||
return result
|
||||
@@ -557,10 +557,12 @@ class PollingLiveSyncSupervisor:
|
||||
)
|
||||
|
||||
def _note_internal_error(self, source_id: str, error: Exception) -> None:
|
||||
# The ordinary local log keeps the exception message for debugging.
|
||||
# The structured diagnostic event deliberately keeps only the stable
|
||||
# error code, source id, type, and relative stack: exception messages
|
||||
# can contain paths, query text, or credentials and are exportable data.
|
||||
# Type alone used to be all this said. It is not enough to act on:
|
||||
# `OperationalError` covers a locked database, a missing table and a
|
||||
# corrupt page alike, and without the message and the frames two dozen
|
||||
# of these in a log could only be guessed at. The stack is the
|
||||
# redacted, `one_memory`-relative one, so it names the call site
|
||||
# without printing anyone's paths or query text.
|
||||
logging.getLogger(__name__).warning(
|
||||
"live_sync_internal_error %s: %s", type(error).__name__, error
|
||||
)
|
||||
@@ -570,7 +572,10 @@ class PollingLiveSyncSupervisor:
|
||||
event="live_sync.internal_error",
|
||||
outcome="failed",
|
||||
error_code="live_sync_internal_error",
|
||||
details={"source_id": source_id},
|
||||
# `reason` is the listed key that can carry the message; see
|
||||
# `diagnostic_log._DETAIL_KEYS`. Truncated because a detail value
|
||||
# longer than 256 characters is rejected.
|
||||
details={"source_id": source_id, "reason": str(error)[:200]},
|
||||
**diagnostic_exception(error),
|
||||
)
|
||||
if isinstance(error, sqlite3.Error):
|
||||
|
||||
@@ -867,7 +867,10 @@ class ImportRunner:
|
||||
outcome="failed",
|
||||
job_id=job_id,
|
||||
error_code="import_heartbeat_failed",
|
||||
details={"phase": "heartbeat"},
|
||||
# `reason` is the listed key for this; see
|
||||
# `diagnostic_log._DETAIL_KEYS`. Truncated: a detail value
|
||||
# over 256 characters is rejected.
|
||||
details={"reason": str(error)[:200]},
|
||||
**diagnostic_exception(error),
|
||||
)
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from pathlib import PurePosixPath
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from one_memory.derived_echo import strip_injected_memory
|
||||
from one_memory.imports.access import (
|
||||
AuthorizedSourceFile,
|
||||
AuthorizedSourceRoot,
|
||||
@@ -214,7 +213,6 @@ def parse_hermes_rollout_record(
|
||||
message.get("content"),
|
||||
allowed_block_types=frozenset({"text", "input_text", "output_text"}),
|
||||
)
|
||||
text = strip_injected_memory(text)
|
||||
if not text:
|
||||
return ParsedSkip(
|
||||
reason_code=ScanIssueCode.NO_TEXT,
|
||||
@@ -388,8 +386,7 @@ def _read_state_db(
|
||||
# The assistant's call sits on its own row as `tool_calls`; the
|
||||
# `tool` rows after it hold only what came back.
|
||||
calls = _tool_calls_text(row["tool_calls"]) if role == "assistant" else None
|
||||
content = strip_injected_memory(str(row["content"] or ""))
|
||||
has_text = bool(content.strip())
|
||||
has_text = bool(str(row["content"] or "").strip())
|
||||
if role not in {"user", "assistant"} or (not has_text and calls is None):
|
||||
yield ScanSkip(
|
||||
cursor=cursor,
|
||||
@@ -404,7 +401,7 @@ def _read_state_db(
|
||||
role=str(role) if has_text else "tool",
|
||||
written_by=_written_by(role, row["session_source"]) if has_text else WrittenBy.TOOL,
|
||||
created_at=parse_timestamp(row["timestamp"]),
|
||||
verbatim=content if has_text else str(calls),
|
||||
verbatim=str(row["content"]) if has_text else str(calls),
|
||||
workspace=workspace,
|
||||
project=safe_project_label(workspace),
|
||||
record_type="state_db_message" if has_text else "state_db_message.tool",
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
"""Passive lifecycle projection used only by retired Governance readers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
|
||||
|
||||
@dataclass(frozen=True, eq=False)
|
||||
class HyLifecycle:
|
||||
hidden: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "hidden", MappingProxyType(dict(self.hidden)))
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return isinstance(other, HyLifecycle) and dict(self.hidden) == dict(
|
||||
other.hidden
|
||||
)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(tuple(sorted(self.hidden.items())))
|
||||
|
||||
def replaced_by(self, memory_id: str) -> tuple[str, ...]:
|
||||
return self.hidden.get(memory_id, ())
|
||||
|
||||
def replacing(self, memory_id: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
old
|
||||
for old, replacements in self.hidden.items()
|
||||
if memory_id in replacements
|
||||
)
|
||||
)
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Write visible hook text into Evidence. Tool output stays out."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
@@ -75,7 +74,6 @@ _FILE_FALLBACK = {
|
||||
}
|
||||
_FALLBACK_EVENTS = frozenset({"Stop", "stop", "SubagentStop", "StopFailure"})
|
||||
_TEXT_BLOCKS = frozenset({"text"})
|
||||
SDK_MIGRATION_CAPTURE_FENCE = ".sdk-migration-capture-fence.json"
|
||||
|
||||
|
||||
def database_path() -> Path:
|
||||
@@ -90,42 +88,17 @@ def capture_stdio(source_id: str, stdin: TextIO, output: TextIO) -> int:
|
||||
del output
|
||||
if source_id not in CAPTURE_SOURCES:
|
||||
return 2
|
||||
database = database_path()
|
||||
if sdk_migration_capture_fenced(database.parent):
|
||||
return 0
|
||||
try:
|
||||
payload = json.loads(stdin.read() or "")
|
||||
except json.JSONDecodeError:
|
||||
return 0
|
||||
try:
|
||||
capture_hook(source_id, payload, database)
|
||||
capture_hook(source_id, payload, database_path())
|
||||
except Exception:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def sdk_migration_capture_fenced(data_dir: Path) -> bool:
|
||||
"""Ignore stale fences; only a live Desktop migration pauses hook writes."""
|
||||
marker = data_dir / SDK_MIGRATION_CAPTURE_FENCE
|
||||
try:
|
||||
document = json.loads(marker.read_text(encoding="utf-8"))
|
||||
pid = int(document["pid"])
|
||||
if pid < 1:
|
||||
raise ValueError("invalid migration fence pid")
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except ProcessLookupError:
|
||||
marker.unlink(missing_ok=True)
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError, OSError):
|
||||
marker.unlink(missing_ok=True)
|
||||
return False
|
||||
|
||||
|
||||
def capture_hook(source_id: str, payload: object, database: Path) -> bool:
|
||||
if source_id not in CAPTURE_SOURCES or not isinstance(payload, dict):
|
||||
return False
|
||||
@@ -251,7 +224,7 @@ def record_capture_rejection(
|
||||
def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed: bool) -> None:
|
||||
if not changed or not _sync_open(store.database_path, record.source_id):
|
||||
return
|
||||
from one_memory.conversation_processing import _conversation_chunks
|
||||
from one_memory.memory_processing import _conversation_chunks
|
||||
from one_memory.memory_server.durable_queue import DurableWriteQueue
|
||||
from one_memory.memory_server.unified_entry import UnifiedWriteIntake
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ class LocalEmbedder:
|
||||
raise MissingEmbeddingAssetsError(f"{_FETCH_HINT} Looked in {directory}.")
|
||||
|
||||
# Imported by name for the same reason kuzu and hy_memory are in
|
||||
# onnxruntime ships no py.typed, and a dynamic
|
||||
# hy_memory_bridge: onnxruntime ships no py.typed, and a dynamic
|
||||
# import keeps the strict checker on everything around it.
|
||||
ort = import_module("onnxruntime")
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
@@ -14,7 +14,6 @@ stuck in it.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
@@ -24,7 +23,6 @@ from types import FrameType
|
||||
logger = logging.getLogger("one_memory.loop_watchdog")
|
||||
|
||||
DEFAULT_STALL_SECONDS = 2.0
|
||||
_SAFE_FALLBACK_FILE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$")
|
||||
|
||||
|
||||
class LoopWatchdog:
|
||||
@@ -122,28 +120,22 @@ def _safe_stack_for_main_thread() -> tuple[str, ...]:
|
||||
|
||||
|
||||
def _safe_stacks_for_all_threads() -> tuple[str, ...]:
|
||||
"""Return bounded stack summaries without recording thread names."""
|
||||
"""Return one content-free, bounded stack summary for every live thread."""
|
||||
names = {thread.ident: thread.name for thread in threading.enumerate()}
|
||||
lines: list[str] = []
|
||||
for _ident, frame in sorted(sys._current_frames().items()):
|
||||
for ident, frame in sorted(sys._current_frames().items()):
|
||||
stack: list[str] = []
|
||||
# Most specific frame first; append only complete frames so truncation
|
||||
# can never turn a valid location into free text.
|
||||
# Most specific frame first so the 256-character privacy bound keeps
|
||||
# the likely culprit rather than only the thread bootstrap frames.
|
||||
for entry in reversed(traceback.extract_stack(frame)[-8:]):
|
||||
parts = entry.filename.replace("\\", "/").split("/")
|
||||
try:
|
||||
root = parts.index("one_memory")
|
||||
relative = "/".join(parts[root:])
|
||||
except ValueError:
|
||||
fallback = parts[-1]
|
||||
relative = fallback if _SAFE_FALLBACK_FILE.fullmatch(fallback) else "unknown.py"
|
||||
relative = parts[-1]
|
||||
name = entry.name if entry.name.isidentifier() else "call"
|
||||
candidate = f"{relative}:{entry.lineno}:{name}"
|
||||
separator = "" if not stack else " > "
|
||||
if len("thread|") + len(" > ".join(stack)) + len(separator) + len(candidate) > 256:
|
||||
break
|
||||
stack.append(candidate)
|
||||
if stack:
|
||||
lines.append(f"thread|{' > '.join(stack)}")
|
||||
if len(lines) == 16:
|
||||
break
|
||||
stack.append(f"{relative}:{entry.lineno}:{name}")
|
||||
thread_name = names.get(ident, "unknown").replace("\n", " ").replace("\r", " ")
|
||||
lines.append(f"thread={thread_name[:48]}|{' > '.join(stack)}"[:256])
|
||||
return tuple(lines)
|
||||
@@ -7,6 +7,7 @@ segment-key breakage). Call sites still import this module; registration is
|
||||
now a no-op that always admits the row.
|
||||
|
||||
Proof rows may still be written by the ingest path; they are ignored on read.
|
||||
See ``新鲜度_todo.md`` for the remaining teardown steps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,7 +22,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.database import enable_write_ahead_log
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.legacy_lifecycle_types import HyLifecycle
|
||||
from one_memory.hy_lifecycle import HyLifecycle
|
||||
from one_memory.library_upgrade import LibraryUpgradeError, PreparedLibrary, holds_legacy_rows
|
||||
from one_memory.memory_server.migrations import governance_ddl
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Literal
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.legacy_lifecycle_types import HyLifecycle
|
||||
from one_memory.hy_lifecycle import HyLifecycle
|
||||
from one_memory.memory_evidence_freshness import register_memory_freshness
|
||||
from one_memory.memory_governance import (
|
||||
GovernedMemory,
|
||||
|
||||
@@ -3,7 +3,7 @@ import json
|
||||
import sqlite3
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal, cast
|
||||
from typing import Literal, cast
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
|
||||
@@ -21,10 +21,11 @@ from one_memory.imports.models import (
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
from one_memory.memory_governance import MemoryGovernanceStore, MemoryStatus
|
||||
from one_memory.memory_governance import MemoryStatus
|
||||
from one_memory.memory_processing import MemoryProcessingSupervisor
|
||||
from one_memory.memory_server.durable_queue import DurableWriteQueue
|
||||
from one_memory.memory_server.unified_entry import UnifiedWriteIntake
|
||||
from one_memory.memory_tasks import MemoryTaskStatus, MemoryTaskStore
|
||||
from one_memory.memory_tasks import MemoryTaskStatus
|
||||
from one_memory.secrets import SecretStoreUnavailableError
|
||||
|
||||
SOURCE_ID = "one-memory-manual"
|
||||
@@ -81,23 +82,11 @@ class SavedMemoryNoteEvidence(BaseModel):
|
||||
verbatim: str
|
||||
|
||||
|
||||
class LegacyMemoryNoteReader:
|
||||
"""Read pre-cutover note receipts without retaining any write runtime."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tasks: MemoryTaskStore,
|
||||
governance: MemoryGovernanceStore,
|
||||
) -> None:
|
||||
self.tasks = tasks
|
||||
self.governance = governance
|
||||
|
||||
|
||||
class MemoryNoteStore:
|
||||
def __init__(
|
||||
self,
|
||||
evidence: EvidenceStore,
|
||||
processor: Any | None,
|
||||
processor: MemoryProcessingSupervisor,
|
||||
intake: UnifiedWriteIntake | None = None,
|
||||
*,
|
||||
sdk_authority: bool = False,
|
||||
@@ -263,8 +252,6 @@ class MemoryNoteStore:
|
||||
memory_ids=(list(durable.sdk_memory_ids) if durable is not None else []),
|
||||
error=durable.detail if durable is not None else "write_missing",
|
||||
)
|
||||
if self.processor is None:
|
||||
raise RuntimeError("legacy_memory_note_processor_unavailable")
|
||||
task = self.processor.tasks.task_for_evidence(evidence_id)
|
||||
memory_ids = []
|
||||
if task:
|
||||
@@ -369,8 +356,6 @@ def memory_note_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
try:
|
||||
if notes.sdk_authority:
|
||||
return await asyncio.to_thread(notes.receipt, record.evidence_id)
|
||||
if notes.processor is None:
|
||||
raise RuntimeError("legacy_memory_note_processor_unavailable")
|
||||
if notes.intake is None:
|
||||
# Isolated tests construct the store without a queue.
|
||||
await notes.processor.enqueue_conversation_off_thread(record)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -69,8 +69,8 @@ _TYPE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
|
||||
def safe_error_code(error: BaseException) -> str:
|
||||
"""Reduce an exception to a storable code, never its message text.
|
||||
|
||||
A recognised code (the message *is* a stable domain code) is kept so
|
||||
callers can classify it.
|
||||
A recognised code (the message *is* the code, as `HyMemoryBridgeError`
|
||||
and the permission guard raise them) is kept so callers can classify it.
|
||||
Everything else collapses to its exception type, which is a fixed
|
||||
identifier rather than content.
|
||||
"""
|
||||
@@ -239,8 +239,8 @@ PERMISSION_REQUIRED_CODES = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
#: Model-side codes the SDK write coordinator raises before it can lease a
|
||||
#: client (no active profile, missing credential, secure storage down). They prove
|
||||
#: Model-side codes `process_records` raises *before* it can open the bridge
|
||||
#: (no active profile, missing credential, secure storage down). They prove
|
||||
#: nothing was sent, so the write parks instead of being lost. Transport codes
|
||||
#: that can arrive after a request was in flight stay `unknown`.
|
||||
MODEL_NOT_READY_CODES = frozenset(
|
||||
@@ -251,9 +251,9 @@ MODEL_NOT_READY_CODES = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
#: Deterministic, pre-SDK refusals: the frozen input no longer matches
|
||||
#: Deterministic, pre-bridge refusals: the frozen input no longer matches
|
||||
#: current Evidence, or a turn exceeds the model's input window. Both are
|
||||
#: raised while validating the records, before any SDK call, so
|
||||
#: raised while validating the records, before any bridge or HY call, so
|
||||
#: nothing was sent and retrying the same intent changes nothing — a terminal
|
||||
#: `failed`, not `unknown` (which would wrongly imply a possibly-applied
|
||||
#: write) and not a retry (which would fail identically).
|
||||
@@ -358,10 +358,12 @@ class UnknownWriteJob:
|
||||
|
||||
`request_id` is the durable queue's idempotency key. It identifies the row
|
||||
for queue inspection and is the value `DurableWriteQueue.get()` accepts.
|
||||
It is the public SDK request id used for exact outcome inquiry. This is
|
||||
discovery only: it does not decide whether HY has the write, and it
|
||||
deliberately exposes no transition method that could turn `unknown` back
|
||||
into retryable work.
|
||||
It is **not** the HY request id: `MemoryProcessingSupervisor` derives the
|
||||
HY handle from the task it creates, so an inquiry implementation must
|
||||
obtain that task-layer handle separately rather than passing this value to
|
||||
HY. This is discovery only: it does not decide whether HY has the write,
|
||||
and it deliberately exposes no transition method that could turn
|
||||
`unknown` back into retryable work.
|
||||
"""
|
||||
|
||||
job_id: str
|
||||
@@ -1757,8 +1759,8 @@ class DurableWriteQueue:
|
||||
"""Retire an `unknown` job after an inquiry, never by guessing.
|
||||
|
||||
`unknown` means HY may already hold the write; the only safe way out is
|
||||
to call the public SDK `get_write_outcome(request_id)` and then record
|
||||
what the inquiry found. `found=True` closes it as
|
||||
to ask by request id (`hy_capabilities` measured that the SDK supports
|
||||
this) and then record what the inquiry found. `found=True` closes it as
|
||||
`completed`; `found=False` gives up and marks it `dead`. The state
|
||||
machine's legal transitions are the authority, so this refuses any
|
||||
change that `job_state.next_state` would reject — in particular it can
|
||||
@@ -2256,11 +2258,12 @@ async def execute_job(
|
||||
admit_hy_invocation: Callable[[str, Callable[[], Awaitable[bool]]], Awaitable[bool]]
|
||||
| None = None,
|
||||
) -> None:
|
||||
"""Run one durable job through the SDK-authority processor.
|
||||
"""Run one durable job through the existing authorized processing path.
|
||||
|
||||
``process_sdk_job`` owns the one public SDK write. ``hy_invoked`` is
|
||||
recorded immediately before that call, so a crash mid-write is later
|
||||
reclaimed to ``unknown`` instead of being retried blindly.
|
||||
The executor does not call HY directly: `process_records` owns the
|
||||
permission guard, the runner and governance. `hy_invoked` is recorded
|
||||
before that call, so a crash mid-write is later reclaimed to `unknown`
|
||||
instead of being retried blindly.
|
||||
"""
|
||||
# Every store call here is synchronous SQLite work on the single writer or
|
||||
# a reader; made from the loop it would block health checks and the API for
|
||||
@@ -2333,13 +2336,38 @@ async def execute_job(
|
||||
try:
|
||||
level = ServiceLevel.REALTIME if request.priority == "live" else ServiceLevel.BACKGROUND
|
||||
with service_level_context(level):
|
||||
sdk_result = await supervisor.process_sdk_job(
|
||||
records,
|
||||
request_id=request.idempotency_key,
|
||||
before_hy_invocation=mark_hy_invocation,
|
||||
)
|
||||
raw_ids = sdk_result.raw_ids
|
||||
memory_ids = sdk_result.memory_ids
|
||||
authority = getattr(supervisor, "sdk_authority", None)
|
||||
if callable(authority) and authority():
|
||||
sdk_result = await supervisor.process_sdk_job(
|
||||
records,
|
||||
request_id=request.idempotency_key,
|
||||
before_hy_invocation=mark_hy_invocation,
|
||||
)
|
||||
raw_ids = sdk_result.raw_ids
|
||||
memory_ids = tuple(memory.memory_id for memory in sdk_result.memories)
|
||||
else:
|
||||
completed_tasks = (
|
||||
await supervisor.process_records(
|
||||
records,
|
||||
before_hy_invocation=mark_hy_invocation,
|
||||
source_admitted=True,
|
||||
)
|
||||
or ()
|
||||
)
|
||||
receipts = []
|
||||
for task in completed_tasks:
|
||||
receipt = await in_the_background(supervisor.tasks.sdk_receipt, task.task_id)
|
||||
if receipt is None:
|
||||
raise RuntimeError("sdk_write_receipt_missing")
|
||||
receipts.append(receipt)
|
||||
raw_ids = tuple(
|
||||
dict.fromkeys(raw_id for receipt in receipts for raw_id in receipt.raw_ids)
|
||||
)
|
||||
memory_ids = tuple(
|
||||
dict.fromkeys(
|
||||
memory_id for receipt in receipts for memory_id in receipt.memory_ids
|
||||
)
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - classified below
|
||||
# Never persist the raw message: it may be a provider response or the
|
||||
# user's own text, and `last_error` is shown over HTTP as `detail`.
|
||||
@@ -2354,7 +2382,7 @@ async def execute_job(
|
||||
):
|
||||
await in_the_background(queue.release_unstarted, job_id, owner=owner)
|
||||
return
|
||||
# The write coordinator classifies these before any SDK client is leased, so HY
|
||||
# The task layer classifies these before any bridge is opened, so HY
|
||||
# was provably not called. They must not read as `failed`:
|
||||
# - authorization stop -> cancelled (the user turned it off);
|
||||
# - no permission yet -> waiting_for_permission (parked, re-admitted
|
||||
@@ -2417,10 +2445,13 @@ async def execute_job(
|
||||
sdk_raw_ids=raw_ids,
|
||||
sdk_memory_ids=memory_ids,
|
||||
)
|
||||
authority = getattr(supervisor, "sdk_authority", None)
|
||||
release = getattr(evidence, "release_sdk_staging", None)
|
||||
durable_receipt = await in_the_background(queue.sdk_receipt, job_id)
|
||||
if (
|
||||
callable(release)
|
||||
callable(authority)
|
||||
and authority()
|
||||
and callable(release)
|
||||
and durable_receipt is not None
|
||||
and (durable_receipt.raw_ids or durable_receipt.memory_ids)
|
||||
):
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""A0-03: measured answer to "can a HY write be made idempotent?".
|
||||
|
||||
The result decides whether the `unknown` job state can ever be retired, so it
|
||||
is probed against the installed SDK rather than written down from memory.
|
||||
|
||||
What is verified against `hy-memory==2.0.5.dev3`:
|
||||
|
||||
- `HyMemoryClient.add(..., request_id=...)` accepts a caller-supplied id and
|
||||
atomically claims it in the SDK cache before invoking a write. Replays return
|
||||
the durable terminal result; running/unknown claims are not invoked again.
|
||||
- `HyMemoryClient.get_write_outcome(request_id)` reads that durable outcome
|
||||
directly; it does not depend on optional pipeline tracing.
|
||||
|
||||
A first pass at this probe got both answers wrong, and the failure mode is
|
||||
worth keeping in mind: searching SDK source for the string "idempotent" hits a
|
||||
docstring about *episode* nodes ("calls are idempotent") and a "dedup" comment
|
||||
about search over-sampling. Neither is write idempotency. So the probe checks
|
||||
the `add` path specifically and names the inspector class explicitly, instead
|
||||
of grepping for a word that appears in unrelated prose.
|
||||
|
||||
Net: MemHub can query and settle an uncertain request without blind replay.
|
||||
`unknown` remains a legitimate operational state while the SDK claim itself is
|
||||
non-terminal, but the same request id cannot write twice.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import inspect
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class HyIdempotencySupport(StrEnum):
|
||||
"""Whether the SDK can dedupe a replayed write."""
|
||||
|
||||
#: A caller key is honoured and a replay is rejected or folded together.
|
||||
KEY = "key"
|
||||
#: Same content is folded by the engine, without an explicit key.
|
||||
CONTENT_HASH = "content_hash"
|
||||
#: A replay writes a second time. Retrying after a crash duplicates.
|
||||
NONE = "none"
|
||||
#: Never probed. Must not survive A0-03.
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class HyResultLookupSupport(StrEnum):
|
||||
"""Whether the outcome of a past write can be queried afterwards."""
|
||||
|
||||
BY_REQUEST_ID = "by_request_id"
|
||||
BY_MEMORY_ID_ONLY = "by_memory_id_only"
|
||||
NONE = "none"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
def _module(name: str) -> Any | None:
|
||||
try:
|
||||
return importlib.import_module(name)
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
def _has_callable(obj: Any, name: str) -> bool:
|
||||
return obj is not None and callable(getattr(obj, name, None))
|
||||
|
||||
|
||||
def hy_write_accepts_request_id() -> bool:
|
||||
"""True when the write path carries a caller-supplied request id."""
|
||||
client_mod = _module("hy_memory.client")
|
||||
if client_mod is None:
|
||||
return False
|
||||
client = getattr(client_mod, "HyMemoryClient", None)
|
||||
if client is None:
|
||||
return False
|
||||
add = getattr(client, "add", None)
|
||||
if add is None:
|
||||
return False
|
||||
return "request_id" in inspect.signature(add).parameters
|
||||
|
||||
|
||||
def _add_path_declares_idempotency() -> bool:
|
||||
"""Whether the *write* path de-duplicates, judged on its own source.
|
||||
|
||||
Deliberately scoped to `add`/`async_add`. A whole-module search matches
|
||||
unrelated prose: "calls are idempotent" describes episode-node upserts, and
|
||||
a "dedup" comment describes search over-sampling. Both are about other
|
||||
features and would make this probe answer `key` for the wrong reason.
|
||||
"""
|
||||
client_mod = _module("hy_memory.client")
|
||||
if client_mod is None:
|
||||
return False
|
||||
client = getattr(client_mod, "HyMemoryClient", None)
|
||||
if client is None:
|
||||
return False
|
||||
sources: list[str] = []
|
||||
for name in ("add", "async_add"):
|
||||
func = getattr(client, name, None)
|
||||
if func is None:
|
||||
continue
|
||||
try:
|
||||
sources.append(inspect.getsource(func).lower())
|
||||
except (OSError, TypeError):
|
||||
continue
|
||||
if not sources:
|
||||
return False
|
||||
if _has_callable(client, "get_write_outcome"):
|
||||
claim = getattr(client, "_claim_write_outcome", None)
|
||||
if claim is not None:
|
||||
return True
|
||||
# Older SDKs may expose an explicit key under one of these names.
|
||||
markers = ("idempotency_key", "idempotent_key", "dedupe_key", "dedup_key")
|
||||
return any(marker in source for source in sources for marker in markers)
|
||||
|
||||
|
||||
def _inspector_can_lookup_by_request_id() -> bool:
|
||||
"""True when pipeline logs can be queried by the write's request id."""
|
||||
inspector = _module("hy_memory.inspector")
|
||||
if inspector is None:
|
||||
return False
|
||||
for cls_name in ("MemoryInspector", "Inspector", "PipelineInspector"):
|
||||
cls = getattr(inspector, cls_name, None)
|
||||
if cls is None:
|
||||
continue
|
||||
for method in ("get_pipeline_logs", "async_get_pipeline_logs"):
|
||||
func = getattr(cls, method, None)
|
||||
if func is None:
|
||||
continue
|
||||
# The parameter is `request_id` on `MemoryInspector`; older
|
||||
# docstrings call the same thing `n`. Accept either so a rename
|
||||
# does not silently downgrade this to "no lookup available".
|
||||
if {"request_id", "n"} & set(inspect.signature(func).parameters):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _probe_idempotency() -> HyIdempotencySupport:
|
||||
if not hy_write_accepts_request_id():
|
||||
# No handle at all: not even a correlation id to investigate later.
|
||||
return HyIdempotencySupport.NONE
|
||||
if _add_path_declares_idempotency():
|
||||
return HyIdempotencySupport.KEY
|
||||
# A request_id is present but nothing de-duplicates on it.
|
||||
return HyIdempotencySupport.NONE
|
||||
|
||||
|
||||
def _probe_result_lookup() -> HyResultLookupSupport:
|
||||
client_mod = _module("hy_memory.client")
|
||||
client = getattr(client_mod, "HyMemoryClient", None) if client_mod is not None else None
|
||||
if _has_callable(client, "get_write_outcome"):
|
||||
return HyResultLookupSupport.BY_REQUEST_ID
|
||||
if _inspector_can_lookup_by_request_id():
|
||||
return HyResultLookupSupport.BY_REQUEST_ID
|
||||
if _has_callable(client, "get"):
|
||||
return HyResultLookupSupport.BY_MEMORY_ID_ONLY
|
||||
return HyResultLookupSupport.NONE
|
||||
|
||||
|
||||
IDEMPOTENCY_SUPPORT: HyIdempotencySupport = _probe_idempotency()
|
||||
RESULT_LOOKUP_SUPPORT: HyResultLookupSupport = _probe_result_lookup()
|
||||
|
||||
|
||||
def probe_hy_write_idempotency() -> dict[str, Any]:
|
||||
"""Full probe report, for STATUS evidence and for debugging a bumped SDK.
|
||||
|
||||
Re-probed on every call so a dependency upgrade that changes the answer is
|
||||
visible, rather than being frozen at import time.
|
||||
"""
|
||||
client_mod = _module("hy_memory.client")
|
||||
client = getattr(client_mod, "HyMemoryClient", None)
|
||||
add = getattr(client, "add", None)
|
||||
return {
|
||||
"sdk_importable": client_mod is not None,
|
||||
"add_signature": str(inspect.signature(add)) if add is not None else "",
|
||||
"accepts_request_id": hy_write_accepts_request_id(),
|
||||
"add_declares_idempotency": _add_path_declares_idempotency(),
|
||||
"durable_write_outcome": (
|
||||
client is not None and _has_callable(client, "get_write_outcome")
|
||||
),
|
||||
"inspector_lookup_by_request_id": _inspector_can_lookup_by_request_id(),
|
||||
"idempotency": _probe_idempotency().value,
|
||||
"result_lookup": _probe_result_lookup().value,
|
||||
}
|
||||
@@ -9,8 +9,9 @@ Evidence→queue crash window stayed open.
|
||||
This adapter is deliberately small. It turns one committed Evidence record
|
||||
into the one-record complete-turn request, submits it once under an evidence
|
||||
derived idempotency key, and returns the durable record. The existing
|
||||
``DurableWriteDrain`` remains the only consumer; it calls the SDK write
|
||||
coordinator once and persists only the public receipt ids.
|
||||
``DurableWriteDrain`` remains the only consumer; it calls
|
||||
``process_records``, which preserves the complete-turn/debounce/settle rules
|
||||
in ``MemoryProcessingSupervisor``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -477,7 +477,7 @@ class MemoryTaskStore:
|
||||
#: Statuses whose input snapshot nothing reads back afterwards.
|
||||
#:
|
||||
#: Deliberately narrow. A `completed` task's inputs are read after the fact:
|
||||
#: SDK write processing compares them against the evidence actually sent to
|
||||
#: `memory_processing` compares them against the evidence actually sent to
|
||||
#: HY to prove coverage, and `matters/discovery` reads them to report how
|
||||
#: many records HY really saw. Clearing those would silently turn a
|
||||
#: verification into a pass and a report into a lie, which is far worse
|
||||
|
||||
@@ -10,6 +10,7 @@ from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import TypedDict
|
||||
from uuid import uuid4
|
||||
@@ -45,7 +46,14 @@ def audit_precipitation_request(request_id: str) -> Iterator[None]:
|
||||
|
||||
|
||||
def _current_prompt_stage() -> str:
|
||||
return "other"
|
||||
try:
|
||||
resolver = getattr( # noqa: B009 - optional SDK version boundary
|
||||
import_module("hy_memory.utils.model_call_context"),
|
||||
"current_model_call_stage",
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
return "other"
|
||||
return str(resolver())
|
||||
|
||||
|
||||
class ModelRequestAuditStore:
|
||||
|
||||
@@ -12,7 +12,7 @@ from typing import Any, Literal, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.hy_sdk_types import HySdkMemoryRecord
|
||||
from one_memory.hy_memory_bridge import HySdkMemoryRecord
|
||||
from one_memory.memory_governance import GovernedMemory, MemoryStatus
|
||||
from one_memory.semantic_deadline import (
|
||||
SemanticMemoryHit,
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from one_memory.conversation_processing import _is_hy_input_turn
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
|
||||
from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
from one_memory.memory_processing import _is_hy_input_turn
|
||||
from one_memory.retrieval.chunking import TextChunk
|
||||
from one_memory.retrieval.mapped_content import PreparedMappedContent, prepare_mapped_content
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import json
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Literal
|
||||
|
||||
from one_memory.conversation_processing import _is_hy_input_turn
|
||||
from one_memory.embedding_assets import BUILTIN_EMBEDDING_MAX_TOKENS
|
||||
from one_memory.imports.models import (
|
||||
MAX_VERBATIM_CHARACTERS,
|
||||
@@ -18,6 +17,7 @@ from one_memory.imports.sources.parsing import (
|
||||
)
|
||||
from one_memory.input_sanitization import sanitize_model_input_with_spans
|
||||
from one_memory.local_embedding import TokenizedText
|
||||
from one_memory.memory_processing import _is_hy_input_turn
|
||||
from one_memory.retrieval.chunking import ChunkingError, OffsetTokenizer, TextChunk, chunk_evidence
|
||||
|
||||
MAP_VERSION = "raw-content-map-v1"
|
||||
|
||||
@@ -6,15 +6,13 @@ import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from contextlib import aclosing
|
||||
from typing import Any, Literal, Protocol
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from one_memory.http_client import create_async_client
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_READER
|
||||
from one_memory.hy_sdk_runtime import sdk_relevance_floor
|
||||
from one_memory.hy_sdk_types import HySdkMemoryRecord, parse_sdk_search
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridge, HySdkMemoryRecord
|
||||
from one_memory.memory_qa import (
|
||||
MemoryAnswerError,
|
||||
_DeferredSecretStore,
|
||||
@@ -38,10 +36,6 @@ _GROUNDING_LIMIT = 6
|
||||
type _Recheck = Callable[[], Awaitable[None]]
|
||||
|
||||
|
||||
class SdkClientHandleLike(Protocol):
|
||||
async def client(self) -> Any: ...
|
||||
|
||||
|
||||
class SdkAskMemory(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
@@ -75,7 +69,7 @@ class SdkMemoryAnswerer:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client_handle: SdkClientHandleLike,
|
||||
bridge: HyMemoryBridge,
|
||||
profiles: ModelProfileStore,
|
||||
secrets: SecretStore,
|
||||
authorize_generation: Callable[
|
||||
@@ -84,7 +78,7 @@ class SdkMemoryAnswerer:
|
||||
| None = None,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
self.client_handle = client_handle
|
||||
self.bridge = bridge
|
||||
self.profiles = profiles
|
||||
self.secrets = secrets
|
||||
self.authorize_generation = authorize_generation
|
||||
@@ -96,27 +90,18 @@ class SdkMemoryAnswerer:
|
||||
*,
|
||||
user_id: str = "local",
|
||||
) -> tuple[SdkAskMemory, ...]:
|
||||
client = await self.client_handle.client()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
hits = await self.bridge.search_memories(
|
||||
question,
|
||||
user_ids=[user_id],
|
||||
agent_ids=[],
|
||||
user_id=user_id,
|
||||
agent_id="*",
|
||||
limit=12,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=12,
|
||||
intention_limit=12,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
)
|
||||
hits = parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
found: list[SdkAskMemory] = []
|
||||
seen: set[str] = set()
|
||||
for hit in hits:
|
||||
if hit.memory_id in seen:
|
||||
continue
|
||||
record = await _get_memory(client, hit.memory_id)
|
||||
record = await self.bridge.get_sdk_memory(hit.memory_id)
|
||||
if record is None or record.status != "active" or not record.is_latest:
|
||||
continue
|
||||
seen.add(record.memory_id)
|
||||
@@ -226,9 +211,8 @@ class SdkMemoryAnswerer:
|
||||
self,
|
||||
memories: tuple[SdkAskMemory, ...],
|
||||
) -> None:
|
||||
client = await self.client_handle.client()
|
||||
for memory in memories:
|
||||
current = await _get_memory(client, memory.memory_id)
|
||||
current = await self.bridge.get_sdk_memory(memory.memory_id)
|
||||
if current is None or _wire_memory(current) != memory:
|
||||
raise MemoryAnswerError("memory_answer_context_changed")
|
||||
|
||||
@@ -356,11 +340,6 @@ def _wire_memory(record: HySdkMemoryRecord) -> SdkAskMemory:
|
||||
)
|
||||
|
||||
|
||||
async def _get_memory(client: Any, memory_id: str) -> HySdkMemoryRecord | None:
|
||||
raw = await asyncio.to_thread(client.get, memory_id)
|
||||
return None if raw is None else HySdkMemoryRecord.from_sdk(raw)
|
||||
|
||||
|
||||
def _summary(memories: tuple[SdkAskMemory, ...]) -> str:
|
||||
statements = [
|
||||
memory.content.replace("用户", "你", 1).replace("The user", "You", 1) for memory in memories
|
||||
|
||||
@@ -1,381 +0,0 @@
|
||||
"""Narrow SDK-native runtime for graph, retrieval, and queue status operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.extraction_settings import ExtractionSettings, ExtractionSettingsStore
|
||||
from one_memory.hy_graph_admin import (
|
||||
GraphWidthDecision,
|
||||
GraphWidthMismatch,
|
||||
GraphWidthOutcome,
|
||||
decide_profile_graph_width,
|
||||
inspect_profile_graph_width,
|
||||
)
|
||||
from one_memory.hy_graph_projection import HyGraphSnapshot, read_graph_snapshot
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_READER
|
||||
from one_memory.hy_sdk_runtime import SdkClientHandle, sdk_relevance_floor
|
||||
from one_memory.hy_sdk_types import HySdkError, parse_sdk_search
|
||||
from one_memory.memory_queue import ExtractionWindowPreset, QueueDepths
|
||||
from one_memory.memory_server.durable_queue import COMPLETED, DurableWriteQueue
|
||||
from one_memory.sdk_write_processor import SdkWriteCoordinator
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
RAW_RETENTION_DAYS = 3
|
||||
RAW_RETENTION_CHECK_SECONDS = 6 * 60 * 60
|
||||
_RAW_STATUSES = (
|
||||
"active",
|
||||
"superseded",
|
||||
"shadow",
|
||||
"negated",
|
||||
"conflicted",
|
||||
"archived",
|
||||
)
|
||||
|
||||
|
||||
class SdkMemoryRuntime:
|
||||
"""Product services that remain after legacy task/governance writes retire."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
coordinator: SdkWriteCoordinator,
|
||||
read_handle: SdkClientHandle,
|
||||
write_queue: DurableWriteQueue,
|
||||
extraction_settings: ExtractionSettingsStore,
|
||||
data_dir: Path,
|
||||
) -> None:
|
||||
self.coordinator = coordinator
|
||||
self.read_handle = read_handle
|
||||
self.write_queue = write_queue
|
||||
self.extraction_settings = extraction_settings
|
||||
self.data_dir = data_dir
|
||||
self._raw_retention_task: asyncio.Task[None] | None = None
|
||||
|
||||
def start_raw_retention(self) -> None:
|
||||
if self._raw_retention_task is None or self._raw_retention_task.done():
|
||||
self._raw_retention_task = asyncio.create_task(
|
||||
self._raw_retention_loop(),
|
||||
name="one-memory-sdk-raw-retention",
|
||||
)
|
||||
|
||||
async def prune_expired_raw(self, *, now: datetime | None = None) -> int:
|
||||
cutoff = (now or datetime.now(UTC)) - timedelta(days=RAW_RETENTION_DAYS)
|
||||
cutoff_timestamp = int(cutoff.timestamp())
|
||||
client = await self.read_handle.client()
|
||||
deleted = 0
|
||||
while True:
|
||||
page = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
user_id="local",
|
||||
layers=["l1_raw"],
|
||||
statuses=list(_RAW_STATUSES),
|
||||
order="asc",
|
||||
limit=500,
|
||||
offset=0,
|
||||
)
|
||||
bucket = page.get("vdb") if isinstance(page, dict) else None
|
||||
rows = bucket.get("memories") if isinstance(bucket, dict) else None
|
||||
if not isinstance(rows, list) or not rows:
|
||||
return deleted
|
||||
expired = [
|
||||
row
|
||||
for row in rows
|
||||
if isinstance(row, dict)
|
||||
and isinstance(row.get("memory_id"), str)
|
||||
and int(row.get("memory_at") or row.get("gmt_created") or 0)
|
||||
< cutoff_timestamp
|
||||
]
|
||||
if not expired:
|
||||
return deleted
|
||||
removed_page = 0
|
||||
for row in expired:
|
||||
result = await asyncio.to_thread(client.delete, row["memory_id"])
|
||||
if isinstance(result, dict) and result.get("success") is True:
|
||||
count = int(result.get("deleted_count") or 0)
|
||||
deleted += count
|
||||
removed_page += count
|
||||
if removed_page == 0:
|
||||
return deleted
|
||||
|
||||
async def _raw_retention_loop(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(RAW_RETENTION_CHECK_SECONDS)
|
||||
try:
|
||||
removed = await self.prune_expired_raw()
|
||||
if removed:
|
||||
logger.info("sdk_raw_retention_deleted count=%d", removed)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as error:
|
||||
logger.warning(
|
||||
"sdk_raw_retention_failed error_type=%s",
|
||||
type(error).__name__,
|
||||
)
|
||||
|
||||
async def semantic_recall(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 12,
|
||||
) -> tuple[SemanticMemoryHit, ...]:
|
||||
client = await self.read_handle.client()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
query,
|
||||
user_ids=["local"],
|
||||
agent_ids=[],
|
||||
limit=limit,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=limit,
|
||||
intention_limit=limit,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
)
|
||||
return parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
|
||||
async def semantic_search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 12,
|
||||
) -> tuple[tuple[str, float], ...]:
|
||||
return tuple(
|
||||
(hit.memory_id, hit.score)
|
||||
for hit in await self.semantic_recall(query, limit=limit)
|
||||
)
|
||||
|
||||
async def graph_snapshot(self, *, limit: int = 300) -> HyGraphSnapshot:
|
||||
try:
|
||||
await self.coordinator.active_profile_credentials()
|
||||
except RuntimeError as error:
|
||||
if str(error) == "model_profile_not_active":
|
||||
return HyGraphSnapshot(consulted=False)
|
||||
raise
|
||||
try:
|
||||
client = await self.coordinator.client()
|
||||
except Exception as error:
|
||||
raise HySdkError("hy_memory_graph_read_failed") from error
|
||||
return await asyncio.to_thread(
|
||||
read_graph_snapshot,
|
||||
client,
|
||||
"local",
|
||||
"*",
|
||||
limit,
|
||||
)
|
||||
|
||||
async def digest(self) -> dict[str, object]:
|
||||
client = await self.coordinator.client()
|
||||
try:
|
||||
raw = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
user_id="local",
|
||||
limit=1_000,
|
||||
offset=0,
|
||||
)
|
||||
agent_ids = _agent_ids(raw)
|
||||
digests = [
|
||||
{
|
||||
"agent_id": agent_id,
|
||||
"schemas_recorded": 0,
|
||||
**dict(
|
||||
await asyncio.to_thread(
|
||||
client.evolve,
|
||||
user_id="local",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
),
|
||||
}
|
||||
for agent_id in agent_ids
|
||||
]
|
||||
except Exception as error:
|
||||
raise HySdkError("hy_memory_digest_failed") from error
|
||||
return {
|
||||
"success": all(result.get("success") is not False for result in digests),
|
||||
"scope_count": len(digests),
|
||||
"skipped_conversation_scopes": 0,
|
||||
"schemas_recorded": 0,
|
||||
"digests": digests,
|
||||
}
|
||||
|
||||
async def graph_width_mismatch(self) -> GraphWidthMismatch | None:
|
||||
try:
|
||||
profile, secret = await self.coordinator.active_profile_credentials()
|
||||
except RuntimeError as error:
|
||||
if str(error) == "model_profile_not_active":
|
||||
return None
|
||||
raise
|
||||
return await asyncio.to_thread(
|
||||
inspect_profile_graph_width,
|
||||
profile,
|
||||
api_key=secret,
|
||||
data_dir=self.data_dir,
|
||||
)
|
||||
|
||||
async def decide_graph_width(
|
||||
self,
|
||||
decision: GraphWidthDecision,
|
||||
) -> GraphWidthOutcome:
|
||||
try:
|
||||
profile, secret = await self.coordinator.active_profile_credentials()
|
||||
except RuntimeError as error:
|
||||
raise HySdkError(str(error)) from error
|
||||
return await asyncio.to_thread(
|
||||
decide_profile_graph_width,
|
||||
profile,
|
||||
api_key=secret,
|
||||
data_dir=self.data_dir,
|
||||
decision=decision,
|
||||
)
|
||||
|
||||
async def update_hy_memory(self, memory_id: str, content: str) -> dict[str, object]:
|
||||
client = await self.coordinator.client()
|
||||
try:
|
||||
result = await asyncio.to_thread(client.update, memory_id, content)
|
||||
except Exception as error:
|
||||
raise HySdkError("hy_memory_update_failed") from error
|
||||
if not isinstance(result, dict) or result.get("success") is not True:
|
||||
raise HySdkError("hy_memory_update_rejected")
|
||||
return result
|
||||
|
||||
async def finish_deletion(self, memory_id: str) -> None:
|
||||
client = await self.coordinator.client()
|
||||
try:
|
||||
result = await asyncio.to_thread(client.delete, memory_id)
|
||||
except Exception as error:
|
||||
raise HySdkError("hy_memory_delete_failed") from error
|
||||
if not isinstance(result, dict):
|
||||
raise HySdkError("hy_memory_delete_rejected")
|
||||
error_code = result.get("error_code")
|
||||
if error_code or result.get("partial") is True:
|
||||
raise HySdkError(str(error_code or "hy_memory_delete_incomplete"))
|
||||
if result.get("success") is not True and result.get("deleted_count") != 0:
|
||||
raise HySdkError("hy_memory_delete_rejected")
|
||||
|
||||
def extraction_window_settings(self) -> ExtractionSettings:
|
||||
return self.extraction_settings.get()
|
||||
|
||||
def set_extraction_window(
|
||||
self,
|
||||
preset: ExtractionWindowPreset,
|
||||
) -> ExtractionSettings:
|
||||
return self.extraction_settings.set_window_preset(preset)
|
||||
|
||||
def queue_depths(self) -> QueueDepths:
|
||||
now = datetime.now(UTC)
|
||||
fresh_since = (now - timedelta(days=7)).isoformat()
|
||||
with main_database(self.write_queue.database_path).read() as database:
|
||||
fresh = int(
|
||||
database.execute(
|
||||
"SELECT COUNT(*) FROM memory_write_jobs "
|
||||
"WHERE state IN ('pending','waiting_for_model','waiting_for_permission') "
|
||||
"AND event_at>=?",
|
||||
(fresh_since,),
|
||||
).fetchone()[0]
|
||||
)
|
||||
backlog = int(
|
||||
database.execute(
|
||||
"SELECT COUNT(*) FROM memory_write_jobs "
|
||||
"WHERE state IN ('pending','waiting_for_model','waiting_for_permission') "
|
||||
"AND event_at<?",
|
||||
(fresh_since,),
|
||||
).fetchone()[0]
|
||||
)
|
||||
recent = tuple(
|
||||
datetime.fromisoformat(str(row[0]))
|
||||
for row in database.execute(
|
||||
"SELECT updated_at FROM memory_write_jobs WHERE state='completed' "
|
||||
"ORDER BY updated_at DESC LIMIT 8"
|
||||
)
|
||||
)
|
||||
return QueueDepths(fresh=fresh, backlog=backlog, recent_completions=recent)
|
||||
|
||||
def active_depths(self) -> tuple[int, int]:
|
||||
with main_database(self.write_queue.database_path).read() as database:
|
||||
running = int(
|
||||
database.execute(
|
||||
"SELECT COUNT(*) FROM memory_write_jobs WHERE state='running'"
|
||||
).fetchone()[0]
|
||||
)
|
||||
return running, running
|
||||
|
||||
def status_counts(self) -> dict[str, int]:
|
||||
with main_database(self.write_queue.database_path).read() as database:
|
||||
return {
|
||||
str(row[0]): int(row[1])
|
||||
for row in database.execute(
|
||||
"SELECT state,COUNT(*) FROM memory_write_jobs GROUP BY state"
|
||||
)
|
||||
}
|
||||
|
||||
def manual_batch_progress(self) -> tuple[int, int]:
|
||||
return 0, 0
|
||||
|
||||
def promote_next_manual_batch(self, *, task_ids: tuple[str, ...] | None = None) -> list[Any]:
|
||||
del task_ids
|
||||
return []
|
||||
|
||||
async def reconcile_unknown_outputs(
|
||||
self,
|
||||
*,
|
||||
limit: int = 128,
|
||||
exclude_error_codes: tuple[str, ...] = (),
|
||||
) -> int:
|
||||
reconciled = 0
|
||||
for job in await asyncio.to_thread(
|
||||
self.write_queue.unknown_jobs,
|
||||
limit=limit,
|
||||
exclude_error_codes=exclude_error_codes,
|
||||
):
|
||||
receipt = await self.coordinator.inquire_write_outcome(job.request_id)
|
||||
if receipt is None:
|
||||
continue
|
||||
await asyncio.to_thread(
|
||||
self.write_queue.settle,
|
||||
job.job_id,
|
||||
state=COMPLETED,
|
||||
sdk_raw_ids=receipt.raw_ids,
|
||||
sdk_memory_ids=receipt.memory_ids,
|
||||
)
|
||||
reconciled += 1
|
||||
return reconciled
|
||||
|
||||
async def resume_allowed(self) -> None:
|
||||
await asyncio.to_thread(self.write_queue.readmit_parked)
|
||||
|
||||
def accepted_backfill_status(self) -> None:
|
||||
return None
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._raw_retention_task is None:
|
||||
return
|
||||
self._raw_retention_task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await self._raw_retention_task
|
||||
self._raw_retention_task = None
|
||||
|
||||
|
||||
def _agent_ids(raw: object) -> tuple[str, ...]:
|
||||
if not isinstance(raw, dict):
|
||||
raise HySdkError("hy_memory_list_shape_unknown")
|
||||
found: set[str] = set()
|
||||
for bucket_name, rows_name in (("vdb", "memories"), ("graph", "nodes")):
|
||||
bucket = raw.get(bucket_name)
|
||||
if bucket is None:
|
||||
continue
|
||||
if not isinstance(bucket, dict) or not isinstance(bucket.get(rows_name), list):
|
||||
raise HySdkError("hy_memory_list_shape_unknown")
|
||||
for row in bucket[rows_name]:
|
||||
if isinstance(row, dict):
|
||||
agent_id = row.get("agent_id")
|
||||
if isinstance(agent_id, str) and agent_id:
|
||||
found.add(agent_id)
|
||||
return tuple(sorted(found))
|
||||
@@ -22,12 +22,6 @@ from one_memory.hy_runtime_config import (
|
||||
HY_MEMORY_NAMESPACE,
|
||||
HY_MEMORY_VERSION,
|
||||
)
|
||||
from one_memory.hy_sdk_migration import (
|
||||
restore_data_plane as restore_sdk_data_plane,
|
||||
)
|
||||
from one_memory.hy_sdk_migration import (
|
||||
snapshot_data_plane,
|
||||
)
|
||||
|
||||
|
||||
class MigrationBundleRejected(RuntimeError):
|
||||
@@ -119,7 +113,11 @@ def create_sdk_migration_bundle(
|
||||
if not sdk_root.is_dir():
|
||||
raise MigrationBundleRejected("sdk_migration_bundle_sdk_root_invalid")
|
||||
if snapshotter is None:
|
||||
snapshotter = snapshot_data_plane
|
||||
try:
|
||||
from one_memory.hy_memory_bridge import snapshot_hy_data_plane
|
||||
except ImportError as error:
|
||||
raise MigrationBundleRejected("sdk_migration_snapshot_unavailable") from error
|
||||
snapshotter = snapshot_hy_data_plane
|
||||
try:
|
||||
snapshotter(
|
||||
source_root=sdk_root,
|
||||
@@ -277,7 +275,11 @@ def restore_sdk_migration_bundle(
|
||||
raise MigrationBundleRejected("sdk_migration_bundle_manifest_invalid")
|
||||
|
||||
if restore_data_plane is None:
|
||||
restore_data_plane = restore_sdk_data_plane
|
||||
try:
|
||||
from one_memory.hy_memory_bridge import restore_hy_data_plane
|
||||
except ImportError as error:
|
||||
raise MigrationBundleRejected("sdk_migration_restore_unavailable") from error
|
||||
restore_data_plane = restore_hy_data_plane
|
||||
|
||||
stamp = (now or datetime.now(UTC)).strftime("%Y%m%dT%H%M%S%fZ")
|
||||
database_live = home / "one-memory.sqlite3"
|
||||
|
||||
@@ -7,13 +7,12 @@ import json
|
||||
import sqlite3
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.hy_sdk_types import HySdkMemoryPage, HySdkMemoryRecord, HySdkRawPage
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
from one_memory.hy_memory_bridge import HySdkMemoryPage, HySdkRawRecord
|
||||
from one_memory.sdk_data_plane_migration import SdkDataPlaneMigrationStore
|
||||
|
||||
|
||||
@@ -36,8 +35,6 @@ class LegacyMemoryInventory:
|
||||
tags: tuple[str, ...]
|
||||
supersedes: tuple[str, ...]
|
||||
superseded_by: tuple[str, ...]
|
||||
productivity_kind: str | None = None
|
||||
productivity_variant: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -47,8 +44,6 @@ class SdkMemoryInventory:
|
||||
agent_id: str
|
||||
content_hash: str
|
||||
status: str
|
||||
productivity_kind: str | None = None
|
||||
productivity_variant: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -97,17 +92,6 @@ _KIND_LAYER = {
|
||||
"schema": "l6_schema",
|
||||
"intention": "l7_intention",
|
||||
}
|
||||
_PRODUCTIVITY_LAYER = {
|
||||
"coding_experience": ("experience", "coding"),
|
||||
"working_experience": ("experience", "working"),
|
||||
"coding_fact": ("fact", "coding"),
|
||||
# Historical receipts called every productivity fact "working_fact",
|
||||
# including facts stored beside a coding-variant experience. Resolve the
|
||||
# actual variant from the SDK exact lookup instead of trusting this label.
|
||||
"working_fact": ("fact", None),
|
||||
}
|
||||
_RAW_TURN_MAX_CHARACTERS = 60_000
|
||||
_RAW_USER_MAX_CHARACTERS = 30_000
|
||||
|
||||
SDK_LIFECYCLE_STATUSES = (
|
||||
"active",
|
||||
@@ -119,8 +103,8 @@ SDK_LIFECYCLE_STATUSES = (
|
||||
)
|
||||
|
||||
|
||||
class SdkMigrationClient(Protocol):
|
||||
def list_memories(
|
||||
class SdkMigrationBridge(Protocol):
|
||||
def list_sdk_memories(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
@@ -131,7 +115,7 @@ class SdkMigrationClient(Protocol):
|
||||
order: str = "desc",
|
||||
) -> Awaitable[HySdkMemoryPage]: ...
|
||||
|
||||
def write_legacy_memory(
|
||||
def import_sdk_memory_exact(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
@@ -149,7 +133,7 @@ class SdkMigrationClient(Protocol):
|
||||
request_id: str,
|
||||
) -> Awaitable[dict[str, object]]: ...
|
||||
|
||||
def read_raw_inventory(
|
||||
def list_sdk_raw(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
@@ -159,9 +143,9 @@ class SdkMigrationClient(Protocol):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
order: str = "desc",
|
||||
) -> Awaitable[HySdkRawPage]: ...
|
||||
) -> Awaitable[tuple[tuple[HySdkRawRecord, ...], int]]: ...
|
||||
|
||||
def write_legacy_raw(
|
||||
def import_sdk_raw_exact(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
@@ -177,29 +161,6 @@ class SdkMigrationClient(Protocol):
|
||||
|
||||
def delete(self, memory_id: str) -> Awaitable[dict[str, object]]: ...
|
||||
|
||||
def read_productivity_memory(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: str,
|
||||
variant: str | None,
|
||||
) -> Awaitable[tuple[HySdkMemoryRecord, str] | None]: ...
|
||||
|
||||
def list_productivity_memories(
|
||||
self,
|
||||
) -> Awaitable[tuple[dict[str, object], ...]]: ...
|
||||
|
||||
def delete_productivity_memory(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
kind: str,
|
||||
variant: str | None,
|
||||
user_id: str,
|
||||
) -> Awaitable[dict[str, object]]: ...
|
||||
|
||||
def delete_raw(self, raw_id: str) -> Awaitable[dict[str, object]]: ...
|
||||
|
||||
|
||||
def read_legacy_memory_inventory(database_path: Path) -> tuple[LegacyMemoryInventory, ...]:
|
||||
"""Read legacy memory intent without constructing a Governance writer."""
|
||||
@@ -243,7 +204,6 @@ def read_legacy_memory_inventory(database_path: Path) -> tuple[LegacyMemoryInven
|
||||
tags = tuple(value for value in tags_value if isinstance(value, str))
|
||||
kind = str(row["kind"])
|
||||
status = str(row["status"])
|
||||
productivity = _PRODUCTIVITY_LAYER.get(mapped_layer or "")
|
||||
records.append(
|
||||
LegacyMemoryInventory(
|
||||
legacy_memory_id=legacy_id,
|
||||
@@ -265,8 +225,6 @@ def read_legacy_memory_inventory(database_path: Path) -> tuple[LegacyMemoryInven
|
||||
tags=tags,
|
||||
supersedes=tuple(supersedes.get(legacy_id, ())),
|
||||
superseded_by=tuple(superseded_by.get(legacy_id, ())),
|
||||
productivity_kind=productivity[0] if productivity else None,
|
||||
productivity_variant=productivity[1] if productivity else None,
|
||||
)
|
||||
)
|
||||
return tuple(records)
|
||||
@@ -281,116 +239,43 @@ def _parse_datetime(value: object) -> datetime | None:
|
||||
return None
|
||||
|
||||
|
||||
def read_legacy_raw_inventory(
|
||||
database_path: Path,
|
||||
*,
|
||||
cutoff: datetime,
|
||||
) -> tuple[LegacyRawInventory, ...]:
|
||||
"""Rebuild recent legacy messages into one user/assistant raw per turn."""
|
||||
if cutoff.tzinfo is None:
|
||||
cutoff = cutoff.replace(tzinfo=UTC)
|
||||
def read_legacy_raw_inventory(database_path: Path) -> tuple[LegacyRawInventory, ...]:
|
||||
with main_database(database_path).read(row_factory=sqlite3.Row) as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT evidence_id,user_id,source_id,workspace,project,
|
||||
conversation_id,message_id,source_ordinal,content_kind,
|
||||
role,written_by,created_at,imported_at,verbatim,content_hash,revision
|
||||
FROM evidence
|
||||
WHERE content_kind='raw_conversation'
|
||||
AND (
|
||||
(role='user' AND written_by='person')
|
||||
OR role='assistant'
|
||||
)
|
||||
ORDER BY source_id,conversation_id,workspace,
|
||||
COALESCE(created_at,imported_at),
|
||||
COALESCE(ingest_seq,rowid),evidence_id
|
||||
FROM evidence ORDER BY COALESCE(ingest_seq,rowid),evidence_id
|
||||
"""
|
||||
).fetchall()
|
||||
turns: list[LegacyRawInventory] = []
|
||||
current_key: tuple[str, str, str, str] | None = None
|
||||
current: list[sqlite3.Row] = []
|
||||
|
||||
def finish() -> None:
|
||||
if not current or str(current[0]["role"]) != "user":
|
||||
return
|
||||
assistants = [row for row in current[1:] if str(row["role"]) == "assistant"]
|
||||
if not assistants:
|
||||
return
|
||||
rows_for_turn = [current[0], *assistants]
|
||||
completed_at = _parse_datetime(
|
||||
assistants[-1]["created_at"] or assistants[-1]["imported_at"]
|
||||
return tuple(
|
||||
LegacyRawInventory(
|
||||
evidence_id=str(row["evidence_id"]),
|
||||
user_id=str(row["user_id"]),
|
||||
agent_id="one-memory",
|
||||
session_id=str(row["conversation_id"]),
|
||||
source=str(row["source_id"]),
|
||||
content=str(row["verbatim"]),
|
||||
content_hash=str(row["content_hash"]),
|
||||
memory_at=_parse_datetime(row["created_at"] or row["imported_at"]),
|
||||
metadata={
|
||||
key: row[key]
|
||||
for key in (
|
||||
"workspace",
|
||||
"project",
|
||||
"message_id",
|
||||
"source_ordinal",
|
||||
"content_kind",
|
||||
"role",
|
||||
"written_by",
|
||||
"revision",
|
||||
)
|
||||
if row[key] is not None
|
||||
},
|
||||
)
|
||||
if completed_at is not None and completed_at.tzinfo is None:
|
||||
completed_at = completed_at.replace(tzinfo=UTC)
|
||||
if completed_at is None or completed_at.astimezone(UTC) < cutoff.astimezone(UTC):
|
||||
return
|
||||
evidence_ids = [str(row["evidence_id"]) for row in rows_for_turn]
|
||||
source = str(current[0]["source_id"])
|
||||
session_id = str(current[0]["conversation_id"])
|
||||
digest = hashlib.sha256(
|
||||
"\0".join((source, session_id, *evidence_ids)).encode()
|
||||
).hexdigest()
|
||||
raw_user_text = str(rows_for_turn[0]["verbatim"] or "")
|
||||
user_text = sanitize_model_input(raw_user_text)[:_RAW_USER_MAX_CHARACTERS]
|
||||
content_parts = [f"[user]: {user_text}"]
|
||||
truncated = len(raw_user_text) > len(user_text)
|
||||
for row in assistants:
|
||||
sanitized = sanitize_model_input(str(row["verbatim"] or ""))
|
||||
text = sanitized
|
||||
prefix = "[assistant]: "
|
||||
remaining = _RAW_TURN_MAX_CHARACTERS - len("\n".join(content_parts)) - 1
|
||||
if remaining <= len(prefix):
|
||||
truncated = True
|
||||
break
|
||||
available = remaining - len(prefix)
|
||||
if len(sanitized) > available:
|
||||
text = text[:available]
|
||||
truncated = True
|
||||
content_parts.append(f"{prefix}{text}")
|
||||
if len(text) < len(sanitized):
|
||||
break
|
||||
content = "\n".join(content_parts)
|
||||
if not content:
|
||||
return
|
||||
turns.append(
|
||||
LegacyRawInventory(
|
||||
evidence_id=f"turn:{digest}",
|
||||
user_id=str(current[0]["user_id"]),
|
||||
agent_id="all-apps",
|
||||
session_id=session_id,
|
||||
source=source,
|
||||
content=content,
|
||||
content_hash=content_hash(content),
|
||||
memory_at=_parse_datetime(
|
||||
current[0]["created_at"] or current[0]["imported_at"]
|
||||
),
|
||||
metadata={
|
||||
"evidence_ids": evidence_ids,
|
||||
"workspace": current[0]["workspace"],
|
||||
"project": current[0]["project"],
|
||||
"input_kind": "legacy_turn",
|
||||
"truncated": truncated,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
for row in rows:
|
||||
key = (
|
||||
str(row["source_id"]),
|
||||
str(row["conversation_id"]),
|
||||
str(row["user_id"]),
|
||||
str(row["workspace"] or ""),
|
||||
)
|
||||
if key != current_key:
|
||||
finish()
|
||||
current_key, current = key, []
|
||||
if str(row["role"]) == "user":
|
||||
finish()
|
||||
current = [row]
|
||||
elif current:
|
||||
current.append(row)
|
||||
finish()
|
||||
return tuple(turns)
|
||||
for row in rows
|
||||
)
|
||||
|
||||
|
||||
def classify_memory_inventory(
|
||||
@@ -398,39 +283,13 @@ def classify_memory_inventory(
|
||||
sdk: tuple[SdkMemoryInventory, ...],
|
||||
) -> tuple[MemoryMigrationDecision, ...]:
|
||||
"""Classify every legacy and SDK memory; no item may silently disappear."""
|
||||
sdk_by_id: dict[str, list[SdkMemoryInventory]] = {}
|
||||
for sdk_item in sdk:
|
||||
sdk_by_id.setdefault(sdk_item.memory_id, []).append(sdk_item)
|
||||
claimed_sdk_items: set[tuple[str, str | None, str | None]] = set()
|
||||
sdk_by_id = {item.memory_id: item for item in sdk}
|
||||
claimed_sdk_ids: set[str] = set()
|
||||
decisions: list[MemoryMigrationDecision] = []
|
||||
for item in legacy:
|
||||
candidates = [
|
||||
candidate
|
||||
for candidate in sdk_by_id.get(item.canonical_memory_id, ())
|
||||
if (
|
||||
candidate.productivity_kind == item.productivity_kind
|
||||
and (
|
||||
item.productivity_kind is None
|
||||
or item.productivity_variant is None
|
||||
or candidate.productivity_variant == item.productivity_variant
|
||||
)
|
||||
)
|
||||
]
|
||||
if len(candidates) > 1:
|
||||
claimed_sdk_items.update(_sdk_inventory_key(candidate) for candidate in candidates)
|
||||
decisions.append(
|
||||
MemoryMigrationDecision(
|
||||
item_key=f"memory:{item.legacy_memory_id}",
|
||||
action="quarantine",
|
||||
legacy_memory_id=item.legacy_memory_id,
|
||||
canonical_memory_id=item.canonical_memory_id,
|
||||
reason="sdk_memory_id_ambiguous",
|
||||
)
|
||||
)
|
||||
continue
|
||||
current = candidates[0] if candidates else None
|
||||
current = sdk_by_id.get(item.canonical_memory_id)
|
||||
if current is not None:
|
||||
claimed_sdk_items.add(_sdk_inventory_key(current))
|
||||
claimed_sdk_ids.add(current.memory_id)
|
||||
if item.status == "deleted":
|
||||
decisions.append(
|
||||
MemoryMigrationDecision(
|
||||
@@ -454,17 +313,6 @@ def classify_memory_inventory(
|
||||
)
|
||||
continue
|
||||
if current is None:
|
||||
if item.productivity_kind is not None:
|
||||
decisions.append(
|
||||
MemoryMigrationDecision(
|
||||
item_key=f"memory:{item.legacy_memory_id}",
|
||||
action="quarantine",
|
||||
legacy_memory_id=item.legacy_memory_id,
|
||||
canonical_memory_id=item.canonical_memory_id,
|
||||
reason="mapped_productivity_memory_missing",
|
||||
)
|
||||
)
|
||||
continue
|
||||
decisions.append(
|
||||
MemoryMigrationDecision(
|
||||
item_key=f"memory:{item.legacy_memory_id}",
|
||||
@@ -475,19 +323,7 @@ def classify_memory_inventory(
|
||||
)
|
||||
)
|
||||
continue
|
||||
if item.productivity_kind is not None and (
|
||||
current.user_id == item.user_id
|
||||
and current.productivity_kind == item.productivity_kind
|
||||
and (
|
||||
item.productivity_variant is None
|
||||
or current.productivity_variant == item.productivity_variant
|
||||
)
|
||||
):
|
||||
action = (
|
||||
"matched" if item.legacy_memory_id == item.canonical_memory_id else "record_alias"
|
||||
)
|
||||
reason = "existing_productivity_identity"
|
||||
elif (
|
||||
if (
|
||||
current.content_hash == item.content_hash
|
||||
and current.status == item.status
|
||||
and current.user_id == item.user_id
|
||||
@@ -510,10 +346,10 @@ def classify_memory_inventory(
|
||||
)
|
||||
|
||||
for sdk_item in sdk:
|
||||
if _sdk_inventory_key(sdk_item) not in claimed_sdk_items:
|
||||
if sdk_item.memory_id not in claimed_sdk_ids:
|
||||
decisions.append(
|
||||
MemoryMigrationDecision(
|
||||
item_key=_sdk_item_key(sdk_item),
|
||||
item_key=f"sdk:{sdk_item.memory_id}",
|
||||
action="adopt",
|
||||
legacy_memory_id=None,
|
||||
canonical_memory_id=sdk_item.memory_id,
|
||||
@@ -523,36 +359,16 @@ def classify_memory_inventory(
|
||||
return tuple(decisions)
|
||||
|
||||
|
||||
def _sdk_inventory_key(
|
||||
item: SdkMemoryInventory,
|
||||
) -> tuple[str, str | None, str | None]:
|
||||
return (
|
||||
item.memory_id,
|
||||
item.productivity_kind,
|
||||
item.productivity_variant,
|
||||
)
|
||||
|
||||
|
||||
def _sdk_item_key(item: SdkMemoryInventory) -> str:
|
||||
namespace = (
|
||||
f"{item.productivity_variant}:{item.productivity_kind}"
|
||||
if item.productivity_kind is not None
|
||||
else "chat"
|
||||
)
|
||||
return f"sdk:{namespace}:{item.memory_id}"
|
||||
|
||||
|
||||
async def read_sdk_memory_inventory(
|
||||
client: SdkMigrationClient,
|
||||
bridge: SdkMigrationBridge,
|
||||
*,
|
||||
legacy: tuple[LegacyMemoryInventory, ...] = (),
|
||||
user_id: str = "local",
|
||||
page_size: int = 500,
|
||||
) -> tuple[SdkMemoryInventory, ...]:
|
||||
records: list[SdkMemoryInventory] = []
|
||||
records: dict[str, SdkMemoryInventory] = {}
|
||||
offset = 0
|
||||
while True:
|
||||
page = await client.list_memories(
|
||||
page = await bridge.list_sdk_memories(
|
||||
user_id=user_id,
|
||||
statuses=SDK_LIFECYCLE_STATUSES,
|
||||
limit=page_size,
|
||||
@@ -560,45 +376,18 @@ async def read_sdk_memory_inventory(
|
||||
order="asc",
|
||||
)
|
||||
for memory in page.memories:
|
||||
records.append(
|
||||
SdkMemoryInventory(
|
||||
memory_id=memory.memory_id,
|
||||
user_id=memory.user_id,
|
||||
agent_id=memory.agent_id,
|
||||
content_hash=content_hash(memory.content),
|
||||
status=memory.status,
|
||||
)
|
||||
records[memory.memory_id] = SdkMemoryInventory(
|
||||
memory_id=memory.memory_id,
|
||||
user_id=memory.user_id,
|
||||
agent_id=memory.agent_id,
|
||||
content_hash=content_hash(memory.content),
|
||||
status=memory.status,
|
||||
)
|
||||
consumed = len(page.memories)
|
||||
offset += consumed
|
||||
if consumed == 0 or offset >= page.total:
|
||||
break
|
||||
del legacy
|
||||
for memory in await client.list_productivity_memories():
|
||||
try:
|
||||
records.append(
|
||||
SdkMemoryInventory(
|
||||
memory_id=str(memory["memory_id"]),
|
||||
user_id=str(memory["user_id"]),
|
||||
agent_id=str(memory["agent_id"]),
|
||||
content_hash=content_hash(str(memory["content"])),
|
||||
status=str(memory["status"]),
|
||||
productivity_kind=str(memory["kind"]),
|
||||
productivity_variant=str(memory["variant"]),
|
||||
)
|
||||
)
|
||||
except KeyError as error:
|
||||
raise RuntimeError("invalid productivity inventory row") from error
|
||||
return tuple(
|
||||
sorted(
|
||||
records,
|
||||
key=lambda item: (
|
||||
item.memory_id,
|
||||
item.productivity_kind or "",
|
||||
item.productivity_variant or "",
|
||||
),
|
||||
)
|
||||
)
|
||||
return tuple(records[key] for key in sorted(records))
|
||||
|
||||
|
||||
def record_memory_migration_plan(
|
||||
@@ -610,12 +399,12 @@ def record_memory_migration_plan(
|
||||
) -> tuple[MemoryMigrationDecision, ...]:
|
||||
decisions = classify_memory_inventory(legacy, sdk)
|
||||
legacy_by_id = {item.legacy_memory_id: item for item in legacy}
|
||||
sdk_by_item_key = {_sdk_item_key(item): item for item in sdk}
|
||||
sdk_by_id = {item.memory_id: item for item in sdk}
|
||||
for decision in decisions:
|
||||
source_hash = (
|
||||
legacy_by_id[decision.legacy_memory_id].content_hash
|
||||
if decision.legacy_memory_id is not None
|
||||
else sdk_by_item_key[decision.item_key].content_hash
|
||||
else sdk_by_id[decision.canonical_memory_id].content_hash
|
||||
)
|
||||
operation_key = hashlib.sha256(
|
||||
f"{migration_id}:{decision.item_key}:{decision.action}:{source_hash}".encode()
|
||||
@@ -636,7 +425,7 @@ def record_memory_migration_plan(
|
||||
|
||||
|
||||
async def execute_memory_migration_plan(
|
||||
client: SdkMigrationClient,
|
||||
bridge: SdkMigrationBridge,
|
||||
store: SdkDataPlaneMigrationStore,
|
||||
*,
|
||||
migration_id: str,
|
||||
@@ -668,15 +457,7 @@ async def execute_memory_migration_plan(
|
||||
continue
|
||||
if decision.action in {"delete", "verified_deleted"}:
|
||||
if decision.action == "delete":
|
||||
if item is not None and item.productivity_kind is not None:
|
||||
await client.delete_productivity_memory(
|
||||
decision.canonical_memory_id,
|
||||
kind=item.productivity_kind,
|
||||
variant=item.productivity_variant,
|
||||
user_id=item.user_id,
|
||||
)
|
||||
else:
|
||||
await client.delete(decision.canonical_memory_id)
|
||||
await bridge.delete(decision.canonical_memory_id)
|
||||
store.settle_item(
|
||||
migration_id=migration_id,
|
||||
item_key=decision.item_key,
|
||||
@@ -691,13 +472,12 @@ async def execute_memory_migration_plan(
|
||||
request_id = hashlib.sha256(
|
||||
f"{migration_id}:{decision.item_key}:import_exact".encode()
|
||||
).hexdigest()
|
||||
await client.write_legacy_memory(
|
||||
await bridge.import_sdk_memory_exact(
|
||||
content=item.content,
|
||||
memory_id=item.canonical_memory_id,
|
||||
user_id=item.user_id,
|
||||
agent_id=item.agent_id,
|
||||
session_id=item.session_id,
|
||||
source="legacy-governance",
|
||||
layer=item.layer,
|
||||
status=item.status,
|
||||
owner=item.owner,
|
||||
@@ -705,8 +485,6 @@ async def execute_memory_migration_plan(
|
||||
supersedes=item.supersedes,
|
||||
superseded_by=item.superseded_by,
|
||||
is_latest=item.status == "active",
|
||||
memory_at=None,
|
||||
metadata={},
|
||||
request_id=request_id,
|
||||
)
|
||||
if decision.action == "record_alias":
|
||||
@@ -725,7 +503,7 @@ async def execute_memory_migration_plan(
|
||||
|
||||
|
||||
async def read_sdk_raw_inventory(
|
||||
client: SdkMigrationClient,
|
||||
bridge: SdkMigrationBridge,
|
||||
*,
|
||||
user_id: str = "local",
|
||||
page_size: int = 500,
|
||||
@@ -733,13 +511,13 @@ async def read_sdk_raw_inventory(
|
||||
records: dict[str, SdkRawInventory] = {}
|
||||
offset = 0
|
||||
while True:
|
||||
page = await client.read_raw_inventory(
|
||||
rows, total = await bridge.list_sdk_raw(
|
||||
user_id=user_id,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
order="asc",
|
||||
)
|
||||
for raw in page.raw:
|
||||
for raw in rows:
|
||||
raw_id = str(raw.raw_id)
|
||||
records[raw_id] = SdkRawInventory(
|
||||
raw_id=raw_id,
|
||||
@@ -749,9 +527,9 @@ async def read_sdk_raw_inventory(
|
||||
source=str(raw.source),
|
||||
content_hash=content_hash(str(raw.content)),
|
||||
)
|
||||
consumed = len(page.raw)
|
||||
consumed = len(rows)
|
||||
offset += consumed
|
||||
if consumed == 0 or offset >= page.total:
|
||||
if consumed == 0 or offset >= total:
|
||||
break
|
||||
return tuple(records[key] for key in sorted(records))
|
||||
|
||||
@@ -814,10 +592,10 @@ def classify_raw_inventory(
|
||||
decisions.append(
|
||||
RawMigrationDecision(
|
||||
item_key=f"sdk-raw:{raw.raw_id}",
|
||||
action="delete",
|
||||
action="adopt",
|
||||
evidence_id=None,
|
||||
raw_id=raw.raw_id,
|
||||
reason="outside_canonical_three_day_turns",
|
||||
reason="sdk_only",
|
||||
)
|
||||
)
|
||||
return tuple(decisions)
|
||||
@@ -854,7 +632,7 @@ def record_raw_migration_plan(
|
||||
|
||||
|
||||
async def execute_raw_migration_plan(
|
||||
client: SdkMigrationClient,
|
||||
bridge: SdkMigrationBridge,
|
||||
store: SdkDataPlaneMigrationStore,
|
||||
*,
|
||||
migration_id: str,
|
||||
@@ -863,12 +641,11 @@ async def execute_raw_migration_plan(
|
||||
) -> None:
|
||||
legacy_by_id = {item.evidence_id: item for item in legacy}
|
||||
for decision in decisions:
|
||||
if decision.action == "delete":
|
||||
await client.delete_raw(decision.raw_id)
|
||||
if decision.action == "adopt":
|
||||
store.settle_item(
|
||||
migration_id=migration_id,
|
||||
item_key=decision.item_key,
|
||||
state="deleted",
|
||||
state="adopted",
|
||||
target_memory_id=decision.raw_id,
|
||||
)
|
||||
continue
|
||||
@@ -877,9 +654,9 @@ async def execute_raw_migration_plan(
|
||||
raise RuntimeError(f"missing legacy raw inventory for {decision.item_key}")
|
||||
if decision.action == "import_exact":
|
||||
request_id = hashlib.sha256(
|
||||
f"{migration_id}:{decision.item_key}:write_legacy_raw".encode()
|
||||
f"{migration_id}:{decision.item_key}:import_raw".encode()
|
||||
).hexdigest()
|
||||
await client.write_legacy_raw(
|
||||
await bridge.import_sdk_raw_exact(
|
||||
content=item.content,
|
||||
raw_id=decision.raw_id,
|
||||
user_id=item.user_id,
|
||||
@@ -897,15 +674,8 @@ async def execute_raw_migration_plan(
|
||||
target_memory_id=decision.raw_id,
|
||||
target_hash=item.content_hash,
|
||||
)
|
||||
evidence_ids = item.metadata.get("evidence_ids")
|
||||
for evidence_id in (
|
||||
evidence_ids
|
||||
if isinstance(evidence_ids, list)
|
||||
else [item.evidence_id]
|
||||
):
|
||||
if isinstance(evidence_id, str) and evidence_id:
|
||||
store.record_raw_alias(
|
||||
migration_id=migration_id,
|
||||
legacy_evidence_id=evidence_id,
|
||||
canonical_raw_id=decision.raw_id,
|
||||
)
|
||||
store.record_raw_alias(
|
||||
migration_id=migration_id,
|
||||
legacy_evidence_id=item.evidence_id,
|
||||
canonical_raw_id=decision.raw_id,
|
||||
)
|
||||
@@ -4,12 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
@@ -28,7 +24,7 @@ from one_memory.sdk_data_plane_migration import (
|
||||
)
|
||||
from one_memory.sdk_migration_bundle import inspect_sdk_migration_bundle
|
||||
from one_memory.sdk_migration_inventory import (
|
||||
SdkMigrationClient,
|
||||
SdkMigrationBridge,
|
||||
execute_memory_migration_plan,
|
||||
execute_raw_migration_plan,
|
||||
read_legacy_memory_inventory,
|
||||
@@ -51,139 +47,6 @@ class BackfillResult:
|
||||
raw_items: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SdkMigrationStaging:
|
||||
migration_id: str
|
||||
staging_home: Path
|
||||
live_root: Path
|
||||
staging_root: Path
|
||||
protective_root: Path
|
||||
state_file: Path
|
||||
|
||||
|
||||
def sdk_migration_staging(
|
||||
*,
|
||||
data_dir: Path,
|
||||
migration_id: str,
|
||||
namespace: str,
|
||||
) -> SdkMigrationStaging:
|
||||
"""Return same-filesystem paths without trusting the migration ID as a path."""
|
||||
token = hashlib.sha256(migration_id.encode()).hexdigest()[:16]
|
||||
home = data_dir.resolve()
|
||||
staging_home = home / ".sdk-migration" / token
|
||||
live_root = home / "hy-memory" / namespace
|
||||
return SdkMigrationStaging(
|
||||
migration_id=migration_id,
|
||||
staging_home=staging_home,
|
||||
live_root=live_root,
|
||||
staging_root=staging_home / "hy-memory" / namespace,
|
||||
protective_root=live_root.parent / f".{namespace}.pre-migration-{token}",
|
||||
state_file=staging_home / "swap-state.json",
|
||||
)
|
||||
|
||||
|
||||
def _write_staging_state(paths: SdkMigrationStaging, phase: str) -> None:
|
||||
paths.staging_home.mkdir(parents=True, exist_ok=True)
|
||||
temporary = paths.state_file.with_suffix(".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(
|
||||
{"schema_version": 1, "migration_id": paths.migration_id, "phase": phase},
|
||||
sort_keys=True,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(paths.state_file)
|
||||
|
||||
|
||||
def recover_sdk_migration_swap(
|
||||
*,
|
||||
paths: SdkMigrationStaging,
|
||||
database_path: Path,
|
||||
) -> None:
|
||||
"""Recover an interrupted directory swap before opening any SDK store."""
|
||||
if not paths.protective_root.exists():
|
||||
return
|
||||
authority = SdkDataPlaneMigrationStore(database_path).authority()
|
||||
if authority.value == "sdk":
|
||||
if not paths.live_root.is_dir():
|
||||
raise DataPlaneMigrationError("sdk_migration_committed_store_missing")
|
||||
shutil.rmtree(paths.protective_root)
|
||||
shutil.rmtree(paths.staging_home, ignore_errors=True)
|
||||
return
|
||||
if paths.live_root.exists():
|
||||
if paths.staging_root.exists():
|
||||
raise DataPlaneMigrationError("sdk_migration_swap_state_ambiguous")
|
||||
paths.staging_root.parent.mkdir(parents=True, exist_ok=True)
|
||||
paths.live_root.replace(paths.staging_root)
|
||||
paths.protective_root.replace(paths.live_root)
|
||||
_write_staging_state(paths, "recovered")
|
||||
|
||||
|
||||
def prepare_sdk_migration_staging(
|
||||
*,
|
||||
paths: SdkMigrationStaging,
|
||||
database_path: Path,
|
||||
) -> Path:
|
||||
"""Build a private copy; incomplete retries restart from the unchanged live root."""
|
||||
recover_sdk_migration_swap(paths=paths, database_path=database_path)
|
||||
migration_store = SdkDataPlaneMigrationStore(database_path)
|
||||
try:
|
||||
progress = migration_store.progress(paths.migration_id)
|
||||
reuse_ready = progress.state == "ready" and progress.phase == "reconciled"
|
||||
except Exception:
|
||||
reuse_ready = False
|
||||
if paths.staging_root.exists():
|
||||
if paths.staging_root.is_symlink() or not paths.staging_root.is_dir():
|
||||
raise DataPlaneMigrationError("sdk_migration_staging_invalid")
|
||||
if reuse_ready:
|
||||
return paths.staging_home
|
||||
shutil.rmtree(paths.staging_root)
|
||||
paths.live_root.mkdir(parents=True, exist_ok=True)
|
||||
from one_memory.hy_sdk_migration import snapshot_data_plane
|
||||
|
||||
snapshot_data_plane(
|
||||
source_root=paths.live_root,
|
||||
target_root=paths.staging_root,
|
||||
writers_stopped=True,
|
||||
)
|
||||
_write_staging_state(paths, "prepared")
|
||||
return paths.staging_home
|
||||
|
||||
|
||||
def activate_sdk_migration_staging(
|
||||
*,
|
||||
paths: SdkMigrationStaging,
|
||||
database_path: Path,
|
||||
commit: Callable[[], CutoverMarker],
|
||||
) -> CutoverMarker:
|
||||
"""Atomically expose staged SDK files and roll them back if cutover fails."""
|
||||
recover_sdk_migration_swap(paths=paths, database_path=database_path)
|
||||
if not paths.staging_root.is_dir():
|
||||
raise DataPlaneMigrationError("sdk_migration_staging_missing")
|
||||
paths.live_root.mkdir(parents=True, exist_ok=True)
|
||||
_write_staging_state(paths, "activating")
|
||||
paths.live_root.replace(paths.protective_root)
|
||||
try:
|
||||
paths.staging_root.replace(paths.live_root)
|
||||
_write_staging_state(paths, "staging_live")
|
||||
marker = commit()
|
||||
except BaseException:
|
||||
if paths.live_root.exists():
|
||||
paths.staging_root.parent.mkdir(parents=True, exist_ok=True)
|
||||
paths.live_root.replace(paths.staging_root)
|
||||
if paths.protective_root.exists():
|
||||
paths.protective_root.replace(paths.live_root)
|
||||
_write_staging_state(paths, "rolled_back")
|
||||
raise
|
||||
with suppress(OSError):
|
||||
_write_staging_state(paths, "committed")
|
||||
# Authority is already committed; leftover paths are detected by
|
||||
# sdk-migration-status and removed by sdk-cleanup on the next startup.
|
||||
shutil.rmtree(paths.protective_root, ignore_errors=True)
|
||||
shutil.rmtree(paths.staging_home, ignore_errors=True)
|
||||
return marker
|
||||
|
||||
|
||||
def prepare_legacy_write_queue_for_cutover(database_path: Path) -> tuple[int, int]:
|
||||
"""Freeze old jobs and move unstarted task rows into the surviving queue."""
|
||||
with main_database(database_path).read(row_factory=sqlite3.Row) as database:
|
||||
@@ -354,6 +217,7 @@ def commit_reconciled_cutover(
|
||||
reconciliation_digest=digest,
|
||||
)
|
||||
inspect_sdk_migration_bundle(data_dir=data_dir, bundle_id=marker.backup_bundle_id)
|
||||
drop_legacy_tables(database_path)
|
||||
return marker
|
||||
|
||||
|
||||
@@ -368,23 +232,13 @@ def resume_legacy_cleanup(*, data_dir: Path, database_path: Path) -> tuple[str,
|
||||
data_dir=data_dir,
|
||||
bundle_id=marker.backup_bundle_id,
|
||||
)
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
|
||||
|
||||
recover_sdk_migration_swap(
|
||||
paths=sdk_migration_staging(
|
||||
data_dir=data_dir,
|
||||
migration_id=marker.migration_id,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
),
|
||||
database_path=database_path,
|
||||
)
|
||||
return drop_legacy_tables(database_path)
|
||||
|
||||
|
||||
async def run_sdk_data_plane_backfill(
|
||||
*,
|
||||
database_path: Path,
|
||||
client: SdkMigrationClient,
|
||||
bridge: SdkMigrationBridge,
|
||||
migration_id: str,
|
||||
bundle_id: str,
|
||||
source_runtime_build: str,
|
||||
@@ -407,28 +261,15 @@ async def run_sdk_data_plane_backfill(
|
||||
store.set_phase(migration_id, phase="inventory", state="running")
|
||||
|
||||
legacy_memories = read_legacy_memory_inventory(database_path)
|
||||
sdk_memories = await read_sdk_memory_inventory(client, legacy=legacy_memories)
|
||||
sdk_memories = await read_sdk_memory_inventory(bridge)
|
||||
memory_decisions = record_memory_migration_plan(
|
||||
store,
|
||||
migration_id=migration_id,
|
||||
legacy=legacy_memories,
|
||||
sdk=sdk_memories,
|
||||
)
|
||||
with main_database(database_path).read(row_factory=sqlite3.Row) as database:
|
||||
started = database.execute(
|
||||
"SELECT created_at FROM sdk_data_plane_migrations WHERE migration_id=?",
|
||||
(migration_id,),
|
||||
).fetchone()
|
||||
if started is None:
|
||||
raise DataPlaneMigrationError("sdk_data_plane_migration_start_missing")
|
||||
started_at = datetime.fromisoformat(str(started["created_at"]))
|
||||
if started_at.tzinfo is None:
|
||||
started_at = started_at.replace(tzinfo=UTC)
|
||||
legacy_raw = read_legacy_raw_inventory(
|
||||
database_path,
|
||||
cutoff=started_at.astimezone(UTC) - timedelta(days=3),
|
||||
)
|
||||
sdk_raw = await read_sdk_raw_inventory(client)
|
||||
legacy_raw = read_legacy_raw_inventory(database_path)
|
||||
sdk_raw = await read_sdk_raw_inventory(bridge)
|
||||
raw_decisions = record_raw_migration_plan(
|
||||
store,
|
||||
migration_id=migration_id,
|
||||
@@ -438,14 +279,14 @@ async def run_sdk_data_plane_backfill(
|
||||
|
||||
store.set_phase(migration_id, phase="backfill", state="running")
|
||||
await execute_memory_migration_plan(
|
||||
client,
|
||||
bridge,
|
||||
store,
|
||||
migration_id=migration_id,
|
||||
legacy=legacy_memories,
|
||||
decisions=memory_decisions,
|
||||
)
|
||||
await execute_raw_migration_plan(
|
||||
client,
|
||||
bridge,
|
||||
store,
|
||||
migration_id=migration_id,
|
||||
legacy=legacy_raw,
|
||||
|
||||
@@ -1,330 +0,0 @@
|
||||
"""Single-write adapter from durable Evidence payloads to the public HY SDK."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from one_memory.hy_sdk_runtime import SdkClientHandle
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ModelProfileStore
|
||||
from one_memory.secrets import SecretStore, SecretStoreUnavailableError
|
||||
|
||||
PRODUCT_AGENT_ID = "all-apps"
|
||||
_MESSAGE_ROLES = frozenset({"user", "assistant", "tool"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SdkWriteReceipt:
|
||||
"""The only SDK write identity retained by MemHub."""
|
||||
|
||||
raw_ids: tuple[str, ...] = ()
|
||||
memory_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class SdkWriteProcessor:
|
||||
"""Turn one durable job into exactly one public SDK write call."""
|
||||
|
||||
def __init__(self, client_handle: SdkClientHandle) -> None:
|
||||
self._client_handle = client_handle
|
||||
|
||||
async def process_sdk_job(
|
||||
self,
|
||||
records: tuple[EvidenceRecord, ...],
|
||||
*,
|
||||
request_id: str,
|
||||
before_hy_invocation: Callable[[], Awaitable[None]],
|
||||
) -> SdkWriteReceipt:
|
||||
if not records:
|
||||
raise RuntimeError("hy_memory_task_without_inputs")
|
||||
|
||||
scope = _write_scope(records)
|
||||
metadata = _write_metadata(records, scope)
|
||||
client = await self._client_handle.client()
|
||||
|
||||
if all(_is_extracted_content(record) for record in records):
|
||||
contents = [sanitize_model_input(record.verbatim) for record in records]
|
||||
await before_hy_invocation()
|
||||
result = await asyncio.to_thread(
|
||||
client.add_extracted,
|
||||
contents,
|
||||
user_id=scope.user_id,
|
||||
agent_id=PRODUCT_AGENT_ID,
|
||||
session_id=scope.session_id,
|
||||
source=scope.source,
|
||||
metadata=metadata,
|
||||
memory_at=scope.memory_at,
|
||||
request_id=request_id,
|
||||
)
|
||||
else:
|
||||
messages = [
|
||||
{
|
||||
"role": record.role,
|
||||
"content": sanitize_model_input(record.verbatim),
|
||||
}
|
||||
for record in records
|
||||
if record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and record.role in _MESSAGE_ROLES
|
||||
]
|
||||
if not messages:
|
||||
raise RuntimeError("hy_memory_task_without_inputs")
|
||||
await before_hy_invocation()
|
||||
result = await asyncio.to_thread(
|
||||
client.add,
|
||||
messages,
|
||||
user_id=scope.user_id,
|
||||
agent_id=PRODUCT_AGENT_ID,
|
||||
session_id=scope.session_id,
|
||||
source=scope.source,
|
||||
metadata=metadata,
|
||||
memory_at=scope.memory_at,
|
||||
scene=None,
|
||||
workspace_id=scope.workspace,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
return _receipt_from_result(result)
|
||||
|
||||
async def inquire_write_outcome(self, request_id: str) -> SdkWriteReceipt | None:
|
||||
"""Return a confirmed successful receipt without scanning SDK stores."""
|
||||
if not request_id:
|
||||
raise ValueError("request_id is required")
|
||||
client = await self._client_handle.client()
|
||||
outcome = await asyncio.to_thread(client.get_write_outcome, request_id)
|
||||
if outcome is None:
|
||||
return None
|
||||
if not isinstance(outcome, Mapping):
|
||||
raise RuntimeError("hy_sdk_write_outcome_shape_invalid")
|
||||
|
||||
status = outcome.get("status")
|
||||
if status != "succeeded":
|
||||
return None
|
||||
result = outcome.get("result")
|
||||
if not isinstance(result, Mapping):
|
||||
raise RuntimeError("hy_sdk_write_outcome_shape_invalid")
|
||||
return _receipt_from_result(result)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._client_handle.close()
|
||||
|
||||
|
||||
class SdkWriteCoordinator:
|
||||
"""Resolve the active model profile, then execute one public SDK write."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
profiles: ModelProfileStore,
|
||||
secrets: SecretStore,
|
||||
data_dir: Any,
|
||||
transport_factory: Callable[[ModelProfileSnapshot], Callable[[], Any] | None],
|
||||
sedimentation_enabled: Callable[[], bool],
|
||||
) -> None:
|
||||
self.profiles = profiles
|
||||
self.secrets = secrets
|
||||
self.data_dir = data_dir
|
||||
self.transport_factory = transport_factory
|
||||
self.sedimentation_enabled = sedimentation_enabled
|
||||
self._handles: dict[tuple[str, int], SdkClientHandle] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def process_sdk_job(
|
||||
self,
|
||||
records: tuple[EvidenceRecord, ...],
|
||||
*,
|
||||
request_id: str,
|
||||
before_hy_invocation: Callable[[], Awaitable[None]],
|
||||
) -> SdkWriteReceipt:
|
||||
if not self.sedimentation_enabled():
|
||||
raise RuntimeError("model_sedimentation_disabled")
|
||||
handle = await self._active_handle()
|
||||
return await SdkWriteProcessor(handle).process_sdk_job(
|
||||
records,
|
||||
request_id=request_id,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
)
|
||||
|
||||
async def inquire_write_outcome(
|
||||
self, request_id: str
|
||||
) -> SdkWriteReceipt | None:
|
||||
return await SdkWriteProcessor(
|
||||
await self._active_handle()
|
||||
).inquire_write_outcome(request_id)
|
||||
|
||||
async def close(self) -> None:
|
||||
async with self._lock:
|
||||
handles, self._handles = tuple(self._handles.values()), {}
|
||||
await asyncio.gather(*(handle.close() for handle in handles))
|
||||
|
||||
async def client(self) -> Any:
|
||||
"""Return the public SDK client selected by the active write profile."""
|
||||
return await (await self._active_handle()).client()
|
||||
|
||||
async def active_profile_credentials(
|
||||
self,
|
||||
) -> tuple[ModelProfileSnapshot, str | None]:
|
||||
"""Resolve one stable profile snapshot for graph administration."""
|
||||
_identity, profile, secret = await asyncio.to_thread(self._read_active_credentials)
|
||||
return profile, secret
|
||||
|
||||
async def _active_handle(self) -> SdkClientHandle:
|
||||
identity, profile, secret = await asyncio.to_thread(
|
||||
self._read_active_credentials
|
||||
)
|
||||
async with self._lock:
|
||||
handle = self._handles.get(identity)
|
||||
if handle is None:
|
||||
handle = SdkClientHandle(
|
||||
profile,
|
||||
api_key=secret,
|
||||
data_dir=self.data_dir,
|
||||
http_transport_factory=self.transport_factory(profile),
|
||||
)
|
||||
self._handles[identity] = handle
|
||||
stale = [
|
||||
value
|
||||
for key, value in self._handles.items()
|
||||
if key != identity
|
||||
]
|
||||
self._handles = {identity: handle}
|
||||
await asyncio.gather(*(value.close() for value in stale))
|
||||
return handle
|
||||
|
||||
def _read_active_credentials(
|
||||
self,
|
||||
) -> tuple[tuple[str, int], ModelProfileSnapshot, str | None]:
|
||||
identity = self.profiles.active_identity()
|
||||
if identity is None:
|
||||
raise RuntimeError("model_profile_not_active")
|
||||
profile = self.profiles.snapshot(*identity)
|
||||
try:
|
||||
secret = self.profiles.resolve_snapshot_secret(
|
||||
*identity,
|
||||
self.secrets,
|
||||
)
|
||||
except SecretStoreUnavailableError:
|
||||
raise RuntimeError("secure_storage_unavailable") from None
|
||||
if self.profiles.active_identity() != identity:
|
||||
raise RuntimeError("model_profile_changed")
|
||||
if profile.has_api_key and secret is None:
|
||||
raise RuntimeError("model_secret_unavailable")
|
||||
return identity, profile, secret
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _WriteScope:
|
||||
user_id: str
|
||||
session_id: str
|
||||
source: str
|
||||
workspace: str | None
|
||||
workspaces: tuple[str, ...]
|
||||
memory_at: datetime | None
|
||||
|
||||
|
||||
def _write_scope(records: tuple[EvidenceRecord, ...]) -> _WriteScope:
|
||||
first = records[0]
|
||||
expected = (first.user_id, first.source_id, first.conversation_id)
|
||||
if any(
|
||||
(record.user_id, record.source_id, record.conversation_id) != expected
|
||||
for record in records[1:]
|
||||
):
|
||||
raise RuntimeError("durable_write_multiple_sources")
|
||||
workspaces = tuple(
|
||||
sorted({record.workspace for record in records if record.workspace is not None})
|
||||
)
|
||||
return _WriteScope(
|
||||
user_id=first.user_id,
|
||||
session_id=first.conversation_id,
|
||||
source=first.source_id,
|
||||
workspace=workspaces[0] if len(workspaces) == 1 else None,
|
||||
workspaces=workspaces,
|
||||
memory_at=max(
|
||||
(record.created_at for record in records if record.created_at is not None),
|
||||
default=None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _write_metadata(
|
||||
records: tuple[EvidenceRecord, ...],
|
||||
scope: _WriteScope,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"source_id": scope.source,
|
||||
"session_id": scope.session_id,
|
||||
"workspace_ids": list(scope.workspaces),
|
||||
"memory_at": scope.memory_at.isoformat() if scope.memory_at is not None else None,
|
||||
"evidence_ids": [record.evidence_id for record in records],
|
||||
"input_kind": (
|
||||
"memory_content_without_raw"
|
||||
if all(_is_extracted_content(record) for record in records)
|
||||
else "raw_conversation"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _is_extracted_content(record: EvidenceRecord) -> bool:
|
||||
if record.content_kind is EvidenceContentKind.MEMORY_CONTENT:
|
||||
return True
|
||||
manual = record.origin_meta.get(EvidenceOrigin.MANUAL_IMPORT)
|
||||
return (
|
||||
record.source_id == "one-memory-manual"
|
||||
and manual is not None
|
||||
and manual.raw_meta.get("input_mode") == "direct_memory_note"
|
||||
)
|
||||
|
||||
|
||||
def _receipt_from_result(value: object) -> SdkWriteReceipt:
|
||||
if not isinstance(value, Mapping):
|
||||
raise RuntimeError("hy_sdk_write_receipt_shape_invalid")
|
||||
if value.get("success") is not True:
|
||||
raise RuntimeError("hy_sdk_write_failed")
|
||||
|
||||
# Read only documented public receipt fields. ``variant`` identifies the
|
||||
# productivity namespace; IDs from both variants are canonical memory IDs.
|
||||
variant = value.get("variant")
|
||||
if variant is not None and not isinstance(variant, str):
|
||||
raise RuntimeError("hy_sdk_write_receipt_shape_invalid")
|
||||
raw_ids = _public_ids(value, singular="raw_id", plural="raw_ids")
|
||||
memory_ids = _ordered_unique(
|
||||
(
|
||||
*_public_ids(value, singular="memory_id", plural="memory_ids"),
|
||||
*_public_ids(value, plural="experience_ids"),
|
||||
*_public_ids(value, plural="fact_ids"),
|
||||
)
|
||||
)
|
||||
return SdkWriteReceipt(raw_ids=raw_ids, memory_ids=memory_ids)
|
||||
|
||||
|
||||
def _public_ids(
|
||||
value: Mapping[object, object],
|
||||
*,
|
||||
singular: str | None = None,
|
||||
plural: str,
|
||||
) -> tuple[str, ...]:
|
||||
found: list[str] = []
|
||||
if singular is not None:
|
||||
item = value.get(singular)
|
||||
if item is not None:
|
||||
if not isinstance(item, str):
|
||||
raise RuntimeError("hy_sdk_write_receipt_shape_invalid")
|
||||
if item:
|
||||
found.append(item)
|
||||
items = value.get(plural)
|
||||
if items is not None:
|
||||
if isinstance(items, (str, bytes)) or not isinstance(items, Sequence):
|
||||
raise RuntimeError("hy_sdk_write_receipt_shape_invalid")
|
||||
for item in items:
|
||||
if not isinstance(item, str):
|
||||
raise RuntimeError("hy_sdk_write_receipt_shape_invalid")
|
||||
if item:
|
||||
found.append(item)
|
||||
return _ordered_unique(found)
|
||||
|
||||
|
||||
def _ordered_unique(values: Sequence[str]) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(values))
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
| 测试文件 | 等级 | 一句话说明 |
|
||||
| --- | --- | --- |
|
||||
| `test_active_credentials.py` | `contract` | Active credentials are read once per call, never reused after a refusal. |
|
||||
| `test_anchor_invalid_rewind.py` | `contract` | Bug 2 real branch: an in-place same-length rewrite invalidates the anchor. |
|
||||
| `test_architecture_layer_gate.py` | `contract` | A0-01: L0-L4 dependency-direction gate. |
|
||||
| `test_archive_detection.py` | `contract` | S4.1: what an uploaded export is, decided before anything parses it. |
|
||||
@@ -54,7 +55,6 @@
|
||||
| `test_claude_adoption_probe.py` | `contract` | The external-use verifier must distinguish a read from merely mentioning it. |
|
||||
| `test_cli_commands.py` | `contract` | 覆盖 status reports the verified runtime as json、doctor fails when runtime identity cannot be verified、serve and legacy empty command share the same entry 等 4 个场景。 |
|
||||
| `test_cli_lifecycle.py` | `integration` | 覆盖 cli starts and stops without residual runtime 场景。 |
|
||||
| `test_cli_migration_error_codes.py` | `unit` | The startup dialog can only act on the code it is handed. |
|
||||
| `test_cli_restore.py` | `contract` | 覆盖 restore requires a source、restore refuses while the runtime is running、restore previews without confirmation 等 4 个场景。 |
|
||||
| `test_codex_header_provenance.py` | `contract` | 覆盖 header workspace versions and native ids reach file origin、new adapter rereads header after a real checkpoint、machine and unknown session sources do not establish human or agent sender 等 14 个场景。 |
|
||||
| `test_connection_lifetime.py` | `integration` | Every SQLite connection a request opens must be closed when the request ends. |
|
||||
@@ -91,6 +91,7 @@
|
||||
| `test_disable_thinking.py` | `contract` | 运行期生成请求须按档案可选关闭思考;默认不向厂商下发未知字段。 |
|
||||
| `test_dual_engine_helper.py` | `contract` | Tests for the dual-engine harness itself. |
|
||||
| `test_event_loop_blocking.py` | `contract` | Proof, as a test, that blocking work on the loop freezes it — and the fix works. |
|
||||
| `test_evidence_reconcile_primitive.py` | `integration` | A bounded, read-only discovery primitive for the H23 Evidence reconcile. |
|
||||
| `test_evidence_store.py` | `contract` | 覆盖 same source message merges origins without duplication、identical text from different sources is not deduplicated、rescanning an unchanged file does not write the message again 等 26 个场景。 |
|
||||
| `test_export_probe.py` | `contract` | What a probe may say about a stranger's chat history, and what it may not. |
|
||||
| `test_external_installation_permissions.py` | `integration` | 覆盖 external marker is not an installation grant、external diagnostic trace uses its own read only tool、raw source grants filter before limit and private body loading 场景。 |
|
||||
@@ -102,6 +103,8 @@
|
||||
| `test_file_import_operations_review.py` | `contract` | Upload refusal/legacy compatibility review with bounded synthetic files only. |
|
||||
| `test_frozen_core_layout.py` | `contract` | 覆盖 single file layout remains available without private preset、folder layout keeps same core and all native dependencies、unknown layout does not silently build another package 等 5 个场景。 |
|
||||
| `test_golden_corpus.py` | `contract` | A7-01: the golden corpus comparison. |
|
||||
| `test_governance_permission_commit.py` | `contract` | 覆盖 permission guard is rechecked inside write transaction and rolls back、guard context does not leak into subsequent manual actions、guard check and commit hold database write authority 场景。 |
|
||||
| `test_governance_read_path_does_not_take_the_writer.py` | `contract` | The recall read path must not acquire the single global writer. |
|
||||
| `test_graph_width_endpoint.py` | `integration` | The width mismatch has to reach the interface, not just the log. |
|
||||
| `test_graph_width_export.py` | `unit` | EXPORT_FIRST has to actually export. |
|
||||
| `test_graph_width_guard.py` | `unit` | Graph width mismatch must ask, never silently archive. |
|
||||
@@ -112,9 +115,16 @@
|
||||
| `test_host_capabilities.py` | `unit` | A plugin does not become realtime on every host or every unknown version. |
|
||||
| `test_hourly_c1_peak_replay_contract.py` | `contract` | Guard the package peak-replay probe's error accounting. |
|
||||
| `test_hourly_c1_recall_stage_probe_contract.py` | `contract` | Guard the packaged recall stage probe's command and error semantics. |
|
||||
| `test_hy_coding_reply.py` | `contract` | R-07: HY's coding-shaped replies read as "no memories at all". |
|
||||
| `test_hy_log_privacy.py` | `contract` | HY SDK records are not rewritten by One Memory's logging setup. |
|
||||
| `test_hy_memory_bridge.py` | `contract` | 覆盖 hy client initialization is serial across bridges、native hy clients share one runtime per data directory、ollama profile ignores ambient remote storage and models 等 41 个场景。 |
|
||||
| `test_hy_metrics_lifecycle.py` | `contract` | What the pinned public SDK's MetricsCollector does, and what that costs us. |
|
||||
| `test_hy_network_egress.py` | `contract` | Where a real HY client is allowed to send bytes, and where it is not. |
|
||||
| `test_hy_response_guard.py` | `contract` | 覆盖 invalid response retries same hy provider without changing prompt、repeated empty response fails instead of becoming no op、failure receipt survives a later successful model call 等 23 个场景。 |
|
||||
| `test_hy_response_shapes.py` | `contract` | HY response shape selection only; no provider, write routing or model calls. |
|
||||
| `test_hy_runtime_config.py` | `contract` | MEMORY.md is the audited record of what we send HY; keep it answerable. |
|
||||
| `test_hy_sdk_idempotency_probe.py` | `contract` | A0-03: probe whether the HY SDK can make a write idempotent. |
|
||||
| `test_hy_source_context.py` | `contract` | 覆盖 bound context rejects a same named conversation in another workspace、bound context rejects a duplicated evidence id、bound context rejects a same conversation from another source 等 9 个场景。 |
|
||||
| `test_hy_step_trace_switch.py` | `contract` | Rule 15: the per-step HY trace is off unless the debug tier asks for it. |
|
||||
| `test_import_api.py` | `integration` | 覆盖 resume during cleanup returns conflict without internal error、authorize detect import and revoke api flow、protected api rejects missing token and foreign origin 等 5 个场景。 |
|
||||
| `test_import_control_races_review.py` | `contract` | Read-only review repro: real runner/store, synthetic delayed adapter cleanup. |
|
||||
@@ -158,11 +168,16 @@
|
||||
| `test_memory_batch_extraction.py` | `contract` | 覆盖 batch extractor keeps only unique known evidence ids、batch extractor maps provider failure to machine error 场景。 |
|
||||
| `test_memory_citation_query_plan.py` | `contract` | Correlated citation checks must look links up by memory, not scan by 'active'. |
|
||||
| `test_memory_content_import.py` | `integration` | 覆盖 existing memory preview keeps unknown author and exact original、invalid existing memory records never leave a partial upload、memory content requires explicit mode and rejects vendor guessing 等 5 个场景。 |
|
||||
| `test_memory_correction_sync.py` | `integration` | R-27: a correction that only lands locally, while HY keeps the old text. |
|
||||
| `test_memory_evidence_freshness.py` | `contract` | 新鲜度拆除后:读路径恒为 current;证据修订 / 畸形 proof / segment ref 不再藏记忆或 503。 |
|
||||
| `test_memory_freshness_integration_review.py` | `integration` | Independent synthetic read-path review; no lifespan, model or user database. |
|
||||
| `test_memory_governance.py` | `contract` | 覆盖 correction keeps verbatim and appends history、mutations are revision checked and request idempotent、delete result survives restart and request replay 等 42 个场景。 |
|
||||
| `test_memory_governance_api.py` | `integration` | 覆盖 memory governance api preserves evidence and reports cleanup、memory validation does not echo content 场景。 |
|
||||
| `test_memory_graph_api.py` | `integration` | 覆盖 graph labels where every edge came from、a created memory is on the map with its source links、drawing the map does not stop the app answering 等 12 个场景。 |
|
||||
| `test_memory_language.py` | `contract` | R-03: memory language follows the message, so Kristin's gets written in English. |
|
||||
| `test_memory_notes.py` | `integration` | 覆盖 empty library can save a note without a matter or configured model、note requires authentication explicit scope and nonempty content、saved note evidence returns raw and consent reference from same record 等 6 个场景。 |
|
||||
| `test_memory_processing.py` | `contract` | 覆盖 stale known reason diagnostics never fail dispatch、memory task logs six step durations without content、assistant turns reach hy so references can be resolved 等 95 个场景。 |
|
||||
| `test_memory_processing_diagnostics.py` | `contract` | 覆盖 conversation prepare failure includes safe message and stack、deletion sync failure includes safe message and stack 场景。 |
|
||||
| `test_memory_qa.py` | `contract` | 覆盖 search terms keep meaningful single chinese character、streamed answer emits recall then incremental grounded text、governance change during generation stops the old answer 等 14 个场景。 |
|
||||
| `test_memory_qa_api.py` | `integration` | 覆盖 ask stream endpoint emits recall deltas and final、ask stream endpoint requires the local token 场景。 |
|
||||
| `test_memory_qa_permissions.py` | `contract` | 覆盖 initial refusal precedes all secret access and network、invalid authorization result fails before secret access、authorization receives exact question and only outbound grounding 等 16 个场景。 |
|
||||
@@ -274,6 +289,7 @@
|
||||
| `test_secret_store_error_code_api.py` | `integration` | 3925a kernel item 3: the error code reaches the HTTP boundary. |
|
||||
| `test_secret_store_error_codes.py` | `integration` | 3925a kernel item 3: keychain failure classification and backoff that heals. |
|
||||
| `test_secure_storage.py` | `integration` | 覆盖 isolated keychain service namespace、custom keychain service is refused outside isolated tests、packaged keychain budget covers loaded process startup 等 20 个场景。 |
|
||||
| `test_sedimentation_concurrency.py` | `contract` | Sedimentation runs several harnesses at once, but one harness at a time. |
|
||||
| `test_sedimentation_switch.py` | `contract` | Stopping model-driven sedimentation without losing the configuration. |
|
||||
| `test_service_level_pressure_acceptance.py` | `contract` | 覆盖 workbuddy recall remains available during 100k import 场景。 |
|
||||
| `test_service_levels.py` | `contract` | 覆盖 background never consumes the realtime reservation、realtime is next for the same source after background yields、import history gets one slice after nine realtime admissions 等 8 个场景。 |
|
||||
@@ -287,7 +303,6 @@
|
||||
| `test_source_authorization_state.py` | `unit` | Why a source cannot be read, named separately from whether it can. |
|
||||
| `test_source_candidate_api.py` | `integration` | Candidate location changes must not implicitly grant or import anything. |
|
||||
| `test_source_candidate_connections.py` | `contract` | 覆盖 empty candidate revisions preserve default fingerprint、persisted candidate requires exact revision、connection resolves candidate in transaction and prior replay never regrants 等 7 个场景。 |
|
||||
| `test_source_candidate_kind_parity.py` | `contract` | 来源根目录类型在三处必须同时存在:python 枚举、TS 契约、前端运行时校验。 |
|
||||
| `test_source_candidates.py` | `contract` | 覆盖 preview is metadata only and cancel changes nothing、select persists only candidate and restarts with revision、compare and swap and aba cannot reuse old preview 等 9 个场景。 |
|
||||
| `test_source_connection_intents.py` | `integration` | No user data: connection retries must retain the original intent and job. |
|
||||
| `test_source_discovery.py` | `contract` | 覆盖 discovery is sorted bounded and suffix scoped、discovery does not follow directory or file symlinks 场景。 |
|
||||
@@ -305,6 +320,7 @@
|
||||
| `test_source_sync_idempotent.py` | `integration` | Final-state Harness sync is idempotent and resumes missed source work. |
|
||||
| `test_source_sync_permissions.py` | `integration` | S1.3 source permission tests use temporary files only, never real host history. |
|
||||
| `test_source_totals.py` | `contract` | 覆盖 the first ask is answered from the rows、same named sessions in two workspaces count as two conversations、insert during count is not marked as already counted 等 7 个场景。 |
|
||||
| `test_source_write_switch.py` | `contract` | A frozen Harness produces no extraction tasks. |
|
||||
| `test_sql_compat.py` | `contract` | The shared compatibility layer, especially 'rowcount'. |
|
||||
| `test_sqlite_semantics_gate.py` | `contract` | D56: prevent SQLite implementation details spreading outside the data layer. |
|
||||
| `test_sqlite_write_contention.py` | `contract` | Measure whether SQLite's single writer actually blocks us. |
|
||||
@@ -329,4 +345,4 @@
|
||||
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
|
||||
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
|
||||
|
||||
共 324 个测试文件。
|
||||
共 340 个测试文件。
|
||||
@@ -14,14 +14,13 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
|
||||
EXPECTED_TEST_FILE_COUNT = 324
|
||||
EXPECTED_TEST_FILES_SHA256 = "3fcc7a3336de56aa8fa6b0dddd6101441a837095c89dd5f31d2a1b66ec7e3764"
|
||||
EXPECTED_TEST_FILE_COUNT = 340
|
||||
EXPECTED_TEST_FILES_SHA256 = "f69d3f33df5277febb09557d5bce7adb7d8719ed47e48a71f852e3e4ecf8fd74"
|
||||
|
||||
# Pure transformations, validation models, parity checks and state machines.
|
||||
# This list is intentionally small: uncertain files default to contract.
|
||||
UNIT_FILES = frozenset(
|
||||
{
|
||||
"test_cli_migration_error_codes.py",
|
||||
"test_context_v2_contract_parity.py",
|
||||
"test_daemon_contract.py",
|
||||
"test_diagnostic_payload_keys.py",
|
||||
@@ -63,6 +62,7 @@ INTEGRATION_FILES = frozenset(
|
||||
"test_d93_same_process_load.py",
|
||||
"test_diagnostic_trace.py",
|
||||
"test_diagnostic_trace_api.py",
|
||||
"test_evidence_reconcile_primitive.py",
|
||||
"test_external_installation_permissions.py",
|
||||
"test_fault_matrix.py",
|
||||
"test_file_import_operations.py",
|
||||
@@ -77,6 +77,7 @@ INTEGRATION_FILES = frozenset(
|
||||
"test_manual_transcript_import.py",
|
||||
"test_measure_c1_loop_blocking_contract.py",
|
||||
"test_memory_content_import.py",
|
||||
"test_memory_correction_sync.py",
|
||||
"test_memory_freshness_integration_review.py",
|
||||
"test_memory_governance_api.py",
|
||||
"test_memory_graph_api.py",
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
|
||||
def model_snapshot(
|
||||
provider: ProviderKind,
|
||||
*,
|
||||
base_url: str | None,
|
||||
has_api_key: bool,
|
||||
embedding_provider: ProviderKind | None = None,
|
||||
embedding_base_url: str | None = None,
|
||||
) -> ModelProfileSnapshot:
|
||||
return ModelProfileSnapshot(
|
||||
profile_id="profile",
|
||||
revision=3,
|
||||
provider_kind=provider,
|
||||
base_url=base_url,
|
||||
model="memory-model",
|
||||
embedding_model="embedding-model",
|
||||
embedding_provider_kind=embedding_provider,
|
||||
embedding_base_url=embedding_base_url,
|
||||
embedding_dimensions=768,
|
||||
timeout_seconds=30,
|
||||
max_concurrency=1,
|
||||
context_window=8192,
|
||||
has_api_key=has_api_key,
|
||||
)
|
||||
|
||||
|
||||
def kuzu_graph_with_embedding_width(db_path: Path, width: int) -> None:
|
||||
kuzu = pytest.importorskip("kuzu")
|
||||
database = kuzu.Database(str(db_path))
|
||||
connection = kuzu.Connection(database)
|
||||
connection.execute(
|
||||
f"CREATE NODE TABLE Memory("
|
||||
f"node_id STRING, embedding FLOAT[{width}], PRIMARY KEY(node_id))"
|
||||
)
|
||||
del connection
|
||||
del database
|
||||
gc.collect()
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Active credentials are read once per call, never reused after a refusal."""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from pydantic import SecretStr
|
||||
|
||||
from one_memory.background_work import shutdown_bulk_pool
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridgeError
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_governance import MemoryGovernanceStore
|
||||
from one_memory.memory_processing import MemoryProcessingSupervisor
|
||||
from one_memory.memory_tasks import MemoryTaskStore
|
||||
from one_memory.model_profiles import (
|
||||
ModelCapabilities,
|
||||
ModelProfileInput,
|
||||
ModelProfilePatch,
|
||||
ModelProfileStore,
|
||||
)
|
||||
from one_memory.secrets import InMemorySecretStore, SecretStoreUnavailableError
|
||||
|
||||
|
||||
class CountingSecrets(InMemorySecretStore):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.reads = []
|
||||
self.denied = False
|
||||
self.after_read = None
|
||||
|
||||
def has(self, secret_ref):
|
||||
# The OS store's has() resolves the secret; the in-memory default does not.
|
||||
return self.resolve(secret_ref) is not None
|
||||
|
||||
def resolve(self, secret_ref):
|
||||
self.reads.append((secret_ref, threading.get_ident()))
|
||||
if self.denied:
|
||||
raise SecretStoreUnavailableError("synthetic reader denial, must not leak")
|
||||
value = super().resolve(secret_ref)
|
||||
callback, self.after_read = self.after_read, None
|
||||
if callback is not None:
|
||||
callback()
|
||||
return value
|
||||
|
||||
|
||||
def create_profile(profiles, secrets, *, keyed=True):
|
||||
profile = profiles.create(
|
||||
ModelProfileInput(
|
||||
name="Synthetic credentials",
|
||||
provider_kind="openai_compatible" if keyed else "ollama",
|
||||
base_url="http://127.0.0.1:1/v1",
|
||||
model="synthetic-model",
|
||||
api_key=SecretStr("synthetic-credential") if keyed else None,
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
profiles.record_capabilities(
|
||||
profile.profile_id,
|
||||
expected_revision=profile.revision,
|
||||
capabilities=ModelCapabilities(
|
||||
endpoint_reachable=True,
|
||||
model_available=True,
|
||||
structured_output=True,
|
||||
embedding_available=True,
|
||||
),
|
||||
secrets=secrets,
|
||||
)
|
||||
return profiles.activate(profile.profile_id, secrets)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup(tmp_path):
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
secrets = CountingSecrets()
|
||||
profiles = ModelProfileStore(database)
|
||||
profile = create_profile(profiles, secrets)
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=MemoryTaskStore(database),
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=EvidenceStore(database),
|
||||
governance=MemoryGovernanceStore(database),
|
||||
data_dir=tmp_path,
|
||||
)
|
||||
secrets.reads.clear()
|
||||
try:
|
||||
yield supervisor, profiles, secrets, profile
|
||||
finally:
|
||||
shutdown_bulk_pool()
|
||||
|
||||
|
||||
def test_resolves_active_immutable_snapshot_once_off_loop(setup):
|
||||
supervisor, profiles, secrets, profile = setup
|
||||
expected = profiles.snapshot(profile.profile_id, profile.revision)
|
||||
caller = threading.get_ident()
|
||||
|
||||
result = asyncio.run(supervisor._active_credentials())
|
||||
|
||||
assert result == (expected, "synthetic-credential")
|
||||
assert secrets.reads == [(profiles.secret_ref(profile.profile_id), secrets.reads[0][1])]
|
||||
assert secrets.reads[0][1] != caller
|
||||
assert supervisor.tasks.list() == []
|
||||
assert supervisor._bridges == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("concurrent", [False, True])
|
||||
def test_each_call_resolves_again_without_cross_request_cache(setup, concurrent):
|
||||
supervisor, _profiles, secrets, _profile = setup
|
||||
|
||||
async def scenario():
|
||||
if concurrent:
|
||||
return await asyncio.gather(*(supervisor._active_credentials() for _ in range(4)))
|
||||
return [await supervisor._active_credentials() for _ in range(4)]
|
||||
|
||||
results = asyncio.run(scenario())
|
||||
|
||||
assert all(result is not None and result[1] == "synthetic-credential" for result in results)
|
||||
assert len(secrets.reads) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["missing", "denied"])
|
||||
def test_prior_success_cannot_mask_current_reader_refusal(setup, failure):
|
||||
supervisor, profiles, secrets, profile = setup
|
||||
|
||||
async def scenario():
|
||||
assert (await supervisor._active_credentials())[1] == "synthetic-credential"
|
||||
secrets.reads.clear()
|
||||
if failure == "missing":
|
||||
secrets.delete(profiles.secret_ref(profile.profile_id))
|
||||
else:
|
||||
secrets.denied = True
|
||||
code = "model_secret_unavailable" if failure == "missing" else "secure_storage_unavailable"
|
||||
with pytest.raises(HyMemoryBridgeError, match=f"^{code}$"):
|
||||
await supervisor._active_credentials()
|
||||
assert len(secrets.reads) == 1
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["deactivate", "switch", "revision"])
|
||||
def test_profile_change_during_secret_read_rejects_old_credentials(setup, change):
|
||||
supervisor, profiles, secrets, profile = setup
|
||||
other = create_profile(profiles, secrets)
|
||||
profiles.activate(profile.profile_id, secrets)
|
||||
secrets.reads.clear()
|
||||
|
||||
def change_identity():
|
||||
# Concurrent committed identity change while the original reader is in flight.
|
||||
with profiles._connect(writing=True) as db:
|
||||
if change == "revision":
|
||||
db.execute(
|
||||
"UPDATE model_profiles SET revision=revision+1 WHERE profile_id=?",
|
||||
(profile.profile_id,),
|
||||
)
|
||||
else:
|
||||
db.execute("UPDATE model_profiles SET is_active=0")
|
||||
if change == "switch":
|
||||
db.execute(
|
||||
"UPDATE model_profiles SET is_active=1 WHERE profile_id=?",
|
||||
(other.profile_id,),
|
||||
)
|
||||
|
||||
secrets.after_read = change_identity
|
||||
with pytest.raises(HyMemoryBridgeError, match="^model_profile_changed$"):
|
||||
asyncio.run(supervisor._active_credentials())
|
||||
assert len(secrets.reads) == 1
|
||||
assert supervisor._bridges == {}
|
||||
|
||||
|
||||
def test_rotation_between_calls_returns_new_snapshot_and_secret(setup):
|
||||
supervisor, profiles, secrets, profile = setup
|
||||
before = asyncio.run(supervisor._active_credentials())
|
||||
updated = profiles.update(
|
||||
profile.profile_id,
|
||||
ModelProfilePatch(expected_revision=profile.revision, api_key=SecretStr("synthetic-new")),
|
||||
secrets,
|
||||
)
|
||||
assert updated.is_active is False
|
||||
assert updated.capabilities == ModelCapabilities()
|
||||
assert profiles.active_identity() is None
|
||||
secrets.reads.clear()
|
||||
|
||||
assert asyncio.run(supervisor._active_credentials()) is None
|
||||
assert secrets.reads == []
|
||||
assert supervisor.tasks.list() == []
|
||||
assert supervisor._bridges == {}
|
||||
|
||||
# Synthetic probe evidence and explicit activation belong to the new revision.
|
||||
profiles.record_capabilities(
|
||||
updated.profile_id,
|
||||
expected_revision=updated.revision,
|
||||
capabilities=profile.capabilities,
|
||||
secrets=secrets,
|
||||
)
|
||||
updated = profiles.activate(updated.profile_id, secrets)
|
||||
assert updated.is_active is True
|
||||
secrets.reads.clear()
|
||||
|
||||
after = asyncio.run(supervisor._active_credentials())
|
||||
|
||||
assert before[0].revision == 1 and before[1] == "synthetic-credential"
|
||||
assert after == (profiles.snapshot(updated.profile_id, updated.revision), "synthetic-new")
|
||||
assert after[0].revision == 2
|
||||
assert len(secrets.reads) == 1
|
||||
|
||||
|
||||
def test_no_active_profile_does_not_read_credentials(setup):
|
||||
supervisor, profiles, secrets, _profile = setup
|
||||
with profiles._connect(writing=True) as db:
|
||||
db.execute("UPDATE model_profiles SET is_active=0")
|
||||
|
||||
assert asyncio.run(supervisor._active_credentials()) is None
|
||||
assert secrets.reads == []
|
||||
|
||||
|
||||
def test_keyless_profile_does_not_require_or_resolve_a_secret(setup):
|
||||
supervisor, profiles, secrets, _profile = setup
|
||||
keyless = create_profile(profiles, secrets, keyed=False)
|
||||
secrets.reads.clear()
|
||||
|
||||
result = asyncio.run(supervisor._active_credentials())
|
||||
|
||||
assert result == (profiles.snapshot(keyless.profile_id, keyless.revision), None)
|
||||
assert result[0].has_api_key is False
|
||||
assert secrets.reads == []
|
||||
@@ -1,14 +1,16 @@
|
||||
"""Executable dependency and memory-authority gates.
|
||||
|
||||
The public target architecture is defined by ``docs/ARCHITECTURE.md``:
|
||||
The current target architecture is defined by ``HANDSOFF.md`` and
|
||||
``docs/ARCHITECTURE.md``:
|
||||
|
||||
* Hy Memory SDK owns memory/raw/vector/graph/coding data;
|
||||
* MemHub owns the management plane and may retain ``memory_id`` references;
|
||||
* product code reaches the SDK only through narrow runtime/migration modules;
|
||||
* MemHub calls public ``HyMemoryClient`` methods only.
|
||||
* product code reaches the SDK through ``hy_memory_bridge`` only.
|
||||
|
||||
The sole exception is the read-only Kuzu projection/admin surface used by the
|
||||
existing graph UI and Graph Width workflow.
|
||||
The legacy implementation still contains Evidence/Governance writers and a
|
||||
small number of private SDK-store accesses. Those are migration debt, not an
|
||||
allowance for new code. The explicit debt lists below may shrink during the
|
||||
refactor; new entries must not be added.
|
||||
|
||||
The gate is deliberately not a blanket ban on ``sqlite3``. Adapters may read a
|
||||
host product's SQLite through the guarded read-only module; they may not own
|
||||
@@ -57,13 +59,13 @@ HOST_READONLY_MODULE = "one_memory.imports.sqlite_access"
|
||||
SRC_ROOT = PACKAGE_ROOT.parents[0]
|
||||
PACKAGE_LABEL = PACKAGE_ROOT.name
|
||||
|
||||
SDK_MODULES = frozenset(
|
||||
{
|
||||
f"{PACKAGE_LABEL}/hy_sdk_runtime.py",
|
||||
f"{PACKAGE_LABEL}/hy_sdk_migration.py",
|
||||
}
|
||||
)
|
||||
SDK_CLIENT_CONSTRUCTOR = f"{PACKAGE_LABEL}/hy_sdk_runtime.py"
|
||||
HY_BRIDGE = f"{PACKAGE_LABEL}/hy_memory_bridge.py"
|
||||
|
||||
#: The one module allowed to *invoke* the SDK.
|
||||
SDK_CALLER = HY_BRIDGE
|
||||
|
||||
#: Type-only importers of HY datatypes; they hold no client and write nothing.
|
||||
HY_TYPE_ONLY: frozenset[str] = frozenset()
|
||||
|
||||
# Frozen migration debt: modules that still write the legacy memory/raw
|
||||
# authority. The target count is zero. A new module cannot join this set.
|
||||
@@ -83,38 +85,20 @@ LEGACY_AUTHORITY_WRITE = re.compile(
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
SDK_PRIVATE_ATTRIBUTES = frozenset(
|
||||
{
|
||||
"_coding_store",
|
||||
"_coding_stores",
|
||||
"_collection",
|
||||
"_embed_service",
|
||||
"_graph_store",
|
||||
"_loop_thread",
|
||||
"_readers",
|
||||
"_reader",
|
||||
"_reconciler",
|
||||
"_registry",
|
||||
"_run",
|
||||
"_shared_embed_service",
|
||||
"_spawn_search_dedup",
|
||||
"_system2_writer",
|
||||
"_vector_store",
|
||||
"_writer",
|
||||
}
|
||||
)
|
||||
SDK_PRIVATE_ACCESS_EXCEPTIONS = {
|
||||
f"{PACKAGE_LABEL}/hy_graph_projection.py": frozenset(
|
||||
{"_graph_store", "_loop_thread", "_run"}
|
||||
),
|
||||
# Startup migration runs while Runtime writers are stopped. It is the one
|
||||
# non-runtime path allowed to manipulate SDK storage directly so that
|
||||
# migration-only mechanics never become public HyMemoryClient methods.
|
||||
f"{PACKAGE_LABEL}/hy_sdk_migration.py": frozenset(
|
||||
{"_embed_service", "_graph_store", "_loop_thread", "_vector_store"}
|
||||
# Existing accesses to SDK implementation details. These are exact file /
|
||||
# attribute debts so deleting one is always green, while adding one fails.
|
||||
SDK_PRIVATE_ACCESS_DEBT: dict[str, frozenset[str]] = {
|
||||
f"{PACKAGE_LABEL}/hy_lifecycle.py": frozenset({"_collection", "_vector_store"}),
|
||||
f"{PACKAGE_LABEL}/hy_memory_bridge.py": frozenset(
|
||||
{"_coding_store", "_graph_store", "_loop_thread", "_vector_store"}
|
||||
),
|
||||
f"{PACKAGE_LABEL}/hy_response_guard.py": frozenset({"_loop_thread", "_registry"}),
|
||||
}
|
||||
|
||||
SDK_PRIVATE_ATTRIBUTES = frozenset(
|
||||
attribute for attributes in SDK_PRIVATE_ACCESS_DEBT.values() for attribute in attributes
|
||||
)
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
return str(path.relative_to(SRC_ROOT))
|
||||
@@ -188,26 +172,49 @@ def test_adapters_use_only_the_readonly_host_sqlite_module() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_hy_sdk_is_confined_to_public_sdk_modules() -> None:
|
||||
"""Only the runtime and offline migration adapters may import the SDK."""
|
||||
def test_hy_sdk_is_confined_to_the_l1_adapter() -> None:
|
||||
"""A0-01: HY SDK stays behind HyMemoryPort; entries must not import it.
|
||||
|
||||
This is the executable form of "Function/MCP/scan/Hook must not call the
|
||||
HY SDK directly". `hy_memory_bridge` is the current L1 adapter and the
|
||||
permission stores quote HY types, so they are the allowed set; anything
|
||||
else reaching for the SDK is a new bypass and must fail here.
|
||||
|
||||
Only `hy_memory_bridge` *calls* the SDK. The two permission stores merely
|
||||
import HY datatypes (`CodingMemory`, `WorkingFact`, memory models) to type
|
||||
what they persist; they hold no client and issue no writes. They are
|
||||
listed here so the gate stays green on today's truth while still failing
|
||||
on any new module that reaches for the SDK.
|
||||
"""
|
||||
allowed = frozenset({HY_BRIDGE}) | HY_TYPE_ONLY
|
||||
offenders: list[str] = []
|
||||
for path in _all_module_files():
|
||||
relative = _relative(path)
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
names = _module_names(tree)
|
||||
if not any(_root(name) in HY_SDK_ROOTS for name in names):
|
||||
if relative in allowed:
|
||||
continue
|
||||
if relative not in SDK_MODULES:
|
||||
names = _module_names(ast.parse(path.read_text(encoding="utf-8")))
|
||||
if any(_root(name) in HY_SDK_ROOTS for name in names):
|
||||
offenders.append(relative)
|
||||
assert not offenders, f"HY SDK imports must stay in {sorted(SDK_MODULES)}: {offenders}"
|
||||
assert not offenders, f"HY SDK must stay in the L1 adapter: {offenders}"
|
||||
|
||||
|
||||
def test_only_runtime_constructs_a_hy_client() -> None:
|
||||
"""Migration borrows a lease; only runtime constructs clients."""
|
||||
def test_only_the_l1_adapter_constructs_a_hy_client() -> None:
|
||||
"""A0-01: type-only imports are tolerated; constructing a client is not.
|
||||
|
||||
A permission store quoting `CodingMemory` cannot write a memory. A module
|
||||
that instantiates `HyMemoryClient` can. So the stricter check is on client
|
||||
construction, and it must pass today rather than merely documenting an
|
||||
aspiration.
|
||||
|
||||
The check is on the AST, not on the substring: a docstring or comment that
|
||||
*mentions* `HyMemoryClient` builds nothing. Matching text would flag prose
|
||||
and, worse, would keep passing while a real caller is added inside a
|
||||
function the text search happened to like.
|
||||
"""
|
||||
offenders: list[str] = []
|
||||
for path in _all_module_files():
|
||||
relative = _relative(path)
|
||||
if relative == SDK_CLIENT_CONSTRUCTOR:
|
||||
if relative == SDK_CALLER:
|
||||
continue
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
for node in ast.walk(tree):
|
||||
@@ -227,7 +234,7 @@ def test_only_runtime_constructs_a_hy_client() -> None:
|
||||
if any(alias.name == "HyMemoryClient" for alias in node.names):
|
||||
offenders.append(relative)
|
||||
break
|
||||
assert not offenders, f"only {SDK_CLIENT_CONSTRUCTOR} may build a HY client: {offenders}"
|
||||
assert not offenders, f"only the L1 adapter may build a HY client: {offenders}"
|
||||
|
||||
|
||||
def test_no_new_module_writes_legacy_memory_authority() -> None:
|
||||
@@ -241,88 +248,39 @@ def test_no_new_module_writes_legacy_memory_authority() -> None:
|
||||
if relative not in LEGACY_AUTHORITY_WRITERS:
|
||||
offenders.append(relative)
|
||||
assert not offenders, (
|
||||
"new Evidence/Governance writers are forbidden; use public SDK client "
|
||||
f"methods instead: {offenders}"
|
||||
"new Evidence/Governance writers are forbidden; write memory/raw via "
|
||||
f"the SDK bridge instead: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def test_no_sdk_private_implementation_access() -> None:
|
||||
"""Runtime uses public APIs; only Graph and stopped-writer migration are privileged."""
|
||||
def test_no_new_sdk_private_store_access() -> None:
|
||||
"""Freeze direct SDK-store access until public client APIs replace it."""
|
||||
offenders: list[str] = []
|
||||
for path in _all_module_files():
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
relative = _relative(path)
|
||||
names = _module_names(tree)
|
||||
if relative not in SDK_PRIVATE_ACCESS_DEBT and not any(
|
||||
_root(name) in HY_SDK_ROOTS for name in names
|
||||
):
|
||||
# Names such as `_registry` and `_collection` are common for
|
||||
# MemHub-owned objects too. They become SDK-private access only
|
||||
# in a module that imports the SDK (or is frozen debt already).
|
||||
continue
|
||||
used = {
|
||||
node.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Attribute)
|
||||
and node.attr in SDK_PRIVATE_ATTRIBUTES
|
||||
and (
|
||||
relative.startswith(f"{PACKAGE_LABEL}/hy_")
|
||||
or (
|
||||
isinstance(node.value, ast.Name)
|
||||
and node.value.id in {"client", "native", "concrete"}
|
||||
)
|
||||
)
|
||||
if isinstance(node, ast.Attribute) and node.attr in SDK_PRIVATE_ATTRIBUTES
|
||||
}
|
||||
if used:
|
||||
unexpected = used - SDK_PRIVATE_ACCESS_EXCEPTIONS.get(
|
||||
relative, frozenset()
|
||||
)
|
||||
if unexpected:
|
||||
offenders.append(f"{relative}: {sorted(unexpected)}")
|
||||
assert not offenders, (
|
||||
"SDK private implementation access is forbidden outside the isolated "
|
||||
f"Graph and offline-migration boundaries: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def test_sdk_modules_import_only_the_public_package_root() -> None:
|
||||
"""Runtime imports only public SDK symbols; offline migration is privileged."""
|
||||
offenders: list[str] = []
|
||||
for path in _all_module_files():
|
||||
relative = _relative(path)
|
||||
if relative == f"{PACKAGE_LABEL}/hy_sdk_migration.py":
|
||||
if not used:
|
||||
continue
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import) and any(
|
||||
alias.name.startswith("hy_memory.") for alias in node.names
|
||||
):
|
||||
offenders.append(relative)
|
||||
break
|
||||
if isinstance(node, ast.ImportFrom) and (
|
||||
node.module or ""
|
||||
).startswith("hy_memory."):
|
||||
offenders.append(relative)
|
||||
break
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id == "import_module"
|
||||
and node.args
|
||||
and isinstance(node.args[0], ast.Constant)
|
||||
and isinstance(node.args[0].value, str)
|
||||
and node.args[0].value.startswith("hy_memory.")
|
||||
):
|
||||
offenders.append(relative)
|
||||
break
|
||||
assert not offenders, f"import SDK symbols from `hy_memory` only: {offenders}"
|
||||
|
||||
|
||||
def test_memhub_never_patches_sdk_prompts() -> None:
|
||||
offenders: list[str] = []
|
||||
forbidden = (
|
||||
"extract_system_prompt_en",
|
||||
"extract_system_prompt_zh",
|
||||
"hy_memory.agent.prompts",
|
||||
unexpected = sorted(used - SDK_PRIVATE_ACCESS_DEBT.get(relative, frozenset()))
|
||||
if unexpected:
|
||||
offenders.append(f"{relative}: {unexpected}")
|
||||
assert not offenders, (
|
||||
"new SDK private-store access is forbidden; add a public HyMemoryClient "
|
||||
f"API and call it through hy_memory_bridge: {offenders}"
|
||||
)
|
||||
for path in _all_module_files():
|
||||
source = path.read_text(encoding="utf-8")
|
||||
hits = [token for token in forbidden if token in source]
|
||||
if hits:
|
||||
offenders.append(f"{_relative(path)}: {hits}")
|
||||
assert not offenders, f"SDK prompt patching is forbidden: {offenders}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("driver", sorted(MEMHUB_DRIVERS))
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
"""The startup dialog can only act on the code it is handed.
|
||||
|
||||
`sdk_migration_backfill_failed` covered a full disk, a missing model profile and
|
||||
a Coding classification gap alike, so every one of them reached the person as
|
||||
the same unactionable sentence. These tests pin the names that replace it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.cli import cutover_failure_code, migration_failure_code
|
||||
|
||||
|
||||
class _Coded(RuntimeError):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def test_running_out_of_disk_is_named_however_deep_it_is_wrapped() -> None:
|
||||
error = OSError(errno.ENOSPC, "No space left on device")
|
||||
wrapped: BaseException = error
|
||||
for _ in range(3):
|
||||
try:
|
||||
raise RuntimeError("wrapper") from wrapped
|
||||
except RuntimeError as caught:
|
||||
wrapped = caught
|
||||
|
||||
assert migration_failure_code(wrapped) == "sdk_migration_disk_space_insufficient"
|
||||
# The explicit context chain counts too: `raise ... from None` discards the
|
||||
# cause but Python still keeps the original as __context__.
|
||||
try:
|
||||
try:
|
||||
raise OSError(errno.ENOSPC, "No space left on device")
|
||||
except OSError:
|
||||
raise RuntimeError("sdk_snapshot_failed") from None
|
||||
except RuntimeError as caught:
|
||||
assert migration_failure_code(caught) == "sdk_migration_disk_space_insufficient"
|
||||
|
||||
|
||||
def test_quota_exhaustion_counts_as_no_space(tmp_path: Path) -> None:
|
||||
quota = getattr(errno, "EDQUOT", errno.ENOSPC)
|
||||
assert (
|
||||
migration_failure_code(OSError(quota, "Disk quota exceeded"))
|
||||
== "sdk_migration_disk_space_insufficient"
|
||||
)
|
||||
|
||||
|
||||
def test_a_coding_inventory_gap_is_not_reported_as_a_generic_failure() -> None:
|
||||
assert (
|
||||
migration_failure_code(_Coded("hy_sdk_productivity_variant_invalid"))
|
||||
== "sdk_migration_coding_inventory_failed"
|
||||
)
|
||||
|
||||
|
||||
def test_the_legacy_working_fact_failure_keeps_its_own_name() -> None:
|
||||
error = _Coded("hy_sdk_memory_import_failed")
|
||||
error.__cause__ = ValueError("Invalid memory layer: working_fact")
|
||||
|
||||
assert migration_failure_code(error) == "sdk_migration_working_fact_conflict"
|
||||
|
||||
|
||||
def test_migration_state_codes_pass_through_because_they_are_already_actionable() -> None:
|
||||
assert (
|
||||
migration_failure_code(_Coded("sdk_data_plane_migration_item_conflict"))
|
||||
== "sdk_data_plane_migration_item_conflict"
|
||||
)
|
||||
assert (
|
||||
migration_failure_code(_Coded("sdk_migration_staging_invalid"))
|
||||
== "sdk_migration_staging_invalid"
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_failures_keep_their_message_out_of_the_code() -> None:
|
||||
error = RuntimeError("/Users/someone/private/path blew up")
|
||||
assert migration_failure_code(error) == "sdk_migration_backfill_failed"
|
||||
assert (
|
||||
migration_failure_code(error, fallback="sdk_migration_preflight_failed")
|
||||
== "sdk_migration_preflight_failed"
|
||||
)
|
||||
|
||||
|
||||
def test_a_non_code_attribute_never_becomes_a_code() -> None:
|
||||
class Odd(RuntimeError):
|
||||
code = 17
|
||||
|
||||
assert migration_failure_code(Odd("x")) == "sdk_migration_backfill_failed"
|
||||
|
||||
|
||||
def test_the_chain_survives_a_cycle() -> None:
|
||||
first = RuntimeError("first")
|
||||
second = RuntimeError("second")
|
||||
first.__cause__ = second
|
||||
second.__cause__ = first
|
||||
|
||||
assert migration_failure_code(first) == "sdk_migration_backfill_failed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"code",
|
||||
[
|
||||
"sdk_migration_disk_space_insufficient",
|
||||
"sdk_migration_working_fact_conflict",
|
||||
"sdk_migration_coding_inventory_failed",
|
||||
"sdk_migration_work_pending",
|
||||
],
|
||||
)
|
||||
def test_the_codes_the_desktop_translates_stay_in_sync(code: str) -> None:
|
||||
"""Desktop maps these exact names to sentences; renaming one blinds it."""
|
||||
desktop = Path(__file__).resolve().parents[2] / "desktop" / "src" / "stable-runtime.ts"
|
||||
assert desktop.is_file(), desktop
|
||||
assert f'"{code}"' in desktop.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_cutover_work_counts_do_not_become_the_dialog_text() -> None:
|
||||
"""The blocker code carries a JSON body; the person gets one name."""
|
||||
blocked = _Coded('sdk_data_plane_cutover_work_pending:{"memory_tasks":591}')
|
||||
|
||||
assert cutover_failure_code(blocked) == "sdk_migration_work_pending"
|
||||
assert (
|
||||
cutover_failure_code(_Coded("sdk_data_plane_cutover_missing"))
|
||||
== "sdk_data_plane_cutover_missing"
|
||||
)
|
||||
assert (
|
||||
cutover_failure_code(RuntimeError("/Users/someone/private"))
|
||||
== "sdk_migration_cutover_failed"
|
||||
)
|
||||
@@ -1,15 +1,10 @@
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
from one_memory import cli
|
||||
from one_memory.config import Settings
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
|
||||
from one_memory.memory_server.migrations import MIGRATIONS
|
||||
from one_memory.runtime_client import RuntimeStatus
|
||||
from one_memory.sdk_data_plane_migration import SdkDataPlaneMigrationStore
|
||||
|
||||
|
||||
def test_restore_requires_a_source(tmp_path, monkeypatch) -> None:
|
||||
@@ -306,15 +301,11 @@ def test_sdk_migrate_runs_offline_backfill_without_cutover(tmp_path, monkeypatch
|
||||
class Secrets:
|
||||
pass
|
||||
|
||||
class Lease:
|
||||
client = SimpleNamespace(
|
||||
_loop_thread=None,
|
||||
_vector_store=None,
|
||||
_embed_service=None,
|
||||
_graph_store=None,
|
||||
)
|
||||
class Bridge:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
async def close(self):
|
||||
return None
|
||||
|
||||
async def backfill(**_kwargs):
|
||||
@@ -329,10 +320,7 @@ def test_sdk_migrate_runs_offline_backfill_without_cutover(tmp_path, monkeypatch
|
||||
|
||||
monkeypatch.setattr("one_memory.model_profiles.ModelProfileStore", Profiles)
|
||||
monkeypatch.setattr("one_memory.secrets.KeyringSecretStore", Secrets)
|
||||
monkeypatch.setattr(
|
||||
"one_memory.hy_sdk_runtime.acquire_client",
|
||||
lambda *_args, **_kwargs: Lease(),
|
||||
)
|
||||
monkeypatch.setattr("one_memory.hy_memory_bridge.HyMemoryBridge", Bridge)
|
||||
monkeypatch.setattr(
|
||||
"one_memory.sdk_migration_runner.run_sdk_data_plane_backfill",
|
||||
backfill,
|
||||
@@ -368,18 +356,10 @@ def test_sdk_cutover_is_offline_confirmed_and_returns_marker(tmp_path, monkeypat
|
||||
lambda **_kwargs: SimpleNamespace(
|
||||
migration_id="migration-1",
|
||||
backup_bundle_id="bundle-1",
|
||||
sdk_version="2.0.5",
|
||||
sdk_version="2.0.5.dev3",
|
||||
completed_at="2026-10-21T00:00:00Z",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"one_memory.sdk_migration_runner.activate_sdk_migration_staging",
|
||||
lambda *, commit, **_kwargs: commit(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"one_memory.sdk_migration_runner.resume_legacy_cleanup",
|
||||
lambda **_kwargs: (),
|
||||
)
|
||||
output = io.StringIO()
|
||||
|
||||
code = cli.run(
|
||||
@@ -398,7 +378,7 @@ def test_sdk_cutover_is_offline_confirmed_and_returns_marker(tmp_path, monkeypat
|
||||
assert code == 0
|
||||
document = output.getvalue().replace(" ", "")
|
||||
assert '"state":"cutover"' in document
|
||||
assert '"sdk_version":"2.0.5"' in document
|
||||
assert '"sdk_version":"2.0.5.dev3"' in document
|
||||
|
||||
|
||||
def test_sdk_cutover_never_runs_while_runtime_is_live(tmp_path, monkeypatch) -> None:
|
||||
@@ -441,41 +421,3 @@ def test_sdk_migration_status_distinguishes_empty_and_legacy_data(tmp_path, monk
|
||||
assert cli.run(["sdk-migration-status", "--json"], output=required) == 0
|
||||
assert '"state": "required"' in required.getvalue()
|
||||
assert '"raw": 1' in required.getvalue()
|
||||
|
||||
|
||||
def test_sdk_migration_status_exposes_a_resumable_active_identity(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
cli,
|
||||
"load_settings",
|
||||
lambda: Settings(_env_file=None, data_dir=tmp_path),
|
||||
)
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
asyncio.run(migrate(SqliteMigrationStore(database), MIGRATIONS))
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO evidence ("
|
||||
"evidence_id,idempotency_key,user_id,source_id,conversation_id,"
|
||||
"content_kind,role,written_by,verbatim,content_hash,revision,"
|
||||
"imported_at,updated_at"
|
||||
") VALUES ('e1','k','local','cursor','s','raw_conversation',"
|
||||
"'user','person','hello','hash',1,'now','now')"
|
||||
)
|
||||
SdkDataPlaneMigrationStore(database).start(
|
||||
migration_id="sdk-authority-v2-0123456789abcdef",
|
||||
source_runtime_build="17",
|
||||
target_sdk_version="2.0.5",
|
||||
source_schema_version=MIGRATIONS[-1].version,
|
||||
namespace_fingerprint="namespace",
|
||||
backend_fingerprint="backend",
|
||||
backup_bundle_id=(
|
||||
"sdk-authority-v2-0123456789abcdef-pre-cutover"
|
||||
),
|
||||
)
|
||||
|
||||
output = io.StringIO()
|
||||
assert cli.run(["sdk-migration-status", "--json"], output=output) == 0
|
||||
document = json.loads(output.getvalue())
|
||||
assert document["migration_id"] == "sdk-authority-v2-0123456789abcdef"
|
||||
assert document["bundle_id"] == (
|
||||
"sdk-authority-v2-0123456789abcdef-pre-cutover"
|
||||
)
|
||||
@@ -50,3 +50,21 @@ def test_governed_memory_carries_the_same_fields_on_both_sides() -> None:
|
||||
}
|
||||
|
||||
assert typescript == set(GovernedMemory.model_fields)
|
||||
|
||||
|
||||
def test_an_agent_owned_memory_is_usable_and_never_passes_as_the_person(tmp_path) -> None:
|
||||
"""助手输出里的事实直接可用,但始终标成助手说的。
|
||||
|
||||
召回读到它时,origin 让别的 AI 应用知道这是模型的陈述、不是本人的判决。
|
||||
已有的同内容记忆,也不会被模型输出改掉来源标记。
|
||||
"""
|
||||
from tests.test_memory_processing import (
|
||||
test_agent_output_keeps_the_origin_of_a_same_content_memory,
|
||||
test_agent_owned_memories_land_usable_but_marked_as_the_assistant,
|
||||
)
|
||||
|
||||
created, existing = tmp_path / "created", tmp_path / "existing"
|
||||
created.mkdir()
|
||||
existing.mkdir()
|
||||
test_agent_owned_memories_land_usable_but_marked_as_the_assistant(created)
|
||||
test_agent_output_keeps_the_origin_of_a_same_content_memory(existing, MemoryOrigin.AGENT)
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Searching imported conversations before any generation model is configured.
|
||||
|
||||
This search is literal, not semantic, and that is a correction rather than a
|
||||
shortcut. Literal conversation search must remain available before a model is
|
||||
configured and independently of SDK semantic-memory extraction.
|
||||
shortcut. The vector store `semantic_search` reads is written only by
|
||||
extraction (`memory_processing.py` EXTRACT tasks), and extraction is precisely
|
||||
what a user without a generation model does not have: their store is empty, so
|
||||
a semantic path would return nothing no matter how it was wired.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
@@ -4,9 +4,9 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from one_memory.conversation_processing import _conversation_chunks
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_processing import _conversation_chunks
|
||||
|
||||
|
||||
def _records(tmp_path, *speakers: tuple[str, str, str]) -> list:
|
||||
|
||||
@@ -1,16 +1,58 @@
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from test_memory_qa import _grounded_setup, _ndjson_stream
|
||||
from test_onboarding_profile import _app, _client
|
||||
from test_onboarding_profile import FakeGraph, _app, _client, _setup, _supervisor
|
||||
|
||||
from one_memory.memory_qa import MemoryAnswerer
|
||||
from one_memory.model_probe import ModelProber
|
||||
from one_memory.secrets import SecretStoreUnavailableError
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"operation,args",
|
||||
[
|
||||
("digest", ()),
|
||||
("graph_snapshot", ()),
|
||||
("graph_width_mismatch", ()),
|
||||
("semantic_search", ("example",)),
|
||||
("update_hy_memory", ("id", "corrected")),
|
||||
("delete_hy_memory", ("id",)),
|
||||
("decide_graph_width", ("keep_existing",)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("phase", ["active", "secret"])
|
||||
def test_credentials_do_not_block_the_event_loop(tmp_path, monkeypatch, operation, args, phase):
|
||||
stores = _setup(tmp_path)
|
||||
supervisor = _supervisor(tmp_path, stores, FakeGraph(()))
|
||||
method = "active_identity" if phase == "active" else "resolve_snapshot_secret"
|
||||
original = getattr(supervisor.profiles, method)
|
||||
called = threading.Event()
|
||||
|
||||
def waiting(*args, **kwargs):
|
||||
called.set()
|
||||
time.sleep(0.25)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(supervisor.profiles, method, waiting)
|
||||
|
||||
async def scenario():
|
||||
task = asyncio.create_task(getattr(supervisor, operation)(*args))
|
||||
started = time.monotonic()
|
||||
await asyncio.sleep(0.04)
|
||||
elapsed = time.monotonic() - started
|
||||
# Some graph operations have no fake implementation; those exceptions
|
||||
# are irrelevant to whether a credential read stops the event loop.
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
assert called.is_set()
|
||||
assert elapsed < 0.18, f"Credential wait blocked the loop for {elapsed:.3f}s"
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_disabled_automatic_processing_does_not_digest_onboarding(tmp_path):
|
||||
async def scenario():
|
||||
app = _app(tmp_path)
|
||||
@@ -22,7 +64,7 @@ def test_disabled_automatic_processing_does_not_digest_onboarding(tmp_path):
|
||||
calls.append(True)
|
||||
return {}
|
||||
|
||||
app.state.sdk_memory_runtime.digest = digest
|
||||
app.state.memory_processing_supervisor.digest = digest
|
||||
response = await client.post("/api/v1/onboarding/complete")
|
||||
assert response.status_code == 200
|
||||
assert calls == []
|
||||
|
||||
@@ -35,7 +35,7 @@ def test_live_log_level_applies_to_full_hy_and_provider_call_chain(
|
||||
set_diagnostic_log_level(tmp_path, "debug")
|
||||
|
||||
assert diagnostic_log_level(tmp_path) == "debug"
|
||||
assert logging.getLogger("one_memory.sdk_write_processor").isEnabledFor(logging.DEBUG)
|
||||
assert logging.getLogger("one_memory.memory_processing").isEnabledFor(logging.DEBUG)
|
||||
assert logging.getLogger("hy_memory.runtime").isEnabledFor(logging.DEBUG)
|
||||
assert logging.getLogger("hy_memory.client").isEnabledFor(logging.DEBUG)
|
||||
assert logging.getLogger("hy_memory.agent.llm_provider").isEnabledFor(logging.DEBUG)
|
||||
|
||||
@@ -18,13 +18,11 @@ from one_memory.diagnostic_log import (
|
||||
diagnostic_exception,
|
||||
diagnostics_enabled,
|
||||
diagnostics_requested,
|
||||
read_trace_events,
|
||||
record_database_error,
|
||||
reset_diagnostics,
|
||||
reset_trace_id,
|
||||
resource_snapshot,
|
||||
)
|
||||
from one_memory.exception_reporting import report_exception
|
||||
|
||||
|
||||
def test_debug_events_are_collected_only_at_debug_level(tmp_path, monkeypatch) -> None:
|
||||
@@ -107,31 +105,7 @@ def test_structured_log_is_private_correlated_and_owner_only(tmp_path):
|
||||
assert log.path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_arbitrary_valid_detail_keys_and_scalars_are_recorded(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
details = {
|
||||
"Custom.metric-v1": "ready",
|
||||
"noneValue": None,
|
||||
"boolValue": False,
|
||||
"intValue": 7,
|
||||
"floatValue": 1.5,
|
||||
}
|
||||
log.write(
|
||||
level="info",
|
||||
component="core.lifecycle",
|
||||
event="core.started",
|
||||
outcome="succeeded",
|
||||
details=details,
|
||||
)
|
||||
assert log.flush()
|
||||
assert json.loads(log.path.read_text())["details"] == details
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
["__proto__", "prototype", "constructor", "_private", "9startsWithDigit", "a" * 65],
|
||||
)
|
||||
def test_dangerous_or_malformed_detail_keys_are_rejected(tmp_path, key):
|
||||
def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_detail_key_invalid"):
|
||||
log.write(
|
||||
@@ -139,76 +113,8 @@ def test_dangerous_or_malformed_detail_keys_are_rejected(tmp_path, key):
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={key: "value"},
|
||||
details={"prompt": "private"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [{}, [], object()])
|
||||
def test_complex_detail_values_are_rejected(tmp_path, value):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={"payload": value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
|
||||
def test_non_finite_detail_values_are_rejected(tmp_path, value):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={"metric": value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [2**53, -(2**53), float(2**53), -float(2**53)])
|
||||
def test_numeric_detail_values_outside_json_safe_range_are_rejected(tmp_path, value):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={"count": value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "nul\0byte", "carriage\rreturn", "line\nfeed", "x" * 257])
|
||||
def test_invalid_detail_strings_are_rejected(tmp_path, value):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={"message": value},
|
||||
)
|
||||
|
||||
|
||||
def test_more_than_32_detail_keys_are_rejected(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_detail_count_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={f"key{index}": index for index in range(33)},
|
||||
)
|
||||
|
||||
|
||||
def test_absolute_or_credential_bearing_stack_is_rejected(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
@@ -217,90 +123,16 @@ def test_absolute_or_credential_bearing_stack_is_rejected(tmp_path):
|
||||
outcome="failed",
|
||||
safe_stack=("/Users/me/private.py:1",),
|
||||
)
|
||||
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
safe_stack=("one_memory/sk-live-secret-token.py:1:call",),
|
||||
)
|
||||
|
||||
|
||||
def test_safe_stack_uses_strict_shared_grammar_and_sixteen_line_limit(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
for line in (
|
||||
"arbitrary stack text",
|
||||
"thread=worker|threading.py:12:wait",
|
||||
"FILE:///Users/me/private.py:1:call",
|
||||
"one_memory/worker.py:9:call /Users/private",
|
||||
"one_memory/GOCSPX-privateCredential.py:9:call",
|
||||
"packages/ASIA1234567890ABCDEF/file.ts:1:1",
|
||||
"packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
|
||||
):
|
||||
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
safe_stack=(line,),
|
||||
)
|
||||
def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) -> None:
|
||||
"""Rejection protects the log; it must not become the caller's outage.
|
||||
|
||||
accepted = (
|
||||
"apps/desktop/src/main.ts:80:1",
|
||||
"one_memory/diagnostic_log.py:895:_safe_stack_line",
|
||||
"threading.py:1045:_bootstrap_inner",
|
||||
"thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
|
||||
)
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
safe_stack=accepted,
|
||||
)
|
||||
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
|
||||
log.write(
|
||||
level="error",
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
safe_stack=tuple(f"one_memory/worker.py:{index + 1}:call" for index in range(17)),
|
||||
)
|
||||
assert log.flush()
|
||||
assert json.loads(log.path.read_text())["safe_stack"] == list(accepted)
|
||||
|
||||
|
||||
def test_invalid_level_is_strict_but_diagnostic_event_remains_fail_open(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
log = DiagnosticLog(tmp_path)
|
||||
with pytest.raises(ValueError, match="diagnostic_level_invalid"):
|
||||
log.write(
|
||||
level="fatal", # type: ignore[arg-type]
|
||||
component="core.lifecycle",
|
||||
event="core.failed",
|
||||
outcome="failed",
|
||||
)
|
||||
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path / "fail-open")
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="fatal",
|
||||
component="core.lifecycle",
|
||||
event="core.failed",
|
||||
outcome="failed",
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
assert not receipt.path.exists()
|
||||
|
||||
|
||||
def test_a_rejected_payload_does_not_break_the_caller_or_later_events(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
Two of our own nodes shipped with unlisted `details` keys. Both sat on a
|
||||
request path, so the rejection surfaced as a 500 to the person using the
|
||||
app — including the sedimentation switch, whose every request failed
|
||||
because logging the toggle raised before the endpoint could answer.
|
||||
"""
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
@@ -309,122 +141,20 @@ def test_a_rejected_payload_does_not_break_the_caller_or_later_events(
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"payload": ["not", "scalar"]},
|
||||
)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"custom.metric": 1},
|
||||
details={"prompt": "private"},
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["details"] for event in events] == [{"custom.metric": 1}]
|
||||
|
||||
|
||||
def test_trace_projection_does_not_expose_arbitrary_local_detail_keys(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
log.write(
|
||||
level="info",
|
||||
component="core.api",
|
||||
event="memory.request.completed",
|
||||
outcome="succeeded",
|
||||
request_id="request-1",
|
||||
details={
|
||||
"phase": "recall",
|
||||
"reason": "/Users/private/query.txt",
|
||||
"status": "sk-live-secret-token",
|
||||
"custom.private": "local-only",
|
||||
},
|
||||
)
|
||||
assert log.flush()
|
||||
events = read_trace_events(tmp_path, request_id="request-1")
|
||||
assert events[0]["details"] == {"phase": "recall"}
|
||||
|
||||
|
||||
def test_trace_projection_rejects_symlinked_directory_and_redacts_top_level_secrets(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
request_id = "request-1"
|
||||
logs = tmp_path / "logs"
|
||||
logs.mkdir()
|
||||
event = {
|
||||
"timestamp": "2026-10-23T01:00:00Z",
|
||||
"request_id": request_id,
|
||||
"component": "secret.token",
|
||||
"event": "credential.failed",
|
||||
"outcome": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
|
||||
"error_code": "GOCSPX-clientSecret_A1",
|
||||
"error_type": "ASIA1234567890ABCDEF",
|
||||
"details": {
|
||||
"phase": "recall",
|
||||
"reason": "x_ASIA1234567890ABCDEF",
|
||||
"mode": "x_GOCSPX-clientSecret_A1",
|
||||
"status": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
|
||||
"failure_site": "AIzaSyExampleCredential1234567890",
|
||||
"count": 1,
|
||||
},
|
||||
}
|
||||
(logs / "diagnostic-core.jsonl").write_text(json.dumps(event) + "\n")
|
||||
projected = read_trace_events(tmp_path, request_id=request_id)
|
||||
assert projected == (
|
||||
{
|
||||
"timestamp": "2026-10-23T01:00:00Z",
|
||||
"component": "diagnostic.redacted",
|
||||
"event": "diagnostic.redacted",
|
||||
"outcome": "redacted",
|
||||
"details": {"phase": "recall", "count": 1},
|
||||
},
|
||||
)
|
||||
|
||||
external = tmp_path / "external"
|
||||
external.mkdir()
|
||||
(logs / "diagnostic-core.jsonl").unlink()
|
||||
logs.rmdir()
|
||||
logs.symlink_to(external, target_is_directory=True)
|
||||
with pytest.raises(ValueError, match="diagnostic_directory_unsafe"):
|
||||
read_trace_events(tmp_path, request_id=request_id)
|
||||
|
||||
|
||||
def test_credential_detection_is_conservative_and_matches_export_rules() -> None:
|
||||
for value in (
|
||||
"ASIA1234567890ABCDEF",
|
||||
"GOCSPX-clientSecret_A1",
|
||||
"Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
|
||||
"Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8_",
|
||||
):
|
||||
assert diagnostic_log._contains_credential_material(value)
|
||||
for value in (
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"abcdef1234567890abcdef1234567890abcdef12",
|
||||
"cursor",
|
||||
):
|
||||
assert not diagnostic_log._contains_credential_material(value)
|
||||
|
||||
|
||||
def test_trace_reader_never_follows_a_diagnostic_file_symlink(tmp_path) -> None:
|
||||
logs = tmp_path / "logs"
|
||||
logs.mkdir()
|
||||
outside = tmp_path / "outside.jsonl"
|
||||
outside.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"timestamp": "2026-10-23T01:00:00Z",
|
||||
"request_id": "request-1",
|
||||
"component": "core.api",
|
||||
"event": "memory.request.completed",
|
||||
"outcome": "succeeded",
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
(logs / "diagnostic-core.jsonl").symlink_to(outside)
|
||||
assert read_trace_events(tmp_path, request_id="request-1") == ()
|
||||
# Rejected, so nothing is written — but the call returned.
|
||||
assert not receipt.path.exists() or receipt.path.read_text() == ""
|
||||
|
||||
|
||||
def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
|
||||
"""`enabled` is a bounded boolean and has to be a legal detail key.
|
||||
|
||||
Omitting it from the allow-list is what made the switch's own endpoint
|
||||
answer 500 while still saving the new value.
|
||||
"""
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
@@ -451,7 +181,7 @@ def test_exception_diagnostic_keeps_code_location_not_message():
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={"prompt": f"{secret}\n"},
|
||||
details={"prompt": secret},
|
||||
)
|
||||
except ValueError as error:
|
||||
safe = diagnostic_exception(error)
|
||||
@@ -461,28 +191,6 @@ def test_exception_diagnostic_keeps_code_location_not_message():
|
||||
assert str(__file__) not in json.dumps(safe)
|
||||
|
||||
|
||||
def test_report_exception_never_writes_the_exception_message_to_diagnostics(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
secret = "sk-live-private-provider-response"
|
||||
try:
|
||||
report_exception(
|
||||
"core.test",
|
||||
"core.test.failed",
|
||||
RuntimeError(f"provider failed with {secret} at /Users/private/query.txt"),
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
payload = receipt.path.read_text()
|
||||
assert secret not in payload
|
||||
assert "/Users/private" not in payload
|
||||
event = json.loads(payload)
|
||||
assert event["details"] == {"count": 1}
|
||||
assert event["error_type"] == "RuntimeError"
|
||||
|
||||
|
||||
def test_old_logs_are_removed_but_symlinks_are_never_followed(tmp_path):
|
||||
log = DiagnosticLog(tmp_path)
|
||||
log.path.parent.mkdir(parents=True)
|
||||
|
||||
@@ -449,7 +449,7 @@ def test_invalid_input_is_still_rejected_on_the_caller_side(tmp_path):
|
||||
component="hy.bridge",
|
||||
event="hy.failed",
|
||||
outcome="failed",
|
||||
details={"invalid key": "private"},
|
||||
details={"prompt": "private"},
|
||||
)
|
||||
log.close()
|
||||
|
||||
|
||||
@@ -1,57 +1,22 @@
|
||||
"""An instrument must never be able to break the thing it measures.
|
||||
|
||||
`diagnostic_event` validates its payload: a malformed `details` key, a missing
|
||||
`level`, or an unsafe detail value is rejected. A diagnostic payload must never
|
||||
be able to break the request path or loop heartbeat it is intended to measure.
|
||||
`diagnostic_event` validates its payload: an unlisted `details` key, a missing
|
||||
`level`, or a detail value over 256 characters all raise. Two of our own nodes
|
||||
shipped with unlisted keys, and because they sat on a request path and on the
|
||||
loop heartbeat, they turned every external recall into a 500 and then killed
|
||||
the process. Both were caught only by running the built runtime, not by tests.
|
||||
|
||||
So this checks the payloads at their source, by walking every literal call.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.diagnostic_log import _DETAIL_KEYS
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
|
||||
DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
|
||||
REFUSED_DETAIL_KEYS = {"__proto__", "prototype", "constructor"}
|
||||
EXPORTED_STRING_DETAIL_KEYS = {
|
||||
"action",
|
||||
"method",
|
||||
"mode",
|
||||
"phase",
|
||||
"platform",
|
||||
"reason",
|
||||
"source_id",
|
||||
"status",
|
||||
"version",
|
||||
"parent_session_id",
|
||||
"runtime_state",
|
||||
"core_state",
|
||||
"broker_state",
|
||||
"instance_id",
|
||||
"operation_kind",
|
||||
"file_kind",
|
||||
"last_probe",
|
||||
"signal",
|
||||
}
|
||||
PRIVATE_DETAIL_KEYS = {
|
||||
"api_key",
|
||||
"authorization",
|
||||
"content",
|
||||
"credential",
|
||||
"memory",
|
||||
"memory_text",
|
||||
"message",
|
||||
"messages",
|
||||
"password",
|
||||
"prompt",
|
||||
"query",
|
||||
"raw_text",
|
||||
"secret",
|
||||
"token",
|
||||
}
|
||||
|
||||
#: Every module, not a hand-picked list. The two payloads that caused real
|
||||
#: outages were in files nobody thought to add here, so the only shape of this
|
||||
@@ -68,19 +33,6 @@ def _literal_calls(path: Path):
|
||||
yield node
|
||||
|
||||
|
||||
def _contains_free_text_conversion(node: ast.AST) -> bool:
|
||||
for item in ast.walk(node):
|
||||
if (
|
||||
isinstance(item, ast.Call)
|
||||
and isinstance(item.func, ast.Name)
|
||||
and item.func.id in {"str", "repr", "_safe_message"}
|
||||
):
|
||||
return True
|
||||
if isinstance(item, ast.Attribute) and item.attr in {"args", "message"}:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("relative", SOURCES)
|
||||
def test_every_diagnostic_payload_is_valid(relative: str) -> None:
|
||||
path = ROOT / relative
|
||||
@@ -93,22 +45,7 @@ def test_every_diagnostic_payload_is_valid(relative: str) -> None:
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg != "details" or not isinstance(keyword.value, ast.Dict):
|
||||
continue
|
||||
for key, expression in zip(keyword.value.keys, keyword.value.values, strict=True):
|
||||
if not isinstance(key, ast.Constant):
|
||||
continue
|
||||
value = key.value
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or not DETAIL_KEY.fullmatch(value)
|
||||
or value in REFUSED_DETAIL_KEYS
|
||||
):
|
||||
problems.append(f"{relative}:{node.lineno}: invalid details key {value!r}")
|
||||
elif value.lower() in PRIVATE_DETAIL_KEYS:
|
||||
problems.append(f"{relative}:{node.lineno}: private details key {value!r}")
|
||||
elif value in EXPORTED_STRING_DETAIL_KEYS and _contains_free_text_conversion(
|
||||
expression
|
||||
):
|
||||
problems.append(
|
||||
f"{relative}:{node.lineno}: exported detail {value!r} uses free text"
|
||||
)
|
||||
for key in keyword.value.keys:
|
||||
if isinstance(key, ast.Constant) and key.value not in _DETAIL_KEYS:
|
||||
problems.append(f"{relative}:{node.lineno}: unlisted details key {key.value!r}")
|
||||
assert not problems, "\n".join(problems)
|
||||
@@ -0,0 +1,400 @@
|
||||
"""A bounded, read-only discovery primitive for the H23 Evidence reconcile.
|
||||
|
||||
The objective's crash requirement is "a committed Evidence row must not be
|
||||
lost". The two-step `Evidence write -> queue/task submit` window can leave a
|
||||
row with no write entry. C0 scheduled that reconcile for H23 and asked C2 to
|
||||
supply the primitive; this module pins the discovery half, which is pure
|
||||
database infrastructure and does not choose the product's grouping semantics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.imports.models import (
|
||||
EvidenceInput,
|
||||
EvidenceOrigin,
|
||||
OriginMetadata,
|
||||
WrittenBy,
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
|
||||
from one_memory.memory_server.durable_queue import (
|
||||
DurableWriteQueue,
|
||||
ReconcileSchemaUnavailableError,
|
||||
)
|
||||
|
||||
|
||||
def _put(store: EvidenceStore, message_id: str) -> str:
|
||||
return store.put(
|
||||
EvidenceInput(
|
||||
source_id="s1",
|
||||
conversation_id="c1",
|
||||
message_id=message_id,
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
verbatim=f"body {message_id}",
|
||||
created_at=datetime.now(UTC),
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record.evidence_id
|
||||
|
||||
|
||||
def test_unsubmitted_evidence_finds_rows_with_no_write_entry(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path)
|
||||
submitted = _put(store, "m1")
|
||||
orphan = _put(store, "m2")
|
||||
queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id="c1",
|
||||
batch_identity=BatchIdentity(internal_session_id="c1", first_round=1, last_round=1),
|
||||
evidence_ids=[submitted],
|
||||
idempotency_key="idem-1",
|
||||
)
|
||||
)
|
||||
|
||||
found = queue.unsubmitted_evidence_ids(limit=10)
|
||||
|
||||
assert found == (orphan,)
|
||||
|
||||
|
||||
def test_reconcile_refuses_when_the_evidence_schema_is_not_migrated(tmp_path) -> None:
|
||||
"""A queue-only database is not an empty reconcile result.
|
||||
|
||||
Returning `()` here would make a pre-migration database look as if every
|
||||
trusted Evidence row were already queued. The caller needs a distinct
|
||||
retryable state so it can run migration first.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
|
||||
with pytest.raises(ReconcileSchemaUnavailableError, match="run migrations"):
|
||||
queue.unsubmitted_evidence_ids()
|
||||
|
||||
|
||||
def test_unsubmitted_evidence_is_bounded_and_resumable(tmp_path) -> None:
|
||||
"""The cursor must be stable and bounded so a startup pass is cheap.
|
||||
|
||||
`after_cursor` is the last id a previous page returned; the next call
|
||||
resumes strictly after it. In this serial fixture the pages partition the
|
||||
backlog once. The primitive is not a cross-call transaction snapshot; a
|
||||
concurrent insert or submit is tolerated by the Evidence-derived
|
||||
idempotency key, not by pretending the read itself is isolated.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path)
|
||||
ids = [_put(store, f"m{index}") for index in range(3)]
|
||||
|
||||
first = queue.unsubmitted_evidence_ids(limit=2)
|
||||
second = queue.unsubmitted_evidence_ids(limit=2, after_cursor=first[-1])
|
||||
|
||||
assert first == (ids[0], ids[1])
|
||||
assert second == (ids[2],)
|
||||
|
||||
|
||||
def test_unsubmitted_evidence_refuses_a_missing_cursor(tmp_path) -> None:
|
||||
"""A vanished cursor must not silently reset pagination to page one.
|
||||
|
||||
Replaying a prefix may be harmless only because the idempotency key exists;
|
||||
the discovery contract itself must fail closed. A caller that deleted the
|
||||
anchor or restored an older database must restart from a fresh scan, not
|
||||
have the primitive quietly change its place.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path)
|
||||
first = _put(store, "m1")
|
||||
second = _put(store, "m2")
|
||||
|
||||
first_page = queue.unsubmitted_evidence_ids(limit=1)
|
||||
assert first_page == (first,)
|
||||
with sqlite3.connect(path) as connection, connection:
|
||||
connection.execute("DELETE FROM evidence_origins WHERE evidence_id=?", (first,))
|
||||
connection.execute("DELETE FROM evidence WHERE evidence_id=?", (first,))
|
||||
|
||||
with pytest.raises(RuntimeError, match="cursor is unavailable"):
|
||||
queue.unsubmitted_evidence_ids(limit=10, after_cursor=first)
|
||||
|
||||
# A deliberate fresh pass still sees the remaining orphan.
|
||||
assert queue.unsubmitted_evidence_ids(limit=10) == (second,)
|
||||
|
||||
|
||||
def test_unsubmitted_evidence_query_uses_the_reference_index(tmp_path) -> None:
|
||||
"""The anti-join must not scan every queue payload.
|
||||
|
||||
At production scale the queue can keep many terminal rows, so discovery
|
||||
uses a dedicated Evidence reference and an indexed `NOT EXISTS` join
|
||||
instead of loading every job's payload or reference JSON.
|
||||
"""
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.memory_server.migrations import memory_write_jobs_sqlite_schema
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
with main_database(path).write(autocommit=True) as connection, connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE evidence (evidence_id TEXT PRIMARY KEY, imported_at TEXT NOT NULL)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE INDEX evidence_imported_order ON evidence(imported_at, evidence_id)"
|
||||
)
|
||||
for statement in memory_write_jobs_sqlite_schema():
|
||||
connection.execute(statement)
|
||||
plan = [
|
||||
str(row[3])
|
||||
for row in connection.execute(
|
||||
"EXPLAIN QUERY PLAN SELECT evidence.evidence_id FROM evidence "
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence ref "
|
||||
"WHERE ref.evidence_id=evidence.evidence_id) "
|
||||
"ORDER BY evidence.imported_at, evidence.evidence_id LIMIT ?",
|
||||
(512,),
|
||||
)
|
||||
]
|
||||
|
||||
joined = " | ".join(plan)
|
||||
assert "idx_write_job_evidence_evidence" in joined
|
||||
assert "SCAN memory_write_jobs" not in joined
|
||||
assert "SCAN ref" not in joined
|
||||
|
||||
|
||||
def test_the_shipped_schema_serves_the_reconcile_order(tmp_path) -> None:
|
||||
"""The real schema must not scan+sort Evidence on every discovery page.
|
||||
|
||||
`unsubmitted_evidence_ids` orders by `(imported_at, evidence_id)` with no
|
||||
`user_id` predicate, so `evidence_imported_index` (leading column
|
||||
`user_id`) cannot serve it. Before the reconcile-order index the real plan
|
||||
was `SCAN evidence` + `USE TEMP B-TREE FOR ORDER BY`; a pass over a large
|
||||
corpus paid that on every page. This pins the shipped schema, not a
|
||||
test-local index, so a regression to the old DDL fails here.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore, migrate
|
||||
from one_memory.memory_server.migrations import MIGRATIONS
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
with main_database(path).read() as connection:
|
||||
plan = [
|
||||
str(row[3])
|
||||
for row in connection.execute(
|
||||
"EXPLAIN QUERY PLAN SELECT evidence.evidence_id FROM evidence "
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence ref "
|
||||
"WHERE ref.evidence_id=evidence.evidence_id) "
|
||||
"ORDER BY evidence.imported_at, evidence.evidence_id LIMIT ?",
|
||||
(512,),
|
||||
)
|
||||
]
|
||||
|
||||
joined = " | ".join(plan)
|
||||
assert "evidence_reconcile_order" in joined, joined
|
||||
assert "USE TEMP B-TREE FOR ORDER BY" not in joined, joined
|
||||
# SQLite reports an in-order covering-index walk as `SCAN evidence USING
|
||||
# COVERING INDEX ...`; the failure mode we forbid is a bare table scan with
|
||||
# a temp sort, so require the index to be named and the sort to be gone.
|
||||
assert "USING COVERING INDEX evidence_reconcile_order" in joined, joined
|
||||
|
||||
|
||||
def test_unsubmitted_evidence_returns_empty_when_all_are_queued(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path)
|
||||
evidence_id = _put(store, "m1")
|
||||
queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id="c1",
|
||||
batch_identity=BatchIdentity(internal_session_id="c1", first_round=1, last_round=1),
|
||||
evidence_ids=[evidence_id],
|
||||
idempotency_key="idem-1",
|
||||
)
|
||||
)
|
||||
|
||||
assert queue.unsubmitted_evidence_ids(limit=10) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("limit", [0, -1, True, 1.5, "10"])
|
||||
def test_unsubmitted_evidence_rejects_an_invalid_limit(tmp_path, limit: object) -> None:
|
||||
queue = DurableWriteQueue(tmp_path / "one-memory.sqlite3")
|
||||
with pytest.raises(ValueError, match="limit must be positive"):
|
||||
queue.unsubmitted_evidence_ids(limit=limit) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cursor", ["", 1, True, ["evidence-id"], object()])
|
||||
def test_unsubmitted_evidence_rejects_an_invalid_cursor(tmp_path, cursor: object) -> None:
|
||||
"""A malformed cursor must fail with the public contract, not SQL errors.
|
||||
|
||||
``after_cursor`` crosses an API/worker boundary and is allowed to be
|
||||
absent, but every present value must be a non-empty evidence id. Passing
|
||||
a list/object into the lookup produced a raw SQLite binding error; an
|
||||
empty string performed a pointless lookup and then reported a missing
|
||||
cursor as if the database had lost a real row. The primitive should
|
||||
reject the caller's input before touching SQL.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
with sqlite3.connect(path) as connection, connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE evidence (evidence_id TEXT PRIMARY KEY, imported_at TEXT NOT NULL)"
|
||||
)
|
||||
queue = DurableWriteQueue(path)
|
||||
|
||||
with pytest.raises(ValueError, match="after_cursor must be a non-empty string"):
|
||||
queue.unsubmitted_evidence_ids(after_cursor=cursor) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_reconcile_pass_enqueues_an_orphan_exactly_once(tmp_path) -> None:
|
||||
"""The acceptance's "enqueue once" half, on the C2-owned primitives.
|
||||
|
||||
A crash between Evidence commit and submit leaves an orphan. The reconcile
|
||||
flow is: discover it, then submit a write with a stable idempotency key
|
||||
derived from the Evidence identity. A second pass must find nothing (the
|
||||
row now references it) and must not create a duplicate — the same
|
||||
idempotency key would in any case return the existing job.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path)
|
||||
orphan = _put(store, "m1") # evidence committed, no submit -> the crash window
|
||||
|
||||
def reconcile_once() -> str | None:
|
||||
ids = queue.unsubmitted_evidence_ids(limit=64)
|
||||
if not ids:
|
||||
return None
|
||||
first = ids[0]
|
||||
record = queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id="c1",
|
||||
batch_identity=BatchIdentity(internal_session_id="c1", first_round=1, last_round=1),
|
||||
evidence_ids=[first],
|
||||
# Derived from the Evidence identity: a replay re-derives the
|
||||
# same key, so the queue refuses to create a second job.
|
||||
idempotency_key=f"reconcile:{first}",
|
||||
)
|
||||
)
|
||||
return record.job_id
|
||||
|
||||
first_job = reconcile_once()
|
||||
assert first_job is not None
|
||||
|
||||
# Second pass: the orphan is gone, and a forced replay is idempotent.
|
||||
assert queue.unsubmitted_evidence_ids(limit=64) == ()
|
||||
replay = queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id="c1",
|
||||
batch_identity=BatchIdentity(internal_session_id="c1", first_round=1, last_round=1),
|
||||
evidence_ids=[orphan],
|
||||
idempotency_key=f"reconcile:{orphan}",
|
||||
)
|
||||
)
|
||||
assert replay.job_id == first_job
|
||||
assert replay.enqueued is False
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM memory_write_jobs").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_two_reconcile_workers_submit_the_same_orphan_once(tmp_path) -> None:
|
||||
"""A restart and a periodic pass may race; the derived key is the arbiter.
|
||||
|
||||
The discovery cursor cannot promise a cross-call snapshot, so two workers
|
||||
can both observe the same orphan. They must not create two jobs:
|
||||
`reconcile:<evidence_id>` is derived from the Evidence identity and the
|
||||
queue's UNIQUE idempotency key makes the second submit an idempotent replay.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
DurableWriteQueue(path)
|
||||
orphan = _put(store, "m1")
|
||||
|
||||
def reconcile_worker() -> str:
|
||||
request = WriteRequest(
|
||||
internal_session_id="c1",
|
||||
batch_identity=BatchIdentity(internal_session_id="c1", first_round=1, last_round=1),
|
||||
evidence_ids=[orphan],
|
||||
idempotency_key=f"reconcile:{orphan}",
|
||||
)
|
||||
return DurableWriteQueue(path).submit(request).job_id
|
||||
|
||||
first = reconcile_worker()
|
||||
second = reconcile_worker()
|
||||
|
||||
assert first == second
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM memory_write_jobs").fetchone()[0] == 1
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT COUNT(*) FROM memory_write_job_evidence WHERE evidence_id=?",
|
||||
(orphan,),
|
||||
).fetchone()[0]
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_a_sigkill_before_submit_leaves_an_orphan_the_primitive_recovers(tmp_path) -> None:
|
||||
"""Real process crash between Evidence commit and submit (objective 4.1-4.2).
|
||||
|
||||
The child writes Evidence and is SIGKILLed before it can submit the write.
|
||||
No task/queue row exists, so only a *discovery over committed rows* can find
|
||||
it. This is the boundary the C1 file-level drain does not cover for manual
|
||||
notes; the primitive + derived key then enqueue it exactly once.
|
||||
"""
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
if sys.platform == "win32":
|
||||
pytest.skip("POSIX process-kill recovery")
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
EvidenceStore(path)
|
||||
child = (
|
||||
"import time\n"
|
||||
"from datetime import UTC, datetime\n"
|
||||
"from pathlib import Path\n"
|
||||
"from one_memory.imports.models import ("
|
||||
"EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy)\n"
|
||||
"from one_memory.imports.storage import EvidenceStore\n"
|
||||
"path = Path(" + repr(str(path)) + ")\n"
|
||||
"store = EvidenceStore(path)\n"
|
||||
"store.put(EvidenceInput(source_id='s1', conversation_id='c1', message_id='m1',\n"
|
||||
" role='user', written_by=WrittenBy.PERSON, verbatim='orphan',\n"
|
||||
" created_at=datetime.now(UTC), origin_meta={EvidenceOrigin.FILE: OriginMetadata()}))\n"
|
||||
"print('written', flush=True)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", child],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
cwd=str(Path(__file__).resolve().parents[1]),
|
||||
env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")},
|
||||
)
|
||||
assert process.stdout is not None
|
||||
assert process.stdout.readline().strip() == "written"
|
||||
os.kill(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=10)
|
||||
|
||||
queue = DurableWriteQueue(path)
|
||||
ids = queue.unsubmitted_evidence_ids(limit=10)
|
||||
assert len(ids) == 1, "the committed Evidence must be discoverable after the crash"
|
||||
queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id="c1",
|
||||
batch_identity=BatchIdentity(internal_session_id="c1", first_round=1, last_round=1),
|
||||
evidence_ids=[ids[0]],
|
||||
idempotency_key=f"reconcile:{ids[0]}",
|
||||
)
|
||||
)
|
||||
assert queue.unsubmitted_evidence_ids(limit=10) == ()
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM memory_write_jobs").fetchone()[0] == 1
|
||||
@@ -0,0 +1,131 @@
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_governance import MemoryDraft, MemoryGovernanceStore, MemoryKind
|
||||
from tests.person_door import as_person
|
||||
|
||||
|
||||
def test_permission_guard_is_rechecked_inside_write_transaction_and_rolls_back(tmp_path):
|
||||
database = tmp_path / "test.sqlite"
|
||||
evidence = EvidenceStore(database)
|
||||
original = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="fixture",
|
||||
conversation_id="one",
|
||||
message_id="one",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
verbatim="synthetic",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
store = MemoryGovernanceStore(database)
|
||||
attempts = []
|
||||
|
||||
def guard():
|
||||
attempts.append("check")
|
||||
if len(attempts) == 2:
|
||||
raise RuntimeError("permission revoked before commit")
|
||||
|
||||
with pytest.raises(RuntimeError, match="revoked before commit"), store.guard_writes(guard):
|
||||
as_person(store).create(
|
||||
MemoryDraft(
|
||||
user_id="local",
|
||||
kind=MemoryKind.CHAT,
|
||||
content="synthetic result",
|
||||
evidence_ids=(original.evidence_id,),
|
||||
scopes=("all-apps",),
|
||||
),
|
||||
request_id="attempt",
|
||||
)
|
||||
assert attempts == ["check", "check"]
|
||||
assert store.list(user_id="local") == []
|
||||
with sqlite3.connect(database) as db:
|
||||
assert db.execute("SELECT count(*) FROM governed_memory_mutations").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_guard_context_does_not_leak_into_subsequent_manual_actions(tmp_path):
|
||||
database = tmp_path / "test.sqlite"
|
||||
evidence = EvidenceStore(database)
|
||||
original = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="fixture",
|
||||
conversation_id="one",
|
||||
message_id="one",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
verbatim="synthetic",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
store = MemoryGovernanceStore(database)
|
||||
|
||||
def denied():
|
||||
raise RuntimeError("denied")
|
||||
|
||||
with store.guard_writes(denied):
|
||||
assert store.list(user_id="local") == []
|
||||
result = as_person(store).create(
|
||||
MemoryDraft(
|
||||
user_id="local",
|
||||
kind=MemoryKind.CHAT,
|
||||
content="manual action",
|
||||
evidence_ids=(original.evidence_id,),
|
||||
scopes=("private",),
|
||||
),
|
||||
request_id="manual",
|
||||
)
|
||||
assert result.content == "manual action"
|
||||
|
||||
|
||||
def test_guard_check_and_commit_hold_database_write_authority(tmp_path):
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
database = tmp_path / "test.sqlite"
|
||||
evidence = EvidenceStore(database)
|
||||
record = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="fixture",
|
||||
conversation_id="one",
|
||||
message_id="one",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
verbatim="synthetic",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
store = MemoryGovernanceStore(database)
|
||||
with sqlite3.connect(database) as db:
|
||||
db.execute("CREATE TABLE permission_fixture (revoked INTEGER)")
|
||||
db.execute("INSERT INTO permission_fixture VALUES (0)")
|
||||
checks = []
|
||||
|
||||
def revoke():
|
||||
with sqlite3.connect(database, timeout=0) as db:
|
||||
db.execute("UPDATE permission_fixture SET revoked=1")
|
||||
|
||||
def guard():
|
||||
with (
|
||||
ThreadPoolExecutor(max_workers=1) as pool,
|
||||
pytest.raises(sqlite3.OperationalError, match="locked"),
|
||||
):
|
||||
pool.submit(revoke).result(timeout=2)
|
||||
checks.append("write-locked")
|
||||
|
||||
with store.guard_writes(guard):
|
||||
result = as_person(store).create(
|
||||
MemoryDraft(
|
||||
user_id="local",
|
||||
kind=MemoryKind.CHAT,
|
||||
content="permitted",
|
||||
evidence_ids=(record.evidence_id,),
|
||||
scopes=("private",),
|
||||
),
|
||||
request_id="permission-checked",
|
||||
)
|
||||
assert checks == ["write-locked", "write-locked"]
|
||||
revoke()
|
||||
assert store.get(result.memory_id, user_id="local") is not None
|
||||
@@ -0,0 +1,100 @@
|
||||
"""The recall read path must not acquire the single global writer.
|
||||
|
||||
`MemoryGovernanceStore.find_confirmed_matching_terms` is the recall read path.
|
||||
It calls `_refresh_content_shadow`, which opened `_connection()` with the
|
||||
default `writing=True`. That means every recall read checked out the ONE global
|
||||
writer connection for a check that changes nothing on a migrated database —
|
||||
serializing unrelated readers behind the writer and inflating `writer_waits`.
|
||||
|
||||
The absent-shadow case must still fail closed (`governance_schema_missing`),
|
||||
and it must do so without taking the writer: the LIKE fallback already handles
|
||||
a missing shadow, and migration 32 — not this read path — owns the table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_governance import HyCreate, MemoryDraft, MemoryGovernanceStore
|
||||
|
||||
|
||||
def _store(tmp_path):
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
store = MemoryGovernanceStore(database)
|
||||
with store._connection() as connection:
|
||||
store._ensure_content_shadow(connection, backfill=False)
|
||||
evidence_id = (
|
||||
EvidenceStore(database)
|
||||
.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
conversation_id="conversation",
|
||||
message_id="m1",
|
||||
correlation_id="m1",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
verbatim="发布窗口是周五上午。",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
)
|
||||
.record.evidence_id
|
||||
)
|
||||
return store, database, evidence_id
|
||||
|
||||
|
||||
def test_a_recall_read_does_not_check_out_the_global_writer(tmp_path):
|
||||
store, database, evidence_id = _store(tmp_path)
|
||||
store.write_from_hy(
|
||||
HyCreate(
|
||||
draft=MemoryDraft(
|
||||
user_id="local",
|
||||
kind="schema",
|
||||
content="发布窗口是周五上午",
|
||||
scopes=("all-apps",),
|
||||
evidence_ids=(evidence_id,),
|
||||
),
|
||||
request_id="r1",
|
||||
memory_id="m1",
|
||||
),
|
||||
user_id="local",
|
||||
derivation=store.hy_derivation(),
|
||||
)
|
||||
db = main_database(database)
|
||||
# Warm once so any lazy one-time initialization is not attributed to the read.
|
||||
store.find_confirmed_matching_terms(user_id="local", terms=["发布"], limit=10)
|
||||
before = db.stats()
|
||||
store.find_confirmed_matching_terms(user_id="local", terms=["发布"], limit=10)
|
||||
after = db.stats()
|
||||
assert after.reader_checkouts - before.reader_checkouts >= 1
|
||||
assert after.writer_checkouts - before.writer_checkouts == 0, (
|
||||
"the recall read path must not take the single global writer"
|
||||
)
|
||||
|
||||
|
||||
def test_a_missing_shadow_still_fails_closed_without_taking_the_writer(tmp_path):
|
||||
store, database, _evidence_id = _store(tmp_path)
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.execute("DROP TABLE governed_memories_terms")
|
||||
connection.execute("DROP TABLE governed_memories_terms_meta")
|
||||
db = main_database(database)
|
||||
before = db.stats()
|
||||
with pytest.raises(Exception, match="governance_schema_missing"):
|
||||
store.find_confirmed_matching_terms(user_id="local", terms=["发布"], limit=10)
|
||||
after = db.stats()
|
||||
assert after.writer_checkouts - before.writer_checkouts == 0, (
|
||||
"a broken schema must fail closed on a read connection, not write"
|
||||
)
|
||||
with sqlite3.connect(database) as connection:
|
||||
names = {
|
||||
row[0]
|
||||
for row in connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name LIKE 'governed_memories_terms%'"
|
||||
)
|
||||
}
|
||||
assert names == set(), "a read must not repair or recreate the shadow tables"
|
||||
@@ -17,18 +17,13 @@ from pydantic import SecretStr
|
||||
|
||||
from one_memory.api import create_app
|
||||
from one_memory.config import Settings
|
||||
from one_memory.hy_sdk_runtime import build_sdk_config
|
||||
from one_memory.hy_memory_bridge import build_hy_memory_config
|
||||
from one_memory.model_profiles import (
|
||||
ModelCapabilities,
|
||||
ModelProfileInput,
|
||||
ProviderKind,
|
||||
)
|
||||
from tests.graph_fixtures import kuzu_graph_with_embedding_width
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _enable_graph_store(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("ONE_MEMORY_GRAPH_STORE", "kuzu")
|
||||
from tests.test_hy_memory_bridge import _kuzu_graph_with_embedding_width
|
||||
|
||||
|
||||
def _app(tmp_path: Path):
|
||||
@@ -85,10 +80,10 @@ def _activate_builtin_profile(app):
|
||||
|
||||
def _plant_wide_graph(app, data_dir: Path) -> Path:
|
||||
snapshot = _activate_builtin_profile(app)
|
||||
config = build_sdk_config(snapshot, api_key=None, data_dir=data_dir)
|
||||
config = build_hy_memory_config(snapshot, api_key=None, data_dir=data_dir)
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
return db_path
|
||||
|
||||
|
||||
|
||||
@@ -15,18 +15,18 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.hy_graph_admin import (
|
||||
from one_memory.hy_memory_bridge import (
|
||||
GraphWidthDecision,
|
||||
apply_graph_width_decision,
|
||||
build_hy_memory_config,
|
||||
inspect_graph_width_mismatch,
|
||||
)
|
||||
from one_memory.hy_sdk_runtime import build_sdk_config
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
from tests.graph_fixtures import kuzu_graph_with_embedding_width, model_snapshot
|
||||
from tests.test_hy_memory_bridge import _kuzu_graph_with_embedding_width, snapshot
|
||||
|
||||
|
||||
def _narrow_profile() -> ModelProfileSnapshot:
|
||||
base = model_snapshot(
|
||||
base = snapshot(
|
||||
ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
has_api_key=False,
|
||||
@@ -38,10 +38,10 @@ def _mismatched(tmp_path: Path) -> tuple[object, Path, Path]:
|
||||
"""A 768 graph under a data dir, against a 512 profile."""
|
||||
data_dir = tmp_path / "data"
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=data_dir)
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=data_dir)
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
return config, db_path, data_dir
|
||||
|
||||
|
||||
|
||||
@@ -13,21 +13,22 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.hy_graph_admin import (
|
||||
from one_memory.hy_memory_bridge import (
|
||||
GraphWidthDecision,
|
||||
GraphWidthMismatchError,
|
||||
apply_graph_width_decision,
|
||||
build_hy_memory_config,
|
||||
create_hy_memory_client,
|
||||
inspect_graph_width_mismatch,
|
||||
)
|
||||
from one_memory.hy_sdk_runtime import acquire_client, build_sdk_config
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
from tests.graph_fixtures import kuzu_graph_with_embedding_width, model_snapshot
|
||||
from tests.test_hy_memory_bridge import _kuzu_graph_with_embedding_width, snapshot
|
||||
|
||||
|
||||
def _narrow_profile(**overrides: object) -> ModelProfileSnapshot:
|
||||
# Width alone is what matters here. The built-in provider arrives in S1.3;
|
||||
# this guard has to hold for any 512-dim profile against a 768 graph.
|
||||
base = model_snapshot(
|
||||
base = snapshot(
|
||||
ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
has_api_key=False,
|
||||
@@ -44,10 +45,10 @@ def test_a_768_graph_is_not_silently_archived_when_switching_to_512(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=tmp_path)
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
|
||||
mismatch = inspect_graph_width_mismatch(config)
|
||||
|
||||
@@ -62,13 +63,13 @@ def test_creating_a_client_against_a_mismatched_graph_raises_instead_of_retiring
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=tmp_path)
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
|
||||
with pytest.raises(GraphWidthMismatchError) as caught:
|
||||
acquire_client(profile, api_key=None, data_dir=tmp_path)
|
||||
create_hy_memory_client(profile, api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert caught.value.mismatch.stored_width == 768
|
||||
assert caught.value.mismatch.expected_width == 512
|
||||
@@ -80,10 +81,10 @@ def test_keep_existing_leaves_the_graph_and_refuses_the_new_width(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=tmp_path)
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
|
||||
outcome = apply_graph_width_decision(config, GraphWidthDecision.KEEP_EXISTING)
|
||||
|
||||
@@ -97,10 +98,10 @@ def test_rebuild_archives_the_graph_and_lets_the_new_width_proceed(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=tmp_path)
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
|
||||
outcome = apply_graph_width_decision(config, GraphWidthDecision.REBUILD_WITH_NEW)
|
||||
|
||||
@@ -113,10 +114,10 @@ def test_rebuild_archives_the_graph_and_lets_the_new_width_proceed(
|
||||
|
||||
def test_export_first_leaves_the_graph_untouched(tmp_path: Path) -> None:
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=tmp_path / "data")
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path / "data")
|
||||
db_path = Path(config.graph_store.db_path)
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
kuzu_graph_with_embedding_width(db_path, 768)
|
||||
_kuzu_graph_with_embedding_width(db_path, 768)
|
||||
|
||||
outcome = apply_graph_width_decision(
|
||||
config,
|
||||
@@ -134,6 +135,6 @@ def test_export_first_leaves_the_graph_untouched(tmp_path: Path) -> None:
|
||||
|
||||
def test_a_fresh_install_with_no_graph_raises_nothing(tmp_path: Path) -> None:
|
||||
profile = _narrow_profile()
|
||||
config = build_sdk_config(profile, api_key=None, data_dir=tmp_path)
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert inspect_graph_width_mismatch(config) is None
|
||||
@@ -0,0 +1,282 @@
|
||||
"""R-07: HY's coding-shaped replies read as "no memories at all".
|
||||
|
||||
`search_ids` reads `memories` and `_listed_memories` reads `vdb.memories`.
|
||||
HY has a second reply shape — `{"scene": "productivity", "coding_memories":
|
||||
[...]}` (`hy_memory/client.py:1145`, `client.py:1038`) — and neither parser
|
||||
recognises it, so it comes back as an empty tuple. Empty is indistinguishable
|
||||
from "this person has no memories", which is the failure this codebase
|
||||
already paid for once on the profile path: see `_basic_info_memories`, which
|
||||
raises on a payload it does not recognise for exactly that reason.
|
||||
|
||||
Read from HY's source, the requirement's diagnosis does not hold: the coding
|
||||
path is not reachable from here at all, in either direction. Both preconditions
|
||||
are pinned below as green controls, because they are what makes this fix a
|
||||
guard rather than a repair — and they are the two things that would have to
|
||||
change before a coding memory could ever be recalled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from one_memory.hy_memory_bridge import (
|
||||
HyMemoryBridge,
|
||||
HyMemoryBridgeError,
|
||||
HyMemoryTurn,
|
||||
_listed_memories,
|
||||
build_hy_memory_config,
|
||||
)
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
CODING_SEARCH_REPLY: Mapping[str, object] = {
|
||||
"request_id": "req-1",
|
||||
"scene": "productivity",
|
||||
"coding_memories": [
|
||||
{"memory_id": "coding-1", "content": "用户在 recall.py 里用过滤下推替换了内存过滤"},
|
||||
{"memory_id": "coding-2", "content": "用户的分页上限是 500"},
|
||||
],
|
||||
"elapsed_ms": 12.0,
|
||||
}
|
||||
|
||||
CODING_LIST_REPLY: Mapping[str, object] = {
|
||||
"scene": "productivity",
|
||||
"coding_memories": [
|
||||
{"memory_id": "coding-1", "agent_id": "workspace-a"},
|
||||
{"memory_id": "coding-2", "agent_id": "workspace-b"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class CodingSceneClient:
|
||||
"""A client that answers in HY's coding shape rather than the chat shape."""
|
||||
|
||||
def __init__(self, *, search_reply=CODING_SEARCH_REPLY, list_reply=CODING_LIST_REPLY):
|
||||
self.search_reply = search_reply
|
||||
self.list_reply = list_reply
|
||||
|
||||
def search(self, query: str, **_kwargs) -> Mapping[str, object]:
|
||||
return self.search_reply
|
||||
|
||||
def list_memories(self, **_kwargs) -> Mapping[str, object]:
|
||||
return self.list_reply
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _profile() -> ModelProfileSnapshot:
|
||||
return ModelProfileSnapshot(
|
||||
profile_id="profile",
|
||||
revision=1,
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="memory-model",
|
||||
embedding_model="embedding-model",
|
||||
embedding_provider_kind=None,
|
||||
embedding_base_url=None,
|
||||
embedding_dimensions=768,
|
||||
timeout_seconds=30,
|
||||
max_concurrency=1,
|
||||
context_window=8192,
|
||||
has_api_key=False,
|
||||
)
|
||||
|
||||
|
||||
def _bridge(client, tmp_path) -> HyMemoryBridge:
|
||||
return HyMemoryBridge(
|
||||
profile=_profile(),
|
||||
api_key=None,
|
||||
data_dir=tmp_path,
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def test_a_coding_shaped_search_reply_is_not_read_as_no_memories(tmp_path) -> None:
|
||||
bridge = _bridge(CodingSceneClient(), tmp_path)
|
||||
|
||||
found = asyncio.run(bridge.search_ids(query="过滤下推", user_id="local", agent_id="*"))
|
||||
|
||||
assert found == ("coding-1", "coding-2")
|
||||
|
||||
|
||||
def test_a_coding_shaped_listing_is_not_read_as_no_scopes(tmp_path) -> None:
|
||||
bridge = _bridge(CodingSceneClient(), tmp_path)
|
||||
|
||||
scopes = asyncio.run(bridge.list_agent_ids(user_id="local"))
|
||||
|
||||
assert scopes == ("workspace-a", "workspace-b")
|
||||
|
||||
|
||||
def test_an_unrecognised_search_reply_is_an_error_not_an_empty_recall(tmp_path) -> None:
|
||||
"""A shape we do not understand must not be reported as an empty library.
|
||||
|
||||
Returning `()` here is what made a coding reply look like "you have no
|
||||
memories". Whatever HY adds next, the honest answer is that we could not
|
||||
read it.
|
||||
"""
|
||||
bridge = _bridge(CodingSceneClient(search_reply={"request_id": "req-2"}), tmp_path)
|
||||
|
||||
with pytest.raises(HyMemoryBridgeError):
|
||||
asyncio.run(bridge.search_ids(query="过滤下推", user_id="local", agent_id="*"))
|
||||
|
||||
|
||||
def test_an_unrecognised_listing_is_an_error_too(tmp_path) -> None:
|
||||
bridge = _bridge(CodingSceneClient(list_reply={"unexpected": {}}), tmp_path)
|
||||
|
||||
with pytest.raises(HyMemoryBridgeError):
|
||||
asyncio.run(bridge.list_agent_ids(user_id="local"))
|
||||
|
||||
|
||||
#: What HY 0.1.6.5 actually answers a chat-path search with
|
||||
#: (`client.py:2378-2389`): `memories` is three channels, not a list. Our
|
||||
#: parser required a list, so every semantic search read as zero hits.
|
||||
THREE_CHANNEL_REPLY: Mapping[str, object] = {
|
||||
"request_id": "req-3",
|
||||
"memories": {
|
||||
"schema": [
|
||||
{"memory_id": "schema-1", "score": 0.61, "layer": "L0_BASIC_INFO"},
|
||||
],
|
||||
"proactive": [
|
||||
{"memory_id": "intention-1", "score": 0.44, "layer": "L7_INTENTION"},
|
||||
],
|
||||
"normal": [
|
||||
{"memory_id": "fact-1", "score": 0.88, "layer": "L2_FACT"},
|
||||
{"memory_id": "fact-2", "score": 0.52, "layer": "L2_FACT"},
|
||||
],
|
||||
},
|
||||
"rewrite": {"used": False, "scene": "normal"},
|
||||
"elapsed_ms": 41.0,
|
||||
}
|
||||
|
||||
#: The failure branch (`client.py:2253`), which is the shape this machine has
|
||||
#: actually been getting: same three channels, all empty. HY logs
|
||||
#: "[search] failed" and returns this.
|
||||
FAILED_SEARCH_REPLY: Mapping[str, object] = {
|
||||
"request_id": "req-4",
|
||||
"memories": {"schema": [], "proactive": [], "normal": []},
|
||||
"elapsed_ms": 3.0,
|
||||
}
|
||||
|
||||
|
||||
def test_the_three_channels_hy_actually_returns_are_all_read(tmp_path) -> None:
|
||||
"""The real defect behind R-07, and it is not about coding at all.
|
||||
|
||||
`memories` is a dict of schema / proactive / normal channels in the current
|
||||
HY SDK; our parser asked `isinstance(memories, list)` and returned an empty
|
||||
tuple otherwise. So semantic recall was dead — including the schema channel,
|
||||
which is the whole reason
|
||||
`schema_min_score` and `schema_limit` are passed in.
|
||||
"""
|
||||
bridge = _bridge(CodingSceneClient(search_reply=THREE_CHANNEL_REPLY), tmp_path)
|
||||
|
||||
found = asyncio.run(bridge.search_ids(query="花名", user_id="local", agent_id="*"))
|
||||
|
||||
assert set(found) == {"fact-1", "schema-1", "fact-2", "intention-1"}
|
||||
|
||||
|
||||
def test_the_channels_are_merged_in_hys_own_order_not_ours(tmp_path) -> None:
|
||||
"""Recall ranks by the order these ids come back in, so the order matters.
|
||||
|
||||
Bucketing into channels loses HY's ranking, but every item still carries
|
||||
the `score` HY ranked it by — a field we were also dropping. Sorting the
|
||||
union by it restores HY's judgement instead of substituting a channel
|
||||
preference of our own, which would be a retrieval-quality decision and
|
||||
not ours to make here.
|
||||
"""
|
||||
bridge = _bridge(CodingSceneClient(search_reply=THREE_CHANNEL_REPLY), tmp_path)
|
||||
|
||||
found = asyncio.run(bridge.search_ids(query="花名", user_id="local", agent_id="*"))
|
||||
|
||||
assert found == ("fact-1", "schema-1", "fact-2", "intention-1")
|
||||
|
||||
|
||||
def test_search_scores_carries_hys_relevance(tmp_path) -> None:
|
||||
"""D102: the score is what lets recall blend the two lanes by relevance."""
|
||||
bridge = _bridge(CodingSceneClient(search_reply=THREE_CHANNEL_REPLY), tmp_path)
|
||||
|
||||
found = asyncio.run(bridge.search_scores(query="花名", user_id="local", agent_id="*"))
|
||||
|
||||
assert found == (
|
||||
("fact-1", 0.88),
|
||||
("schema-1", 0.61),
|
||||
("fact-2", 0.52),
|
||||
("intention-1", 0.44),
|
||||
)
|
||||
|
||||
|
||||
def test_a_failed_hy_search_is_not_an_error_here_but_is_still_empty(tmp_path) -> None:
|
||||
"""HY reports search failure only in its log, and returns empty channels.
|
||||
|
||||
On this machine every real search has been `[search] failed: Connection
|
||||
error.` — the embedding endpoint is unreachable — and the reply is
|
||||
indistinguishable from a genuine zero-hit search. We cannot tell them
|
||||
apart at this seam, so this must not become a hard error; it is recorded
|
||||
as an upstream gap instead.
|
||||
"""
|
||||
bridge = _bridge(CodingSceneClient(search_reply=FAILED_SEARCH_REPLY), tmp_path)
|
||||
|
||||
assert asyncio.run(bridge.search_ids(query="花名", user_id="local", agent_id="*")) == ()
|
||||
|
||||
|
||||
def test_the_chat_shape_still_reads_the_same(tmp_path) -> None:
|
||||
"""The control: the shape we actually get today must not move.
|
||||
|
||||
Green before and after. Every real reply on this machine is this one, so
|
||||
a change here would be the fix breaking recall to satisfy a shape we have
|
||||
never seen.
|
||||
"""
|
||||
chat_client = CodingSceneClient(
|
||||
search_reply={"memories": [{"memory_id": "chat-1"}, {"id": "chat-2"}]},
|
||||
list_reply={"vdb": {"memories": [{"agent_id": "one-memory"}]}},
|
||||
)
|
||||
bridge = _bridge(chat_client, tmp_path)
|
||||
|
||||
assert asyncio.run(bridge.search_ids(query="花名", user_id="local", agent_id="*")) == (
|
||||
"chat-1",
|
||||
"chat-2",
|
||||
)
|
||||
assert asyncio.run(bridge.list_agent_ids(user_id="local")) == ("one-memory",)
|
||||
assert _listed_memories({"vdb": {"memories": []}}) == ()
|
||||
|
||||
|
||||
def test_an_empty_chat_reply_is_still_just_empty(tmp_path) -> None:
|
||||
"""No memories yet is not a shape error, and must stay quiet."""
|
||||
bridge = _bridge(
|
||||
CodingSceneClient(search_reply={"memories": []}, list_reply={"vdb": {"memories": []}}),
|
||||
tmp_path,
|
||||
)
|
||||
|
||||
assert asyncio.run(bridge.search_ids(query="花名", user_id="local", agent_id="*")) == ()
|
||||
assert asyncio.run(bridge.list_agent_ids(user_id="local")) == ()
|
||||
|
||||
|
||||
def test_a_tool_message_can_reach_hy_so_it_can_judge_the_route_itself() -> None:
|
||||
"""HY only considers its coding writer when the messages contain a
|
||||
tool-role message (`has_any_tool_message`). Kristin's call: send the turn
|
||||
as it happened, tool calls included, and let HY decide."""
|
||||
assert HyMemoryTurn(role="tool", content="[Bash] ls -la").role == "tool"
|
||||
with pytest.raises(ValidationError):
|
||||
HyMemoryTurn(role="system", content="hidden")
|
||||
|
||||
|
||||
def test_our_search_call_shape_cannot_reach_hys_coding_path(tmp_path) -> None:
|
||||
"""Why no coding reply can arrive today — the read side.
|
||||
|
||||
HY decides the scene per search (`client.py:2020-2116`): an explicit
|
||||
`scene`, or an LLM judge when query rewrite is on, or the multi-`queries`
|
||||
judge. Otherwise a single query defaults to `normal`. We pass one query,
|
||||
no scene, and leave rewrite at HY's default of off, so the coding branch
|
||||
is unreachable — `config.coding.enable` being true only pays for the
|
||||
store, not the routing.
|
||||
|
||||
Turning any of that on costs an LLM call per search and, with the store
|
||||
empty, buys nothing. Left as Kristin's call rather than switched on here.
|
||||
"""
|
||||
config = build_hy_memory_config(_profile(), api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert config.coding.enable is True
|
||||
assert config.llm.enable_query_rewrite is False
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import logging
|
||||
|
||||
from one_memory.hy_runtime_config import configure_hy_runtime_environment
|
||||
from one_memory.hy_memory_bridge import configure_hy_memory_environment
|
||||
|
||||
|
||||
def test_the_extractors_parse_failures_pass_through_unchanged(tmp_path):
|
||||
@@ -19,7 +19,7 @@ def test_the_extractors_parse_failures_pass_through_unchanged(tmp_path):
|
||||
original_handlers = list(logger.handlers)
|
||||
logger.handlers = [capture]
|
||||
logger.setLevel(logging.WARNING)
|
||||
configure_hy_runtime_environment(tmp_path)
|
||||
configure_hy_memory_environment(tmp_path)
|
||||
try:
|
||||
secret = "x" * 5000
|
||||
logger.warning(
|
||||
@@ -52,7 +52,7 @@ def test_other_warnings_pass_through_unchanged(tmp_path):
|
||||
original_handlers = list(logger.handlers)
|
||||
logger.handlers = [capture]
|
||||
logger.setLevel(logging.WARNING)
|
||||
configure_hy_runtime_environment(tmp_path)
|
||||
configure_hy_memory_environment(tmp_path)
|
||||
try:
|
||||
logger.warning("[coding-extract] judge failed, fallback to chat: %s", "reason")
|
||||
finally:
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,113 @@
|
||||
"""What the pinned public SDK's MetricsCollector does, and what that costs us.
|
||||
|
||||
These integration tests ask whether the pending-task warning is a leak that
|
||||
grows per request, and whether reusing one client is enough to avoid it.
|
||||
Reading the warning alone cannot answer either question, so the tests measure it.
|
||||
|
||||
The answer that came out is narrower and worse than "a leak": the collector is
|
||||
a process singleton, so the two loops start once rather than once per client --
|
||||
but the flag that says "already started" is sticky and there is no stop. Once
|
||||
the loop those tasks were created on is gone, a later loop gets nothing, and
|
||||
flush and cleanup are silently dead for the rest of the process.
|
||||
|
||||
That makes reusing the client necessary but not sufficient: what has to survive
|
||||
is the event loop. These tests pin both halves so an SDK upgrade that fixes it
|
||||
shows up as a failure here instead of going unnoticed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("hy_memory")
|
||||
|
||||
|
||||
def background_loop_count() -> int:
|
||||
return sum(
|
||||
1
|
||||
for task in asyncio.all_tasks()
|
||||
if task.get_coro() is not None
|
||||
and (task.get_coro().__qualname__ or "").startswith("MetricsCollector._")
|
||||
)
|
||||
|
||||
|
||||
def test_the_collector_is_one_per_process_not_one_per_client() -> None:
|
||||
"""If it were per-client, reusing the client would be the whole fix."""
|
||||
from hy_memory.metrics import MetricsCollector
|
||||
|
||||
assert MetricsCollector.get() is MetricsCollector.get()
|
||||
|
||||
|
||||
def test_metrics_collector_can_stop_and_restart_on_a_second_event_loop(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""The current SDK owns an explicit lifecycle and restarts cleanly."""
|
||||
from hy_memory.metrics import MetricsCollector
|
||||
|
||||
# A fresh collector, so an earlier test in the session cannot decide this.
|
||||
monkeypatch.setattr(MetricsCollector, "_instance", None)
|
||||
collector = MetricsCollector.get()
|
||||
|
||||
async def cycle() -> tuple[int, int]:
|
||||
await collector.start_background_tasks()
|
||||
started = background_loop_count()
|
||||
await collector.stop_background_tasks()
|
||||
return started, background_loop_count()
|
||||
|
||||
first = asyncio.run(cycle())
|
||||
second = asyncio.run(cycle())
|
||||
|
||||
assert first == (2, 0)
|
||||
assert second == (2, 0)
|
||||
|
||||
|
||||
def test_the_runtime_reuses_one_client_per_profile_revision(tmp_path: Path) -> None:
|
||||
"""Our side of the workaround: never build a client per request.
|
||||
|
||||
A new client per request would re-enter HY initialization every time; the
|
||||
collector would survive as a singleton but everything else would not.
|
||||
"""
|
||||
from one_memory.memory_processing import MemoryProcessingSupervisor
|
||||
|
||||
built = []
|
||||
|
||||
def factory(profile, secret, data_dir):
|
||||
built.append((profile.profile_id, profile.revision))
|
||||
return object()
|
||||
|
||||
resolver = MemoryProcessingSupervisor._bridge
|
||||
queue = object.__new__(MemoryProcessingSupervisor)
|
||||
queue._bridges = {}
|
||||
queue.bridge_factory = factory
|
||||
|
||||
profile = _snapshot()
|
||||
first = resolver(queue, profile, None, tmp_path)
|
||||
again = resolver(queue, profile, None, tmp_path)
|
||||
newer = resolver(queue, profile.model_copy(update={"revision": 2}), None, tmp_path)
|
||||
|
||||
assert first is again, "a second request built a second client"
|
||||
assert newer is not first, "a new revision reused the old revision's client"
|
||||
assert built == [("profile", 1), ("profile", 2)]
|
||||
|
||||
|
||||
def _snapshot():
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
return ModelProfileSnapshot(
|
||||
profile_id="profile",
|
||||
revision=1,
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="memory-model",
|
||||
embedding_model="builtin",
|
||||
embedding_provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
embedding_base_url=None,
|
||||
embedding_dimensions=384,
|
||||
timeout_seconds=30,
|
||||
max_concurrency=1,
|
||||
context_window=8192,
|
||||
has_api_key=False,
|
||||
)
|
||||
@@ -1,42 +1,45 @@
|
||||
"""Public local-embedding injection must not create network egress."""
|
||||
"""Where a real HY client is allowed to send bytes, and where it is not.
|
||||
|
||||
The integration test goes beyond checking whether the config names a local
|
||||
host. It verifies that a live HY instance doing real work opens no socket to
|
||||
an external endpoint the user did not configure. SDK defaults can cover model,
|
||||
graph, vector, and metrics services; a config assertion cannot rule out a code
|
||||
path that reads those defaults directly.
|
||||
|
||||
So these tests watch the socket layer itself while a real client indexes and
|
||||
searches. The first test is the control: it proves the watcher notices a
|
||||
connection at all, because a watcher that sees nothing everywhere would pass
|
||||
this file without meaning anything.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
from one_memory.hy_sdk_runtime import LocalOnnxEmbeddingService, build_sdk_config
|
||||
import pytest
|
||||
|
||||
from one_memory.hy_memory_bridge import (
|
||||
HyMemoryBridge,
|
||||
build_hy_memory_config,
|
||||
)
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
|
||||
class _Watcher:
|
||||
def __init__(self, monkeypatch) -> None:
|
||||
self.targets: list[str] = []
|
||||
real_connect = socket.socket.connect
|
||||
|
||||
def connect(sock, address, *args, **kwargs):
|
||||
self.targets.append(str(address))
|
||||
return real_connect(sock, address, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(socket.socket, "connect", connect)
|
||||
|
||||
|
||||
class _LocalEmbedder:
|
||||
def encode(self, texts: list[str]) -> list[list[float]]:
|
||||
return [[1.0, 0.0] for _text in texts]
|
||||
|
||||
|
||||
def _profile() -> ModelProfileSnapshot:
|
||||
def local_onnx_profile() -> ModelProfileSnapshot:
|
||||
"""A profile whose embedder runs in-process, so search needs no provider."""
|
||||
return ModelProfileSnapshot(
|
||||
profile_id="egress",
|
||||
profile_id="egress-integration",
|
||||
revision=1,
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="memory-model",
|
||||
embedding_provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
embedding_model="builtin",
|
||||
embedding_dimensions=512,
|
||||
embedding_provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
embedding_base_url=None,
|
||||
embedding_dimensions=384,
|
||||
timeout_seconds=30,
|
||||
max_concurrency=1,
|
||||
context_window=8192,
|
||||
@@ -44,8 +47,44 @@ def _profile() -> ModelProfileSnapshot:
|
||||
)
|
||||
|
||||
|
||||
class Watcher:
|
||||
"""Record every address the process tries to reach, and let it through.
|
||||
|
||||
Recording rather than denying keeps the failure legible: a denied socket
|
||||
turns into whatever error the caller invents for it, while a recorded one
|
||||
names the host in the assertion.
|
||||
"""
|
||||
|
||||
def __init__(self, monkeypatch) -> None:
|
||||
self.targets: list[str] = []
|
||||
real_connect = socket.socket.connect
|
||||
real_resolve = socket.getaddrinfo
|
||||
|
||||
def connect(sock, address, *args, **kwargs):
|
||||
self.targets.append(self._name(address))
|
||||
return real_connect(sock, address, *args, **kwargs)
|
||||
|
||||
def resolve(host, port, *args, **kwargs):
|
||||
self.targets.append(f"{host}:{port}")
|
||||
return real_resolve(host, port, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(socket.socket, "connect", connect)
|
||||
monkeypatch.setattr(socket, "getaddrinfo", resolve)
|
||||
|
||||
@staticmethod
|
||||
def _name(address: object) -> str:
|
||||
if isinstance(address, tuple) and len(address) >= 2:
|
||||
return f"{address[0]}:{address[1]}"
|
||||
return str(address)
|
||||
|
||||
@property
|
||||
def hosts(self) -> set[str]:
|
||||
return {target.rsplit(":", 1)[0] for target in self.targets}
|
||||
|
||||
|
||||
def test_the_watcher_reports_a_connection_that_really_happens(monkeypatch) -> None:
|
||||
watcher = _Watcher(monkeypatch)
|
||||
"""Control. Without this, an empty recording proves nothing."""
|
||||
watcher = Watcher(monkeypatch)
|
||||
listener = socket.socket()
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
listener.listen(1)
|
||||
@@ -57,33 +96,121 @@ def test_the_watcher_reports_a_connection_that_really_happens(monkeypatch) -> No
|
||||
finally:
|
||||
listener.close()
|
||||
|
||||
assert watcher.targets
|
||||
assert "127.0.0.1" in watcher.hosts
|
||||
|
||||
|
||||
def test_public_local_embedding_service_opens_no_socket(monkeypatch) -> None:
|
||||
watcher = _Watcher(monkeypatch)
|
||||
service = LocalOnnxEmbeddingService(_LocalEmbedder()) # type: ignore[arg-type]
|
||||
|
||||
assert asyncio.run(service.embed("长期偏好")) == [1.0, 0.0]
|
||||
assert watcher.targets == []
|
||||
|
||||
|
||||
def test_no_ambient_remote_backend_survives_sdk_config(
|
||||
def test_a_real_search_with_the_built_in_embedder_reaches_nothing(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Search is local when the embedder is local, or the claim is wrong.
|
||||
|
||||
A miss here means HY contacted an external endpoint the user never
|
||||
configured, which is the integration behavior that config assertions
|
||||
alone cannot verify.
|
||||
"""
|
||||
pytest.importorskip("hy_memory")
|
||||
from one_memory.embedding_assets import assets_present
|
||||
|
||||
if not assets_present():
|
||||
pytest.skip("built-in encoder weights are not installed")
|
||||
|
||||
embedded = _count_local_embeds(monkeypatch)
|
||||
watcher = Watcher(monkeypatch)
|
||||
asyncio.run(_search(local_onnx_profile(), tmp_path))
|
||||
|
||||
# An empty recording is only evidence if the search really ran. The
|
||||
# embedder loads lazily, so "reached nothing" and "did nothing" look the
|
||||
# same from the socket layer until we confirm a vector was produced.
|
||||
assert embedded, "the search never got as far as embedding the query"
|
||||
# Loopback to the configured Ollama port would still be a finding here:
|
||||
# search declares itself model-free, so it must not call the provider.
|
||||
assert watcher.targets == [], watcher.targets
|
||||
|
||||
|
||||
def test_the_watcher_catches_a_remote_embedder_on_that_same_path(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Control: the SDK's remote embedding leaf is observable by the watcher.
|
||||
|
||||
An empty memory store may short-circuit search before embedding, so the
|
||||
control calls the same configured EmbedService network leaf directly.
|
||||
"""
|
||||
pytest.importorskip("hy_memory")
|
||||
from hy_memory.core.embed_service import EmbedService
|
||||
|
||||
remote = local_onnx_profile().model_copy(
|
||||
update={
|
||||
"embedding_provider_kind": ProviderKind.OPENAI_COMPATIBLE,
|
||||
"embedding_base_url": "http://127.0.0.1:59/v1",
|
||||
"embedding_model": "remote-embedder",
|
||||
"has_api_key": True,
|
||||
}
|
||||
)
|
||||
config = build_hy_memory_config(remote, api_key="test-key", data_dir=tmp_path)
|
||||
service = EmbedService(config)
|
||||
watcher = Watcher(monkeypatch)
|
||||
with contextlib.suppress(Exception):
|
||||
asyncio.run(service.embed("synthetic remote embedding", use_cache=False))
|
||||
|
||||
assert "127.0.0.1" in watcher.hosts, watcher.targets
|
||||
|
||||
|
||||
def _count_local_embeds(monkeypatch) -> list[int]:
|
||||
"""A truthy list once the in-process encoder has produced a vector."""
|
||||
from one_memory.hy_memory_bridge import LocalHyEmbedService
|
||||
|
||||
seen: list[int] = []
|
||||
original = LocalHyEmbedService.embed
|
||||
|
||||
async def counted(self, text, use_cache=True):
|
||||
vector = await original(self, text, use_cache)
|
||||
seen.append(len(vector))
|
||||
return vector
|
||||
|
||||
monkeypatch.setattr(LocalHyEmbedService, "embed", counted)
|
||||
return seen
|
||||
|
||||
|
||||
async def _search(
|
||||
profile: ModelProfileSnapshot,
|
||||
data_dir: Path,
|
||||
*,
|
||||
api_key: str | None = None,
|
||||
) -> None:
|
||||
bridge = HyMemoryBridge(profile=profile, api_key=api_key, data_dir=data_dir)
|
||||
try:
|
||||
await bridge.search_ids("长期偏好", user_id="local", agent_id="*")
|
||||
finally:
|
||||
await bridge.close()
|
||||
|
||||
|
||||
def test_no_internal_gateway_or_server_default_survives_our_config(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""The SDK's shipped defaults must not be reachable through our config.
|
||||
|
||||
Ambient SDK configuration may name remote stores and endpoints. We pin the
|
||||
OpenAI-compatible provider and local stores, so none may survive.
|
||||
"""
|
||||
for name, value in (
|
||||
("MEMORY_LLM_PROVIDER", "openai"),
|
||||
("MEMORY_LLM_BASE_URL", "https://ambient.example/v1"),
|
||||
("MEMORY_GRAPH_PROVIDER", "memgraph"),
|
||||
("MEMGRAPH_URL", "bolt://memgraph.example.com:7687"),
|
||||
("MEMORY_VECTOR_PROVIDER", "qdrant"),
|
||||
("MEMORY_API_HOST", "0.0.0.0"),
|
||||
):
|
||||
monkeypatch.setenv(name, value)
|
||||
|
||||
config = build_sdk_config(_profile(), api_key=None, data_dir=tmp_path)
|
||||
rendered = repr(config).lower()
|
||||
config = build_hy_memory_config(local_onnx_profile(), api_key=None, data_dir=tmp_path)
|
||||
|
||||
rendered = repr(config)
|
||||
for forbidden in ("ambient.example", "memgraph", "qdrant", "bolt://"):
|
||||
assert forbidden not in rendered
|
||||
assert forbidden not in rendered.lower(), forbidden
|
||||
assert config.llm.provider == "openai"
|
||||
assert config.vector_store.provider == "chroma"
|
||||
assert config.graph_store.provider == "kuzu"
|
||||
assert str(tmp_path) in config.graph_store.db_path
|
||||
@@ -0,0 +1,518 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.diagnostic_log import configure_diagnostics, reset_diagnostics
|
||||
from one_memory.hy_response_guard import (
|
||||
HyReconcileGuard,
|
||||
HyResponseGuard,
|
||||
install_hy_write_guards,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", ["empty", "length"])
|
||||
def test_invalid_response_retries_same_hy_provider_without_changing_prompt(invalid):
|
||||
class Provider:
|
||||
calls = []
|
||||
|
||||
async def complete(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
return SimpleNamespace(
|
||||
content="" if invalid == "empty" and len(self.calls) == 1 else "[]",
|
||||
finish_reason="length" if invalid == "length" and len(self.calls) == 1 else "stop",
|
||||
tool_calls=None,
|
||||
)
|
||||
|
||||
provider = Provider()
|
||||
response = asyncio.run(
|
||||
HyResponseGuard(provider).complete(prompt="native prompt", max_tokens=2000)
|
||||
)
|
||||
assert response.content == "[]"
|
||||
assert len(provider.calls) == 2
|
||||
assert all(call["prompt"] == "native prompt" for call in provider.calls)
|
||||
|
||||
|
||||
def test_repeated_empty_response_fails_instead_of_becoming_no_op():
|
||||
class Provider:
|
||||
calls = 0
|
||||
|
||||
async def complete_messages(self, **kwargs):
|
||||
self.calls += 1
|
||||
return SimpleNamespace(content="", finish_reason="stop", tool_calls=None)
|
||||
|
||||
provider = Provider()
|
||||
with pytest.raises(ValueError, match="hy_memory_model_empty_response"):
|
||||
asyncio.run(HyResponseGuard(provider).complete_messages(messages=[]))
|
||||
assert provider.calls == 2
|
||||
|
||||
|
||||
def test_failure_receipt_survives_a_later_successful_model_call():
|
||||
class Provider:
|
||||
succeed = False
|
||||
|
||||
async def complete_messages(self, **kwargs):
|
||||
return SimpleNamespace(
|
||||
content="{}" if self.succeed else "", finish_reason="stop", tool_calls=None
|
||||
)
|
||||
|
||||
provider = Provider()
|
||||
guard = HyResponseGuard(provider)
|
||||
with pytest.raises(ValueError, match="hy_memory_model_empty_response"):
|
||||
asyncio.run(guard.complete_messages(messages=[]))
|
||||
provider.succeed = True
|
||||
asyncio.run(guard.complete_messages(messages=[]))
|
||||
assert guard.failures == ["hy_memory_model_empty_response"]
|
||||
|
||||
|
||||
def test_provider_exception_is_recorded_even_if_hy_later_swallows_it():
|
||||
class Provider:
|
||||
async def complete_messages(self, **kwargs):
|
||||
raise RuntimeError("provider unavailable")
|
||||
|
||||
guard = HyResponseGuard(Provider())
|
||||
with pytest.raises(RuntimeError):
|
||||
asyncio.run(guard.complete_messages(messages=[]))
|
||||
assert guard.failures == ["hy_memory_model_request_failed"]
|
||||
|
||||
|
||||
def test_payment_error_is_recorded_without_retrying_or_leaking_provider_details():
|
||||
class PaymentError(RuntimeError):
|
||||
status_code = 402
|
||||
|
||||
class Provider:
|
||||
calls = 0
|
||||
|
||||
async def complete_messages(self, **kwargs):
|
||||
self.calls += 1
|
||||
raise PaymentError("Private provider account details")
|
||||
|
||||
provider = Provider()
|
||||
guard = HyResponseGuard(provider)
|
||||
with pytest.raises(PaymentError):
|
||||
asyncio.run(guard.complete_messages(messages=[]))
|
||||
assert provider.calls == 1
|
||||
assert guard.failures == ["hy_memory_model_payment_required"]
|
||||
|
||||
|
||||
def observe_provider_failure(error, *, method="complete_messages"):
|
||||
class Provider:
|
||||
calls = 0
|
||||
|
||||
async def complete(self, *args, **kwargs):
|
||||
self.calls += 1
|
||||
raise error
|
||||
|
||||
complete_messages = complete
|
||||
|
||||
provider = Provider()
|
||||
guard = HyResponseGuard(provider)
|
||||
with pytest.raises(type(error)) as raised:
|
||||
asyncio.run(getattr(guard, method)("private prompt", api_key="private key"))
|
||||
assert raised.value is error
|
||||
assert provider.calls == 1
|
||||
return guard
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["complete", "complete_messages"])
|
||||
@pytest.mark.parametrize("location", ["direct", "response", "attempts"])
|
||||
@pytest.mark.parametrize(
|
||||
"status,code",
|
||||
[
|
||||
(401, "hy_memory_model_authentication_failed"),
|
||||
(402, "hy_memory_model_payment_required"),
|
||||
(403, "hy_memory_model_access_denied"),
|
||||
(429, "hy_memory_model_rate_limited"),
|
||||
(500, "hy_memory_model_service_unavailable"),
|
||||
(599, "hy_memory_model_service_unavailable"),
|
||||
(408, "hy_memory_model_timeout"),
|
||||
(400, "hy_memory_model_request_incompatible"),
|
||||
(404, "hy_memory_model_request_incompatible"),
|
||||
(422, "hy_memory_model_request_incompatible"),
|
||||
],
|
||||
)
|
||||
def test_provider_status_is_classified_without_retry_or_private_logging(
|
||||
status, code, location, method, caplog
|
||||
):
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
private = "private account key prompt response"
|
||||
if location == "attempts":
|
||||
error = LLMRetryError(
|
||||
private,
|
||||
[
|
||||
{"status_code": 401, "error": private},
|
||||
{"status_code": status, "error_type": private, "response_body": private},
|
||||
],
|
||||
)
|
||||
else:
|
||||
error = RuntimeError(private)
|
||||
if location == "direct":
|
||||
error.status_code = status
|
||||
else:
|
||||
error.response = SimpleNamespace(status_code=status, text=private)
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
guard = observe_provider_failure(error, method=method)
|
||||
assert guard.failures == [code]
|
||||
assert caplog.messages == [f"{code}: status_code={status}"]
|
||||
assert "private" not in caplog.text
|
||||
assert all(record.exc_info is None and record.stack_info is None for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", ["direct", "response", "attempts"])
|
||||
@pytest.mark.parametrize("status", [None, True, False, "402", 402.0, -1, 600, 200, 302, 423])
|
||||
def test_missing_invalid_or_unmapped_status_is_generic(location, status, caplog):
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
if location == "attempts":
|
||||
# A valid earlier failure must not override the final unknown result.
|
||||
error = LLMRetryError("private status=402", [{"status_code": 402}, {"status_code": status}])
|
||||
else:
|
||||
error = RuntimeError("private status=402")
|
||||
if location == "direct":
|
||||
error.status_code = status
|
||||
else:
|
||||
error.response = SimpleNamespace(status_code=status)
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
guard = observe_provider_failure(error)
|
||||
assert guard.failures == ["hy_memory_model_request_failed"]
|
||||
expected_status = status if type(status) is int and 100 <= status <= 599 else "unknown"
|
||||
assert caplog.messages == [f"hy_memory_model_request_failed: status_code={expected_status}"]
|
||||
assert "private" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attempts", [None, [], {}, "402", [None], ["402"], [{"error": "402"}]])
|
||||
def test_malformed_attempts_do_not_mask_original_exception(attempts, caplog):
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
error = LLMRetryError("private message", attempts)
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
assert observe_provider_failure(error).failures == ["hy_memory_model_request_failed"]
|
||||
assert caplog.messages == ["hy_memory_model_request_failed: status_code=unknown"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"direct,response,expected",
|
||||
[
|
||||
(401, 429, "authentication_failed"),
|
||||
(None, 429, "rate_limited"),
|
||||
("401", "429", "payment_required"),
|
||||
],
|
||||
)
|
||||
def test_status_lookup_precedence_is_direct_then_response_then_final_attempt(
|
||||
direct, response, expected
|
||||
):
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
error = LLMRetryError("private message", [{"status_code": 402}])
|
||||
error.status_code = direct
|
||||
error.response = SimpleNamespace(status_code=response)
|
||||
assert observe_provider_failure(error).failures == [f"hy_memory_model_{expected}"]
|
||||
|
||||
|
||||
def test_failure_diagnostics_never_stringify_exception_or_illegal_status(caplog):
|
||||
class PrivateError(RuntimeError):
|
||||
def __str__(self):
|
||||
pytest.fail("Diagnostics must not stringify the provider exception")
|
||||
|
||||
class PrivateStatus:
|
||||
def __str__(self):
|
||||
pytest.fail("Diagnostics must not stringify an untrusted status")
|
||||
|
||||
error = PrivateError()
|
||||
error.status_code = PrivateStatus()
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
assert observe_provider_failure(error).failures == ["hy_memory_model_request_failed"]
|
||||
assert caplog.messages == ["hy_memory_model_request_failed: status_code=unknown"]
|
||||
|
||||
|
||||
def test_status_attribute_failure_preserves_original_provider_exception(caplog):
|
||||
class BrokenStatusError(RuntimeError):
|
||||
@property
|
||||
def status_code(self):
|
||||
raise ValueError("private attribute failure")
|
||||
|
||||
error = BrokenStatusError("private provider message")
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
assert observe_provider_failure(error).failures == ["hy_memory_model_request_failed"]
|
||||
assert caplog.messages == ["hy_memory_model_request_failed: status_code=unknown"]
|
||||
|
||||
|
||||
def test_integer_subclass_is_not_a_strict_http_status(caplog):
|
||||
class Status(int):
|
||||
pass
|
||||
|
||||
error = RuntimeError("private message")
|
||||
error.status_code = Status(402)
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
assert observe_provider_failure(error).failures == ["hy_memory_model_request_failed"]
|
||||
assert caplog.messages == ["hy_memory_model_request_failed: status_code=unknown"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["complete", "complete_messages"])
|
||||
def test_terminal_sanitized_client_timeout_is_not_an_invented_http_status(method, caplog):
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
error = LLMRetryError("private error", [{"failure_kind": "timeout", "error": "private"}])
|
||||
caplog.set_level(logging.WARNING, logger="one_memory.hy_response_guard")
|
||||
assert observe_provider_failure(error, method=method).failures == ["hy_memory_model_timeout"]
|
||||
assert caplog.messages == ["hy_memory_model_timeout: status_code=unknown"]
|
||||
assert "private" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", [None, "connection", "unknown", "timeout private", True, {}, []])
|
||||
def test_only_final_exact_timeout_category_is_used(kind, caplog):
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
error = LLMRetryError("private error", [{"failure_kind": "timeout"}, {"failure_kind": kind}])
|
||||
assert observe_provider_failure(error).failures == ["hy_memory_model_request_failed"]
|
||||
assert "private" not in caplog.text
|
||||
|
||||
|
||||
def test_real_http_status_takes_precedence_over_client_timeout_kind():
|
||||
from hy_memory.agent.llm_provider import LLMRetryError
|
||||
|
||||
error = LLMRetryError("private error", [{"failure_kind": "timeout", "status_code": 402}])
|
||||
assert observe_provider_failure(error).failures == ["hy_memory_model_payment_required"]
|
||||
|
||||
|
||||
def test_valid_empty_json_array_is_not_confused_with_a_transport_empty_response():
|
||||
class Provider:
|
||||
calls = 0
|
||||
|
||||
async def complete(self, **kwargs):
|
||||
self.calls += 1
|
||||
return SimpleNamespace(content="[]", finish_reason="stop", tool_calls=None)
|
||||
|
||||
provider = Provider()
|
||||
assert asyncio.run(HyResponseGuard(provider).complete(prompt="native")).content == "[]"
|
||||
assert provider.calls == 1
|
||||
|
||||
|
||||
def test_installation_reaches_ordinary_and_ultra_inner_writers():
|
||||
from one_memory.hy_write_verification import HyExtractionGuard
|
||||
|
||||
def native_writer():
|
||||
native = SimpleNamespace()
|
||||
agent = SimpleNamespace(llm_provider=native, extractor=SimpleNamespace(llm=native))
|
||||
reconciler = SimpleNamespace(_get_llm=lambda: native)
|
||||
return SimpleNamespace(mem_agent=agent, reconciler=reconciler), agent, reconciler
|
||||
|
||||
ordinary, ordinary_agent, ordinary_reconciler = native_writer()
|
||||
ultra_inner, ultra_agent, ultra_reconciler = native_writer()
|
||||
|
||||
class Registry:
|
||||
async def get_writer(self):
|
||||
return ordinary
|
||||
|
||||
async def get_system2_writer(self):
|
||||
return SimpleNamespace(_writer=ultra_inner)
|
||||
|
||||
client = SimpleNamespace(_registry=Registry(), _loop_thread=SimpleNamespace(run=asyncio.run))
|
||||
install_hy_write_guards(client)
|
||||
for writer, agent, reconciler in (
|
||||
(ordinary, ordinary_agent, ordinary_reconciler),
|
||||
(ultra_inner, ultra_agent, ultra_reconciler),
|
||||
):
|
||||
assert isinstance(agent.extractor.llm, HyResponseGuard)
|
||||
assert agent.llm_provider is agent.extractor.llm
|
||||
assert isinstance(agent.extractor, HyExtractionGuard)
|
||||
assert agent.extractor in client._one_memory_write_guards
|
||||
assert isinstance(reconciler._llm, HyResponseGuard)
|
||||
assert type(writer._reconciler).__name__ == "HyReconcileGuard"
|
||||
|
||||
|
||||
def test_installed_guards_label_model_calls_by_hy_stage(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
|
||||
class Provider:
|
||||
async def complete(self, **kwargs):
|
||||
return SimpleNamespace(content="[]", finish_reason="stop", tool_calls=None)
|
||||
|
||||
def native_writer():
|
||||
native = Provider()
|
||||
agent = SimpleNamespace(llm_provider=native, extractor=SimpleNamespace(llm=native))
|
||||
reconciler = SimpleNamespace(_get_llm=lambda: native)
|
||||
return SimpleNamespace(mem_agent=agent, reconciler=reconciler), agent, reconciler
|
||||
|
||||
ordinary, ordinary_agent, ordinary_reconciler = native_writer()
|
||||
ultra_inner, _ultra_agent, _ultra_reconciler = native_writer()
|
||||
|
||||
class Registry:
|
||||
async def get_writer(self):
|
||||
return ordinary
|
||||
|
||||
async def get_system2_writer(self):
|
||||
return SimpleNamespace(_writer=ultra_inner)
|
||||
|
||||
client = SimpleNamespace(_registry=Registry(), _loop_thread=SimpleNamespace(run=asyncio.run))
|
||||
install_hy_write_guards(client)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
asyncio.run(ordinary_agent.llm_provider.complete(prompt="private-extract"))
|
||||
asyncio.run(ordinary_reconciler._llm.complete(prompt="private-reconcile"))
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
written = receipt.path.read_text()
|
||||
phases = [json.loads(line)["details"]["phase"] for line in written.splitlines()]
|
||||
assert phases == ["extract", "reconcile"]
|
||||
assert "private-" not in written
|
||||
|
||||
|
||||
def test_model_call_log_records_stage_and_duration_without_prompt(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
|
||||
class Provider:
|
||||
async def complete(self, *, prompt, **kwargs):
|
||||
return SimpleNamespace(content="[]", finish_reason="stop", tool_calls=None)
|
||||
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
asyncio.run(HyResponseGuard(Provider()).complete(prompt="private-prompt-marker"))
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
written = receipt.path.read_text()
|
||||
event = json.loads(written)
|
||||
assert event["event"] == "hy.model_call.completed"
|
||||
assert event["details"] == {"phase": "extract", "attempt": 1, "call_count": 1}
|
||||
assert event["duration_ms"] >= 0
|
||||
assert "private-prompt-marker" not in written
|
||||
|
||||
|
||||
def test_reconcile_log_counts_operations_without_content(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
|
||||
class Reconciler:
|
||||
async def reconcile(self, **kwargs):
|
||||
return SimpleNamespace(
|
||||
success=True,
|
||||
ops=[
|
||||
SimpleNamespace(op="ADD", owner="user", content="private-a", memory_id=None),
|
||||
SimpleNamespace(op="UPDATE", owner="user", content="private-b", memory_id=None),
|
||||
SimpleNamespace(
|
||||
op="SUPERSEDE", owner="user", content="private-c", memory_id=None
|
||||
),
|
||||
SimpleNamespace(op="DELETE", owner="user", content="private-d", memory_id=None),
|
||||
],
|
||||
)
|
||||
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
asyncio.run(HyReconcileGuard(Reconciler()).reconcile())
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
written = receipt.path.read_text()
|
||||
event = json.loads(written)
|
||||
assert event["event"] == "hy.reconcile.completed"
|
||||
assert event["details"] == {
|
||||
"created_count": 1,
|
||||
"updated_count": 1,
|
||||
"superseded_count": 1,
|
||||
"deleted_count": 1,
|
||||
"result_count": 4,
|
||||
}
|
||||
assert "private-a" not in written
|
||||
|
||||
|
||||
def test_native_no_candidate_reconcile_retains_extractor_owner(tmp_path, monkeypatch):
|
||||
from hy_memory.agent.reconciler import MemoryReconciler
|
||||
from hy_memory.config import MemoryConfig
|
||||
from hy_memory.pipelines._retrieval.reconcile_retrieval import ReconcileHybridRetriever
|
||||
|
||||
from one_memory.hy_response_guard import HyReconcileGuard
|
||||
|
||||
class Embeddings:
|
||||
async def embed_batch(self, texts):
|
||||
return [[1.0, 0.0] for _ in texts]
|
||||
|
||||
class Store:
|
||||
async def get_embeddings(self, identifiers):
|
||||
return {}
|
||||
|
||||
async def no_candidates(*args, **kwargs):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(ReconcileHybridRetriever, "search_candidates", no_candidates)
|
||||
native = MemoryReconciler(MemoryConfig())
|
||||
native._enable_search_query = False
|
||||
arguments = {
|
||||
"new_memories": ["Agent建议启用备份,用户尚未确认。"],
|
||||
"user_id": "test",
|
||||
"agent_id": "test",
|
||||
"embed_service": Embeddings(),
|
||||
"vector_store": Store(),
|
||||
"new_memories_with_meta": [
|
||||
{"content": "Agent建议启用备份,用户尚未确认。", "owner": "agent"}
|
||||
],
|
||||
}
|
||||
result = asyncio.run(HyReconcileGuard(native).reconcile(**arguments))
|
||||
assert result.success and len(result.ops) == 1
|
||||
assert result.ops[0].owner == "agent"
|
||||
assert result.ops[0].content == arguments["new_memories"][0]
|
||||
|
||||
|
||||
def test_report_binding_rejects_a_reconciler_input_not_in_the_supplied_file():
|
||||
from one_memory.hy_response_guard import HyReconcileGuard
|
||||
|
||||
guard = HyReconcileGuard(None)
|
||||
guard.imported_contents = frozenset({"supplied"})
|
||||
with pytest.raises(ValueError, match="import_input_mismatch"):
|
||||
asyncio.run(
|
||||
guard.reconcile(
|
||||
new_memories=["unbound"], new_memories_with_meta=[{"content": "unbound"}]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target_state", ["valid", "missing", "other-user", "other-agent", "read-error"]
|
||||
)
|
||||
def test_update_verification_reads_the_real_hy_store_interface_and_checks_identity(target_state):
|
||||
from hy_memory.agent.reconciler import ReconcileOp, ReconcileResult
|
||||
|
||||
from one_memory.hy_response_guard import HyReconcileGuard
|
||||
|
||||
node = SimpleNamespace(user_id="local", agent_id="atlas", owner="user", content="Keep history.")
|
||||
if target_state == "other-user":
|
||||
node.user_id = "another-user"
|
||||
if target_state == "other-agent":
|
||||
node.agent_id = "another-agent"
|
||||
|
||||
class Store:
|
||||
calls = []
|
||||
|
||||
async def get_by_id(self, key):
|
||||
self.calls.append(key)
|
||||
if target_state == "read-error":
|
||||
raise OSError("private storage details must not reach the receipt")
|
||||
return None if target_state == "missing" else node
|
||||
|
||||
class Native:
|
||||
async def reconcile(self, **kwargs):
|
||||
return ReconcileResult(
|
||||
ops=[
|
||||
ReconcileOp(op="UPDATE", memory_id="prior", owner="user", content=node.content)
|
||||
]
|
||||
)
|
||||
|
||||
def _get_llm(self):
|
||||
pytest.fail("An unchanged, identity-checked source needs no new model call")
|
||||
|
||||
store = Store()
|
||||
guard = HyReconcileGuard(Native())
|
||||
arguments = {"user_id": "local", "agent_id": "atlas", "vector_store": store}
|
||||
if target_state == "valid":
|
||||
assert asyncio.run(guard.reconcile(**arguments)).success
|
||||
assert guard.failures == []
|
||||
else:
|
||||
code = (
|
||||
"hy_memory_write_verification_failed"
|
||||
if target_state == "read-error"
|
||||
else "hy_memory_write_verification_target_invalid"
|
||||
)
|
||||
with pytest.raises(ValueError, match=code):
|
||||
asyncio.run(guard.reconcile(**arguments))
|
||||
assert guard.failures == [code]
|
||||
assert store.calls == ["prior"]
|
||||
@@ -0,0 +1,164 @@
|
||||
"""HY response shape selection only; no provider, write routing or model calls."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridgeError, _replied_memories
|
||||
from tests.test_hy_coding_reply import CodingSceneClient, _bridge
|
||||
|
||||
ERROR = "hy_memory_search_shape_unknown"
|
||||
CODING = {"memory_id": "synthetic-coding", "task": "synthetic task", "solution": "proposal"}
|
||||
CHAT = {"memory_id": "synthetic-chat", "content": "synthetic chat", "score": 0.8}
|
||||
|
||||
|
||||
def groups():
|
||||
return {"schema": [], "proactive": [], "normal": []}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scene", ["productivity", "working"])
|
||||
@pytest.mark.parametrize("chat_has_rows", [False, True])
|
||||
def test_explicit_coding_scene_selects_coding_even_with_chat_groups(scene, chat_has_rows):
|
||||
chat = groups()
|
||||
if chat_has_rows:
|
||||
chat["normal"] = [CHAT]
|
||||
reply = {"scene": scene, "coding_memories": [CODING], "memories": chat}
|
||||
before = deepcopy(reply)
|
||||
result = _replied_memories(reply, ERROR)
|
||||
assert result == (CODING,)
|
||||
assert result[0] is CODING
|
||||
assert reply == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scene", ["normal", "chat"])
|
||||
@pytest.mark.parametrize("grouped", [False, True])
|
||||
def test_explicit_chat_scene_never_merges_other_channel(scene, grouped):
|
||||
chat = {**groups(), "normal": [CHAT]} if grouped else [CHAT]
|
||||
reply = {"scene": scene, "memories": chat, "coding_memories": [CODING]}
|
||||
assert _replied_memories(reply, ERROR) == (CHAT,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reply,expected",
|
||||
[
|
||||
({"memories": [CHAT]}, (CHAT,)),
|
||||
({"memories": []}, ()),
|
||||
({"memories": groups()}, ()),
|
||||
({"coding_memories": [CODING]}, (CODING,)),
|
||||
({"coding_memories": []}, ()),
|
||||
({"scene": "productivity", "coding_memories": []}, ()),
|
||||
({"scene": "normal", "memories": []}, ()),
|
||||
({"scene": "productivity", "coding_memories": [], "memories": [CHAT]}, ()),
|
||||
({"scene": "normal", "memories": [], "coding_memories": [CODING]}, ()),
|
||||
],
|
||||
)
|
||||
def test_legacy_single_channel_and_selected_empty_shapes_remain_valid(reply, expected):
|
||||
assert _replied_memories(reply, ERROR) == expected
|
||||
|
||||
|
||||
def test_all_chat_groups_preserve_rows_and_order_without_reinterpreting_metadata():
|
||||
schema = {"id": "schema", "score": 0.3}
|
||||
proactive = {"id": "proactive", "score": 0.2}
|
||||
reply = {
|
||||
"memories": {"schema": [schema], "proactive": [proactive], "normal": [CHAT]},
|
||||
"rewrite": {"used": False, "scene": "normal"},
|
||||
"elapsed_ms": 1,
|
||||
}
|
||||
assert _replied_memories(reply, ERROR) == (schema, proactive, CHAT)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scene", [None, False, 1, [], {}, "", "other", "coding", "NORMAL", " chat "]
|
||||
)
|
||||
def test_unknown_or_malformed_explicit_scene_never_falls_back(scene):
|
||||
with pytest.raises(HyMemoryBridgeError, match=f"^{ERROR}$"):
|
||||
_replied_memories({"scene": scene, "memories": [CHAT]}, ERROR)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reply",
|
||||
[
|
||||
{"scene": "productivity", "memories": [CHAT]},
|
||||
{"scene": "working", "memories": groups()},
|
||||
{"scene": "normal", "coding_memories": [CODING]},
|
||||
{"scene": "chat", "coding_memories": []},
|
||||
{"memories": [], "coding_memories": [CODING]},
|
||||
{"memories": [CHAT], "coding_memories": []},
|
||||
{"memories": groups(), "coding_memories": []},
|
||||
{"memories": [CHAT], "coding_memories": [CODING]},
|
||||
],
|
||||
)
|
||||
def test_missing_selected_channel_or_ambiguous_unlabelled_channels_are_rejected(reply):
|
||||
with pytest.raises(HyMemoryBridgeError, match=f"^{ERROR}$"):
|
||||
_replied_memories(reply, ERROR)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reply",
|
||||
[
|
||||
None,
|
||||
[],
|
||||
{},
|
||||
{"memories": None},
|
||||
{"memories": {}},
|
||||
{"memories": {"normal": [CHAT]}},
|
||||
{"memories": {**groups(), "unexpected": [CHAT]}},
|
||||
{"memories": {**groups(), "schema": None}},
|
||||
{"memories": {**groups(), "normal": [CHAT, 1]}},
|
||||
{"memories": [CHAT, None]},
|
||||
{"memories": [1]},
|
||||
{"memories": (CHAT,)},
|
||||
{"coding_memories": {"normal": [CODING]}},
|
||||
{"coding_memories": [CODING, False]},
|
||||
{"scene": "productivity", "coding_memories": [1], "memories": groups()},
|
||||
{"scene": "productivity", "coding_memories": [CODING], "memories": {"bad": []}},
|
||||
{"scene": "normal", "memories": [CHAT], "coding_memories": None},
|
||||
],
|
||||
)
|
||||
def test_malformed_channel_or_row_fails_whole_reply_instead_of_losing_rows(reply):
|
||||
with pytest.raises(HyMemoryBridgeError, match=f"^{ERROR}$"):
|
||||
_replied_memories(reply, ERROR)
|
||||
|
||||
|
||||
def test_errors_use_supplied_safe_code_without_echoing_payload():
|
||||
secret_marker = "synthetic-private-payload-must-not-appear"
|
||||
error_code = "hy_memory_list_shape_unknown"
|
||||
with pytest.raises(HyMemoryBridgeError) as raised:
|
||||
_replied_memories({"memories": [CHAT, secret_marker]}, error_code)
|
||||
assert str(raised.value) == error_code
|
||||
assert secret_marker not in repr(raised.value)
|
||||
|
||||
|
||||
def test_documented_dual_reply_reaches_existing_search_and_listing_consumers(tmp_path):
|
||||
row = {**CODING, "agent_id": "synthetic-workspace"}
|
||||
reply: Mapping[str, object] = {
|
||||
"scene": "productivity",
|
||||
"coding_memories": [row],
|
||||
"memories": groups(),
|
||||
}
|
||||
|
||||
class Client(CodingSceneClient):
|
||||
def __init__(self):
|
||||
super().__init__(search_reply=reply, list_reply=reply)
|
||||
self.search_args = []
|
||||
|
||||
def search(self, query, **kwargs):
|
||||
self.search_args.append(kwargs)
|
||||
return self.search_reply
|
||||
|
||||
async def scenario():
|
||||
client = Client()
|
||||
bridge = _bridge(client, tmp_path)
|
||||
try:
|
||||
assert await bridge.search_ids("synthetic query", user_id="local", agent_id="*") == (
|
||||
"synthetic-coding",
|
||||
)
|
||||
assert await bridge.list_agent_ids(user_id="local") == ("synthetic-workspace",)
|
||||
assert client.search_args[0]["scene"] == "normal"
|
||||
assert client.search_args[0]["reader"] == "hybrid"
|
||||
finally:
|
||||
await bridge.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
Loaded 100 of 343 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user