Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d0a069950 | ||
|
|
cdb44658cf | ||
|
|
2ab04291a9 | ||
|
|
80d3b55723 | ||
|
|
b5ec0861f0 | ||
|
|
47e938da1b |
No files matched your search
@@ -40,7 +40,6 @@ MEMORY_EMBEDDING_DIMS=1024
|
||||
# MEMORY_EMBEDDER_TIMEOUT=15
|
||||
# MEMORY_EMBEDDER_MAX_RETRIES=2
|
||||
# MEMORY_EMBEDDER_RETRY_DELAY=0.3
|
||||
# MEMORY_EMBEDDER_BATCH_SIZE=64
|
||||
# MEMORY_EMBEDDER_EXTRA_HEADERS={}
|
||||
# MEMORY_EMBEDDER_EXTRA_BODY={}
|
||||
|
||||
|
||||
@@ -4,46 +4,6 @@ Hy Memory and MemHub are versioned and released independently.
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Hy Memory: new `hy_memory.reembed` module for switching the embedding
|
||||
model. Writers must be paused for the whole run. Nothing is derived from
|
||||
the provider or endpoint: source and target are named explicitly, and the
|
||||
identity of a space is the model name plus the width (`space_identity`).
|
||||
Only Chroma and Kuzu are supported.
|
||||
- `vector_space_names(config, base_name=None)` returns every physical
|
||||
collection a client opens for the six vector roles: memory, entities,
|
||||
coding keys and working facts, each in the working and coding variants.
|
||||
- `reembed_vector_space(source, target, embedding=...)` pages every raw
|
||||
source collection, covering all users and all layers including L1_RAW. It
|
||||
embeds the text the write path embedded (`reembed_text`: the canonical
|
||||
state's `state_retrieval_text`, a schema's `retrieval_text`, otherwise the
|
||||
document). It then writes each record into the target with the same id,
|
||||
document and metadata. Each batch's widths are checked before anything is
|
||||
written. A cancel stops at a batch boundary (`ReembedCancelled`). A rerun
|
||||
skips the ids the target already holds, so it resumes, and a merge never
|
||||
overwrites the target. The source is never written.
|
||||
- `plan_reembed`, `verify_reembed` and `drop_vector_space` complete the
|
||||
vector side. `verify_reembed` checks ids, metadata, per-user counts and
|
||||
the width, and runs a self-recall sample. `drop_vector_space` refuses
|
||||
collections that a live client in this process holds open.
|
||||
- `scan_vector_spaces(persist_directory)` is read-only. It lists every
|
||||
space on disk with its roles, counts, stored width, per-user counts and
|
||||
the models recorded in metadata. `diff_spaces(source, target)` returns the
|
||||
ids that the source holds and the target lacks, whatever their status.
|
||||
- `reembed_graph(source_path, target_path, embedding=..., target_dims=...)`
|
||||
builds a new Kuzu graph at the new width. It copies every node and edge
|
||||
and recomputes `Memory.embedding`, `Memory.beh_embedding`, and the Topic
|
||||
and HierarchyTopic embeddings, then rebuilds the HNSW index. The source
|
||||
graph is opened read-only and kept for rollback. A run that fails or is
|
||||
cancelled leaves no target behind, and a rerun starts over.
|
||||
`verify_graph_reembed` and `drop_graph` complete the graph side.
|
||||
- Hy Memory: `EmbedService.embed_batch` now splits large batches into
|
||||
requests of `embedder.batch_size` texts each (`MEMORY_EMBEDDER_BATCH_SIZE`,
|
||||
default 64); before, it sent them all in one request. On a 429 it waits for
|
||||
the `Retry-After` time plus jitter, capped at 60 s. It takes a per-call
|
||||
`max_retries`. A returned vector whose width differs from the configured
|
||||
`embedding_dims` now raises `EmbeddingDimensionMismatchError` and is not
|
||||
retried.
|
||||
|
||||
## MemHub 1.0.5 — 2026-10-28
|
||||
|
||||
Build 36, on `hy-memory==2.0.8`. Linux x64/arm64 Runtime and macOS arm64
|
||||
|
||||
@@ -48,7 +48,6 @@
|
||||
| `MEMORY_EMBEDDER_TIMEOUT` | `15` | 单次超时,秒 |
|
||||
| `MEMORY_EMBEDDER_MAX_RETRIES` | `2` | 最大重试次数 |
|
||||
| `MEMORY_EMBEDDER_RETRY_DELAY` | `0.3` | 重试间隔,秒 |
|
||||
| `MEMORY_EMBEDDER_BATCH_SIZE` | `64` | `embed_batch` 单次请求最多发送的文本条数,超出按此切块 |
|
||||
| `MEMORY_EMBEDDER_EXTRA_HEADERS` | — | JSON object,附加 HTTP headers |
|
||||
| `MEMORY_EMBEDDER_EXTRA_BODY` | — | JSON object,附加请求 body |
|
||||
|
||||
|
||||
@@ -55,33 +55,6 @@ except Exception:
|
||||
from .client import HyMemoryClient
|
||||
from .inspector import MemoryInspector
|
||||
from .runtime import SharedRuntime
|
||||
from .reembed import (
|
||||
EmbeddingSpaceIdentity,
|
||||
GraphReembedPlan,
|
||||
GraphReembedProgress,
|
||||
GraphReembedReport,
|
||||
GraphVerification,
|
||||
ReembedCancelled,
|
||||
ReembedPlan,
|
||||
ReembedProgress,
|
||||
ReembedReport,
|
||||
ReembedVerification,
|
||||
ScannedSpace,
|
||||
SpaceDiff,
|
||||
VectorSpace,
|
||||
diff_spaces,
|
||||
drop_graph,
|
||||
drop_vector_space,
|
||||
plan_reembed,
|
||||
reembed_graph,
|
||||
reembed_text,
|
||||
reembed_vector_space,
|
||||
scan_vector_spaces,
|
||||
space_identity,
|
||||
vector_space_names,
|
||||
verify_graph_reembed,
|
||||
verify_reembed,
|
||||
)
|
||||
from .snapshot import (
|
||||
DataPlaneRestore,
|
||||
DataPlaneSnapshot,
|
||||
@@ -156,32 +129,6 @@ __all__ = [
|
||||
"DataPlaneSnapshotRejected",
|
||||
"restore_local_data_plane",
|
||||
"snapshot_local_data_plane",
|
||||
# 换 embedding 模型:把向量库和图重新 embed 进新的 embedding 空间
|
||||
"EmbeddingSpaceIdentity",
|
||||
"GraphReembedPlan",
|
||||
"GraphReembedProgress",
|
||||
"GraphReembedReport",
|
||||
"GraphVerification",
|
||||
"ReembedCancelled",
|
||||
"ReembedPlan",
|
||||
"ReembedProgress",
|
||||
"ReembedReport",
|
||||
"ReembedVerification",
|
||||
"ScannedSpace",
|
||||
"SpaceDiff",
|
||||
"VectorSpace",
|
||||
"diff_spaces",
|
||||
"drop_graph",
|
||||
"drop_vector_space",
|
||||
"plan_reembed",
|
||||
"reembed_graph",
|
||||
"reembed_text",
|
||||
"reembed_vector_space",
|
||||
"scan_vector_spaces",
|
||||
"space_identity",
|
||||
"vector_space_names",
|
||||
"verify_graph_reembed",
|
||||
"verify_reembed",
|
||||
# 配置
|
||||
"MemoryConfig",
|
||||
# 公共扩展协议 / 精确读取类型
|
||||
|
||||
@@ -260,7 +260,6 @@ class EmbedderConfig:
|
||||
max_retries: int = None # 最大重试次数
|
||||
retry_delay: float = None # 重试间隔(秒)
|
||||
timeout: int = None # 请求超时(秒)
|
||||
batch_size: int = None # 单次 HTTP 请求最多带多少条 input(embed_batch 按此切块)
|
||||
|
||||
# 内部平台扩展(extra_headers / extra_body 会透传给 OpenAI client)
|
||||
extra_headers: Optional[Dict[str, str]] = None
|
||||
@@ -295,10 +294,6 @@ class EmbedderConfig:
|
||||
# 请求切断并制造重试风暴。SDK 内层重试已关闭,15s 是每个显式 attempt
|
||||
# 的真实上限;部署可通过 MEMORY_EMBEDDER_TIMEOUT 覆盖。
|
||||
self.timeout = _get_env_int("MEMORY_EMBEDDER_TIMEOUT", 15)
|
||||
if self.batch_size is None:
|
||||
# 大批量(如重新 embed 整个库)不能一次 HTTP 发几千条:多数服务对
|
||||
# input 数组有上限,超了直接 400。按 batch_size 切块逐块发送。
|
||||
self.batch_size = _get_env_int("MEMORY_EMBEDDER_BATCH_SIZE", 64)
|
||||
if self.extra_headers is None:
|
||||
val = os.getenv("MEMORY_EMBEDDER_EXTRA_HEADERS")
|
||||
if val:
|
||||
|
||||
+19
-105
@@ -38,12 +38,10 @@ import importlib.util
|
||||
import json
|
||||
from json import JSONDecodeError
|
||||
import logging
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
|
||||
from ..config import MemoryConfig
|
||||
from ..embedding import EmbeddingDimensionMismatchError
|
||||
from .workload import embedding_slot
|
||||
from ..utils.log_setup import get_request_id
|
||||
|
||||
@@ -189,41 +187,6 @@ def _redact_headers(headers: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any
|
||||
return out
|
||||
|
||||
|
||||
# Retry-After 的上限:服务端给出超长等待(或恶意值)时不让单次调用挂几十分钟。
|
||||
_RETRY_AFTER_MAX_S = 60.0
|
||||
|
||||
|
||||
def _retry_after_seconds(exc: Exception) -> Optional[float]:
|
||||
"""从 429 异常的响应头里取 Retry-After(秒数或 HTTP 日期),取不到返回 None。"""
|
||||
response = getattr(exc, "response", None)
|
||||
headers = getattr(response, "headers", None)
|
||||
if not headers:
|
||||
return None
|
||||
try:
|
||||
raw = headers.get("retry-after") or headers.get("Retry-After")
|
||||
except Exception:
|
||||
return None
|
||||
if raw is None:
|
||||
return None
|
||||
raw = str(raw).strip()
|
||||
try:
|
||||
seconds = float(raw)
|
||||
except ValueError:
|
||||
try:
|
||||
from email.utils import parsedate_to_datetime
|
||||
from datetime import datetime, timezone
|
||||
|
||||
when = parsedate_to_datetime(raw)
|
||||
if when.tzinfo is None:
|
||||
when = when.replace(tzinfo=timezone.utc)
|
||||
seconds = (when - datetime.now(timezone.utc)).total_seconds()
|
||||
except Exception:
|
||||
return None
|
||||
if seconds != seconds: # NaN
|
||||
return None
|
||||
return min(max(0.0, seconds), _RETRY_AFTER_MAX_S)
|
||||
|
||||
|
||||
def _embeddings_api_url(base_url: str) -> str:
|
||||
"""OpenAI SDK posts to ``{base_url}/embeddings`` (base_url usually ends with /v1)."""
|
||||
base = (base_url or "").rstrip("/")
|
||||
@@ -242,8 +205,7 @@ class EmbedServiceConfig:
|
||||
api_key: str = ""
|
||||
base_url: str = ""
|
||||
embedding_dims: int = 1536
|
||||
# embed_batch 单次 HTTP 最多带多少条 input;超出按块顺序发送
|
||||
batch_size: int = 64
|
||||
batch_size: int = 100
|
||||
enable_cache: bool = True
|
||||
cache_size: int = 10000
|
||||
max_retries: int = 5
|
||||
@@ -325,7 +287,6 @@ class EmbedService:
|
||||
max_retries=embedder.max_retries if embedder.max_retries is not None else 5,
|
||||
retry_delay=embedder.retry_delay if embedder.retry_delay is not None else 3.0,
|
||||
timeout=embedder.timeout if embedder.timeout is not None else 60,
|
||||
batch_size=max(1, int(getattr(embedder, "batch_size", None) or 64)),
|
||||
queue_batch_size=int(os.getenv("MEMORY_EMBED_BATCH_SIZE", "8")),
|
||||
queue_batch_window_ms=float(os.getenv("MEMORY_EMBED_BATCH_WINDOW_MS", "100")),
|
||||
extra_headers=embedder.extra_headers,
|
||||
@@ -679,8 +640,6 @@ class EmbedService:
|
||||
self,
|
||||
texts: List[str],
|
||||
use_cache: bool = True,
|
||||
*,
|
||||
max_retries: Optional[int] = None,
|
||||
) -> List[List[float]]:
|
||||
"""
|
||||
批量向量化
|
||||
@@ -688,23 +647,17 @@ class EmbedService:
|
||||
Args:
|
||||
texts: 文本列表
|
||||
use_cache: 是否使用缓存
|
||||
max_retries: 本次调用的重试次数;None 用配置值。长批量任务(重新
|
||||
embed 整个库)可以给更宽的重试预算,不影响在线路径。
|
||||
|
||||
Returns:
|
||||
向量列表
|
||||
"""
|
||||
vectors, _metas = await self.embed_batch_with_meta(
|
||||
texts, use_cache=use_cache, max_retries=max_retries,
|
||||
)
|
||||
vectors, _metas = await self.embed_batch_with_meta(texts, use_cache=use_cache)
|
||||
return vectors
|
||||
|
||||
async def embed_batch_with_meta(
|
||||
self,
|
||||
texts: List[str],
|
||||
use_cache: bool = True,
|
||||
*,
|
||||
max_retries: Optional[int] = None,
|
||||
) -> Tuple[List[List[float]], List[Dict[str, Any]]]:
|
||||
"""批量向量化,同时返回每条的 provider metadata(非 vector)。"""
|
||||
if not texts:
|
||||
@@ -729,12 +682,7 @@ class EmbedService:
|
||||
indices_to_embed.append(i)
|
||||
|
||||
if texts_to_embed:
|
||||
# max_retries 只在显式给出时透传:保持 `_embed_texts_with_meta(texts)`
|
||||
# 这个既有调用形状不变(测试与下游替身按它打桩)。
|
||||
retry_kwargs = {} if max_retries is None else {"max_retries": max_retries}
|
||||
vectors, batch_metas = await self._embed_texts_with_meta(
|
||||
texts_to_embed, **retry_kwargs,
|
||||
)
|
||||
vectors, batch_metas = await self._embed_texts_with_meta(texts_to_embed)
|
||||
|
||||
for j, vector in enumerate(vectors):
|
||||
idx = indices_to_embed[j]
|
||||
@@ -752,9 +700,9 @@ class EmbedService:
|
||||
return results, metas
|
||||
|
||||
async def _embed_texts_with_meta(
|
||||
self, texts: List[str], *, max_retries: Optional[int] = None,
|
||||
self, texts: List[str]
|
||||
) -> Tuple[List[List[float]], List[Dict[str, Any]]]:
|
||||
"""实际的向量化调用:**一次 HTTP 带走一块**;返回 (vectors, metas)。
|
||||
"""实际的向量化调用:**一次 HTTP 带走整批**;返回 (vectors, metas)。
|
||||
|
||||
早期实现是 `for text in texts: await _embed_openai([text])` 逐条串行,
|
||||
原意是"避免 batch size 限制",但代价极大:线上一批 8 条 × ~5.6s 串行
|
||||
@@ -771,34 +719,21 @@ class EmbedService:
|
||||
截断在这里做(发送前逐条过 `_truncate_for_embedding`):所有 embedding
|
||||
路径(embed / embed_batch / embed_queued / *_with_meta)最终都落到这个
|
||||
方法 —— 放在各入口会漏。理由与阈值见模块头部 `_EMBED_MAX_TOKENS`。
|
||||
|
||||
切块:一次带走整批只适用于小批。几千条的 input 数组会被多数服务直接
|
||||
400(input 上限),所以按 `batch_size`(默认 64)切块,块之间顺序发送。
|
||||
不超过一块时行为与切块前逐字相同(`batch_mode="single_request"`)。
|
||||
"""
|
||||
if not texts:
|
||||
return [], []
|
||||
|
||||
safe_texts = [_truncate_for_embedding(t) for t in texts]
|
||||
chunk = max(1, int(self._embed_config.batch_size or 64))
|
||||
vectors: List[List[float]] = []
|
||||
vectors, meta = await self._embed_openai(safe_texts)
|
||||
base_meta = dict(meta or {})
|
||||
base_meta["batch_mode"] = "single_request"
|
||||
base_meta["batch_input_count"] = len(safe_texts)
|
||||
|
||||
all_metas: List[Dict[str, Any]] = []
|
||||
chunk_count = (len(safe_texts) + chunk - 1) // chunk
|
||||
for start in range(0, len(safe_texts), chunk):
|
||||
part = safe_texts[start:start + chunk]
|
||||
retry_kwargs = {} if max_retries is None else {"max_retries": max_retries}
|
||||
part_vectors, meta = await self._embed_openai(part, **retry_kwargs)
|
||||
base_meta = dict(meta or {})
|
||||
base_meta["batch_mode"] = "single_request" if chunk_count == 1 else "chunked"
|
||||
base_meta["batch_input_count"] = len(part)
|
||||
if chunk_count > 1:
|
||||
base_meta["batch_chunk_index"] = start // chunk
|
||||
base_meta["batch_chunk_count"] = chunk_count
|
||||
for idx in range(len(part)):
|
||||
m = dict(base_meta)
|
||||
m["batch_index"] = start + idx
|
||||
all_metas.append(m)
|
||||
vectors.extend(part_vectors)
|
||||
for idx in range(len(safe_texts)):
|
||||
m = dict(base_meta)
|
||||
m["batch_index"] = idx
|
||||
all_metas.append(m)
|
||||
|
||||
self._last_embed_meta = all_metas[-1] if all_metas else {}
|
||||
return vectors, all_metas
|
||||
@@ -848,7 +783,7 @@ class EmbedService:
|
||||
)
|
||||
|
||||
async def _embed_openai(
|
||||
self, texts: List[str], *, max_retries: Optional[int] = None,
|
||||
self, texts: List[str]
|
||||
) -> Tuple[List[List[float]], Dict[str, Any]]:
|
||||
"""
|
||||
使用 OpenAI 兼容接口向量化
|
||||
@@ -938,9 +873,7 @@ class EmbedService:
|
||||
"endpoint_overridden": bool(_ov),
|
||||
}
|
||||
|
||||
_max_retries = max(0, int(
|
||||
(self._embed_config.max_retries or 0) if max_retries is None else max_retries
|
||||
))
|
||||
_max_retries = max(0, int(self._embed_config.max_retries or 0))
|
||||
_retry_delay = max(0.0, float(self._embed_config.retry_delay or 0.0))
|
||||
last_exc: Optional[Exception] = None
|
||||
|
||||
@@ -986,17 +919,6 @@ class EmbedService:
|
||||
f"got={len(vectors)}"
|
||||
)
|
||||
dims = [len(v) for v in vectors]
|
||||
expected_dims = self._embed_config.embedding_dims
|
||||
if expected_dims:
|
||||
bad = [d for d in dims if d != int(expected_dims)]
|
||||
if bad:
|
||||
# 宽度不对的向量写进集合会把集合的维度钉死在错误值上(Chroma
|
||||
# 首次 add 定维),或在已有集合上整批失败。确定性错误,不重试。
|
||||
raise EmbeddingDimensionMismatchError(
|
||||
"embedding dimension mismatch: "
|
||||
f"expected={int(expected_dims)} got={bad[0]} "
|
||||
f"model={self._embed_config.model}"
|
||||
)
|
||||
meta = {
|
||||
"http_status": raw.status_code,
|
||||
"response_headers": resp_headers,
|
||||
@@ -1032,8 +954,6 @@ class EmbedService:
|
||||
},
|
||||
)
|
||||
return vectors, meta
|
||||
except EmbeddingDimensionMismatchError:
|
||||
raise
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
_err_name = type(e).__name__
|
||||
@@ -1067,18 +987,12 @@ class EmbedService:
|
||||
if _retryable and attempt < _max_retries:
|
||||
# 逐出可能卡死的 client,下个 attempt 重建干净连接(恢复原防御语义)
|
||||
self._evict_client(_eff_base_url, _eff_api_key, _eff_timeout)
|
||||
_delay = _retry_delay
|
||||
_retry_after = _retry_after_seconds(e) if _status == 429 else None
|
||||
if _retry_after is not None:
|
||||
# 限流时服务端明确告诉了要等多久:按它等,再加一点抖动,
|
||||
# 避免多个 worker 在同一时刻一起醒来再次撞限流。
|
||||
_delay = max(_delay, _retry_after) + random.uniform(0, 0.25 * _retry_after)
|
||||
logger.warning(
|
||||
f"[embed] attempt {attempt + 1}/{_max_retries + 1} failed "
|
||||
f"({_err_name}): {_err_msg[:200]}; retry in {_delay:.2f}s"
|
||||
f"({_err_name}): {_err_msg[:200]}; retry in {_retry_delay}s"
|
||||
)
|
||||
if _delay > 0:
|
||||
await asyncio.sleep(_delay)
|
||||
if _retry_delay > 0:
|
||||
await asyncio.sleep(_retry_delay)
|
||||
continue
|
||||
self._log_embed_call(
|
||||
status="error",
|
||||
|
||||
@@ -22,7 +22,6 @@ import logging
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
||||
from ..models.memory import MemoryNode, MemoryLayer, MemoryStatus
|
||||
from ..config import MemoryConfig
|
||||
@@ -78,21 +77,6 @@ async def _run_in_kuzu_pool(func, *args, **kwargs):
|
||||
return await run_in_executor_guarded(_kuzu_executor, _measured)
|
||||
|
||||
|
||||
# 本进程里已打开、尚未 close 的 store。drop_graph 据此拒绝删掉一个正被
|
||||
# 活着的 client 使用的图库。WeakSet:store 被回收后自动消失。
|
||||
_LIVE_STORES: "weakref.WeakSet[KuzuGraphStore]" = weakref.WeakSet()
|
||||
|
||||
|
||||
def open_graph_paths() -> List[str]:
|
||||
"""Real paths of the graph databases live stores in this process hold open."""
|
||||
import os
|
||||
return [
|
||||
os.path.realpath(str(store._db_path))
|
||||
for store in list(_LIVE_STORES)
|
||||
if store._db is not None
|
||||
]
|
||||
|
||||
|
||||
def kuzu_executor_snapshot() -> Dict[str, Any]:
|
||||
with _kuzu_stats_lock:
|
||||
return {
|
||||
@@ -366,7 +350,6 @@ class KuzuGraphStore(GraphStoreBase):
|
||||
)
|
||||
self._db = db
|
||||
self._conn = conn
|
||||
_LIVE_STORES.add(self)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
f"GraphStore init timed out after 5s (Kuzu file lock contention). "
|
||||
@@ -2458,7 +2441,6 @@ class KuzuGraphStore(GraphStoreBase):
|
||||
async def close(self) -> None:
|
||||
"""关闭连接"""
|
||||
conn, db = self._conn, self._db
|
||||
_LIVE_STORES.discard(self)
|
||||
self._conn = None
|
||||
self._db = None
|
||||
self._available = False
|
||||
|
||||
@@ -89,17 +89,6 @@ def passes_created_after(
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def physical_collection_name(base_name: Optional[str], dims: Optional[int]) -> str:
|
||||
"""The collection name a store opens for ``base_name`` at ``dims``.
|
||||
|
||||
The dims suffix keeps two embedding widths from sharing one collection.
|
||||
`hy_memory.reembed.vector_space_names` uses this too, so callers never
|
||||
re-derive the rule.
|
||||
"""
|
||||
base = base_name or "agent_memories"
|
||||
return f"{base}_{dims}" if dims else base
|
||||
|
||||
|
||||
class VectorStoreBase(ABC):
|
||||
"""
|
||||
向量存储抽象基类
|
||||
@@ -109,10 +98,13 @@ class VectorStoreBase(ABC):
|
||||
|
||||
def __init__(self, config: MemoryConfig):
|
||||
self.config = config
|
||||
base_name = config.vector_store.collection_name or "agent_memories"
|
||||
dims = config.vector_store.embedding_dims
|
||||
# 自动拼接维度后缀,避免不同 embedding 模型写入同一 collection 导致维度冲突
|
||||
self._collection_name = physical_collection_name(
|
||||
config.vector_store.collection_name, config.vector_store.embedding_dims,
|
||||
)
|
||||
if dims:
|
||||
self._collection_name = f"{base_name}_{dims}"
|
||||
else:
|
||||
self._collection_name = base_name
|
||||
|
||||
_supports_retrieval_clauses: bool = False
|
||||
_supports_narrow_episode_queries: bool = False
|
||||
|
||||
@@ -9,12 +9,10 @@ Agent Memory V2 - VectorStore (Chroma)
|
||||
- 同步 API → _run_in_vdb_pool() 包装为异步
|
||||
"""
|
||||
|
||||
from typing import Optional, List, Dict, Any, Set, Tuple
|
||||
from typing import Optional, List, Dict, Any
|
||||
import concurrent.futures
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import weakref
|
||||
|
||||
from ..models.memory import (
|
||||
MemoryNode, MemoryLayer, MemoryStatus,
|
||||
@@ -35,30 +33,6 @@ _vdb_executor = concurrent.futures.ThreadPoolExecutor(
|
||||
)
|
||||
|
||||
|
||||
# 本进程里已 initialize、尚未 close 的 store。drop_vector_space 据此拒绝删掉
|
||||
# 一个正被活着的 client 使用的集合。WeakSet:store 被回收后自动消失。
|
||||
_LIVE_STORES: "weakref.WeakSet[ChromaVectorStore]" = weakref.WeakSet()
|
||||
|
||||
|
||||
def _location_key(vs_config) -> Tuple[str, str]:
|
||||
if getattr(vs_config, "host", None):
|
||||
return ("http", f"{vs_config.host}:{vs_config.port or 8000}")
|
||||
return ("local", os.path.realpath(str(vs_config.persist_directory or "")))
|
||||
|
||||
|
||||
def open_collection_names(location: Tuple[str, str]) -> Set[str]:
|
||||
"""Collections that live stores in this process hold open at ``location``."""
|
||||
names: Set[str] = set()
|
||||
for store in list(_LIVE_STORES):
|
||||
if store._client is None or _location_key(store.config.vector_store) != location:
|
||||
continue
|
||||
names.add(store._collection_name)
|
||||
if store._entity_collection is not None:
|
||||
from ..pipelines._retrieval.config import entity_collection_name
|
||||
names.add(entity_collection_name(store._collection_name))
|
||||
return names
|
||||
|
||||
|
||||
async def _run_in_vdb_pool(func, *args, **kwargs):
|
||||
"""Run a synchronous Chroma call in the dedicated VDB executor."""
|
||||
call = functools.partial(func, *args, **kwargs) if args or kwargs else func
|
||||
@@ -114,7 +88,6 @@ class ChromaVectorStore(VectorStoreBase):
|
||||
return client, collection
|
||||
|
||||
self._client, self._collection = await _run_in_vdb_pool(_init)
|
||||
_LIVE_STORES.add(self)
|
||||
logger.debug(f"VectorStore initialized (Chroma), collection={self._collection_name}")
|
||||
|
||||
# ================================================================
|
||||
@@ -588,7 +561,6 @@ class ChromaVectorStore(VectorStoreBase):
|
||||
async def close(self) -> None:
|
||||
"""关闭客户端"""
|
||||
# Chroma PersistentClient 无需显式关闭
|
||||
_LIVE_STORES.discard(self)
|
||||
self._client = None
|
||||
self._collection = None
|
||||
self._entity_collection = None
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+111
-2
@@ -2018,12 +2018,17 @@ async function runCommand(
|
||||
return sourceProgressExitCode(document);
|
||||
}
|
||||
if (named === "profile") {
|
||||
const profileFlags = arguments_.slice(1);
|
||||
if (
|
||||
arguments_.filter((item) => item === "--json").length > 1 ||
|
||||
arguments_.slice(1).some((value) => value !== "--json")
|
||||
profileFlags.filter((item) => item === "--json").length > 1 ||
|
||||
profileFlags.filter((item) => item === "--versions").length > 1 ||
|
||||
profileFlags.some((value) => value !== "--json" && value !== "--versions")
|
||||
) {
|
||||
return usage();
|
||||
}
|
||||
if (profileFlags.includes("--versions")) {
|
||||
return await profileVersionsCommand(environment, { asJson, userId });
|
||||
}
|
||||
const connected = await connectRuntime(environment, {
|
||||
readerTimeoutMs: 15_000,
|
||||
});
|
||||
@@ -3888,6 +3893,108 @@ async function emitModelReceipt(
|
||||
|
||||
/** Attach to a verified Runtime, telling a stopped one (exit 3) from one
|
||||
* that is up but unusable (exit 1). */
|
||||
/**
|
||||
* `memhub profile --versions`: the long-term profile's L0 version chains,
|
||||
* latest first, so a value that replaced a better one can be found by id.
|
||||
*/
|
||||
async function profileVersionsCommand(
|
||||
environment: NodeJS.ProcessEnv,
|
||||
options: { asJson: boolean; userId?: string },
|
||||
): Promise<number> {
|
||||
const fail = (state: string, code: string, exitCode: number): number => {
|
||||
emit(
|
||||
options.asJson
|
||||
? `${JSON.stringify({ schema_version: 2, product: PRODUCT, state, code })}\n`
|
||||
: `画像版本读取失败:${code}\n`,
|
||||
);
|
||||
return exitCode;
|
||||
};
|
||||
const connected = await connectRuntime(environment, {
|
||||
readerTimeoutMs: 15_000,
|
||||
});
|
||||
if (!connected.ok) {
|
||||
return fail("unavailable", connected.code, connected.exitCode);
|
||||
}
|
||||
const outcome = await connected.reader<unknown>(
|
||||
"/api/v2/user-profile/versions",
|
||||
userIdParameters(options.userId),
|
||||
);
|
||||
const chains = outcome.state === "ok" ? profileChains(outcome.data) : null;
|
||||
if (outcome.state !== "ok" || chains === null) {
|
||||
return fail(
|
||||
outcome.state === "ok" ? "invalid" : outcome.state,
|
||||
outcome.state === "ok" ? "response_invalid" : outcome.code,
|
||||
EXIT_FAILED,
|
||||
);
|
||||
}
|
||||
if (options.asJson) {
|
||||
emit(
|
||||
`${JSON.stringify({
|
||||
schema_version: 2,
|
||||
product: PRODUCT,
|
||||
state: "ok",
|
||||
...(outcome.data as object),
|
||||
})}\n`,
|
||||
);
|
||||
return EXIT_OK;
|
||||
}
|
||||
if (chains.length === 0) {
|
||||
emit("尚未形成用户画像版本。\n");
|
||||
return EXIT_OK;
|
||||
}
|
||||
const lines: string[] = [];
|
||||
for (const chain of chains) {
|
||||
lines.push(`画像 ${chain.memory_id}(${chain.status})`);
|
||||
for (const version of chain.versions) {
|
||||
const when =
|
||||
typeof version.gmt_created === "number"
|
||||
? new Date(version.gmt_created * 1000).toISOString()
|
||||
: "时间未知";
|
||||
lines.push(
|
||||
` ${version.current ? "*" : " "} ${version.memory_id} ${when} ${version.content}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
"* 表示当前生效的版本;删除错误版本:memhub memories delete <memory-id> --yes",
|
||||
);
|
||||
emit(`${lines.join("\n")}\n`);
|
||||
return EXIT_OK;
|
||||
}
|
||||
|
||||
type ProfileChainView = {
|
||||
memory_id: string;
|
||||
status: string;
|
||||
versions: {
|
||||
memory_id: string;
|
||||
content: string;
|
||||
gmt_created?: number | null;
|
||||
current: boolean;
|
||||
}[];
|
||||
};
|
||||
|
||||
function profileChains(value: unknown): ProfileChainView[] | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const chains = (value as { chains?: unknown }).chains;
|
||||
if (!Array.isArray(chains)) return null;
|
||||
const valid = chains.every(
|
||||
(chain) =>
|
||||
chain &&
|
||||
typeof chain === "object" &&
|
||||
typeof (chain as ProfileChainView).memory_id === "string" &&
|
||||
typeof (chain as ProfileChainView).status === "string" &&
|
||||
Array.isArray((chain as ProfileChainView).versions) &&
|
||||
(chain as ProfileChainView).versions.every(
|
||||
(version) =>
|
||||
version &&
|
||||
typeof version.memory_id === "string" &&
|
||||
typeof version.content === "string" &&
|
||||
typeof version.current === "boolean",
|
||||
),
|
||||
);
|
||||
return valid ? (chains as ProfileChainView[]) : null;
|
||||
}
|
||||
|
||||
async function connectRuntime(
|
||||
environment: NodeJS.ProcessEnv,
|
||||
options?: { readerTimeoutMs?: number },
|
||||
@@ -4155,6 +4262,8 @@ const SDK_MIGRATION_FAILURES: Record<string, string> = {
|
||||
"核对发现新版本还读不到你的全部记忆,没有切换数据源。",
|
||||
sdk_migration_model_profile_required:
|
||||
"迁移需要先有可用的模型配置,用来核对新版本能否读回记忆。请先用 memhub models setup 完成配置,再运行 memhub restart 重试。",
|
||||
sdk_migration_vector_profile_missing:
|
||||
"记忆的向量来自一个已删除的模型配置,当前配置读不回它们,所以没有迁移。请重新添加并启用那个模型配置,再运行 memhub restart 重试。",
|
||||
};
|
||||
|
||||
const MIGRATION_ISSUES_URL =
|
||||
|
||||
@@ -434,9 +434,18 @@ export const COMMANDS: readonly CommandDescriptor[] = [
|
||||
},
|
||||
{
|
||||
name: "profile",
|
||||
summary: "读取 SDK 组装后的用户长期画像(L0 + 已启用的长期画像补充)。",
|
||||
usage: "memhub profile [--user-id <id>] [--json]",
|
||||
options: [USER_ID_OPTION, JSON_OPTION],
|
||||
summary:
|
||||
"读取 SDK 组装后的用户长期画像(L0 + 已启用的长期画像补充);--versions 列出 L0 画像的版本链(最新在前,* 为当前版本)。",
|
||||
usage: "memhub profile [--versions] [--user-id <id>] [--json]",
|
||||
options: [
|
||||
{
|
||||
flag: "--versions",
|
||||
summary:
|
||||
"列出 L0 基本信息的版本链(get_user_basic_info),用于找出覆盖了正确值的错误版本。",
|
||||
},
|
||||
USER_ID_OPTION,
|
||||
JSON_OPTION,
|
||||
],
|
||||
exit_codes: RUNTIME_EXITS,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -17,6 +17,7 @@ const READ_ONLY_PATHS = [
|
||||
"/api/v2/home-summary",
|
||||
"/api/v2/memory-library",
|
||||
"/api/v2/user-profile",
|
||||
"/api/v2/user-profile/versions",
|
||||
] as const;
|
||||
const SDK_MEMORY_DETAIL = /^\/api\/v2\/memories\/[A-Za-z0-9][\w.-]{0,127}$/;
|
||||
/** Status of one `POST /api/v2/memories` addition, by its request id. */
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { run } from "./cli.js";
|
||||
|
||||
const transport = vi.hoisted(() => ({
|
||||
get: vi.fn(),
|
||||
post: vi.fn(),
|
||||
put: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
}));
|
||||
vi.mock("axios", () => ({
|
||||
default: {
|
||||
create: vi.fn(() => transport),
|
||||
get: vi.fn(),
|
||||
post: vi.fn(),
|
||||
isAxiosError: () => false,
|
||||
},
|
||||
}));
|
||||
|
||||
const runtime = vi.hoisted(() => ({
|
||||
inspectRuntimeStatus: vi.fn(),
|
||||
discoverRuntimeAttachment: vi.fn(),
|
||||
}));
|
||||
vi.mock("@one-memory/runtime-control", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@one-memory/runtime-control")>()),
|
||||
...runtime,
|
||||
}));
|
||||
|
||||
const VERSIONS = {
|
||||
user_id: "wife",
|
||||
request_id: "r-versions",
|
||||
elapsed_ms: 2,
|
||||
chains: [
|
||||
{
|
||||
memory_id: "l0-head",
|
||||
content: "The user's name is Fei.",
|
||||
status: "active",
|
||||
memory_at: null,
|
||||
gmt_created: 1_790_000_100,
|
||||
basic_info: { name: "Fei" },
|
||||
versions: [
|
||||
{
|
||||
memory_id: "l0-head",
|
||||
content: "The user's name is Fei.",
|
||||
memory_at: null,
|
||||
gmt_created: 1_790_000_100,
|
||||
current: true,
|
||||
},
|
||||
{
|
||||
memory_id: "l0-old",
|
||||
content: "The user's name is 费天翔.",
|
||||
memory_at: null,
|
||||
gmt_created: 1_790_000_000,
|
||||
current: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
for (const method of Object.values(transport)) method.mockReset();
|
||||
runtime.inspectRuntimeStatus.mockResolvedValue({
|
||||
state: "running",
|
||||
code: "runtime_verified",
|
||||
instanceId: "instance-1",
|
||||
host: "127.0.0.1",
|
||||
port: 8765,
|
||||
version: "1.2.3",
|
||||
});
|
||||
runtime.discoverRuntimeAttachment.mockResolvedValue({
|
||||
instanceId: "instance-1",
|
||||
apiToken: "synthetic-runtime-token",
|
||||
host: "127.0.0.1",
|
||||
port: 8765,
|
||||
servicePid: 4242,
|
||||
health: {
|
||||
status: "ok",
|
||||
service: "one-memory-core",
|
||||
version: "1.2.3",
|
||||
instance_id: "instance-1",
|
||||
},
|
||||
});
|
||||
transport.get.mockImplementation(async (url: string) =>
|
||||
url === "/api/v2/user-profile/versions"
|
||||
? { status: 200, data: VERSIONS }
|
||||
: { status: 404, data: {} },
|
||||
);
|
||||
});
|
||||
|
||||
describe("memhub profile --versions", () => {
|
||||
it("reads the L0 version chain for the named user", async () => {
|
||||
const result = await capturedRun([
|
||||
"profile",
|
||||
"--versions",
|
||||
"--user-id",
|
||||
"wife",
|
||||
"--json",
|
||||
]);
|
||||
expect(result.code).toBe(0);
|
||||
expect(transport.get).toHaveBeenCalledWith(
|
||||
"/api/v2/user-profile/versions",
|
||||
{ params: { user_id: "wife" } },
|
||||
);
|
||||
const document = JSON.parse(result.stdout);
|
||||
expect(document.state).toBe("ok");
|
||||
expect(document.chains[0].versions).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("marks the current version in text and names no user for local", async () => {
|
||||
const result = await capturedRun(["profile", "--versions"]);
|
||||
expect(result.code).toBe(0);
|
||||
const [url, config] = transport.get.mock.calls[0] ?? [];
|
||||
expect(url).toBe("/api/v2/user-profile/versions");
|
||||
expect(config?.params?.user_id).toBeUndefined();
|
||||
expect(result.stdout).toContain("* l0-head");
|
||||
expect(result.stdout).toContain(" l0-old");
|
||||
expect(result.stdout).toContain("费天翔");
|
||||
});
|
||||
|
||||
it("refuses an unknown flag and a repeated --versions", async () => {
|
||||
expect((await capturedRun(["profile", "--version-chain"])).code).toBe(2);
|
||||
expect(
|
||||
(await capturedRun(["profile", "--versions", "--versions"])).code,
|
||||
).toBe(2);
|
||||
expect(transport.get).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports a malformed answer as invalid", async () => {
|
||||
transport.get.mockResolvedValue({ status: 200, data: { chains: "x" } });
|
||||
const result = await capturedRun(["profile", "--versions", "--json"]);
|
||||
expect(result.code).toBe(1);
|
||||
expect(JSON.parse(result.stdout).code).toBe("response_invalid");
|
||||
});
|
||||
});
|
||||
|
||||
async function capturedRun(arguments_: string[]) {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const stdoutSpy = vi
|
||||
.spyOn(process.stdout, "write")
|
||||
.mockImplementation((chunk) => {
|
||||
stdout += String(chunk);
|
||||
return true;
|
||||
});
|
||||
const stderrSpy = vi
|
||||
.spyOn(process.stderr, "write")
|
||||
.mockImplementation((chunk) => {
|
||||
stderr += String(chunk);
|
||||
return true;
|
||||
});
|
||||
try {
|
||||
return {
|
||||
code: await run(arguments_, {
|
||||
ONE_MEMORY_DATA_DIR: "/synthetic/data",
|
||||
}),
|
||||
stdout,
|
||||
stderr,
|
||||
};
|
||||
} finally {
|
||||
stdoutSpy.mockRestore();
|
||||
stderrSpy.mockRestore();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -94,7 +95,6 @@ from one_memory.hy_runtime_config import (
|
||||
from one_memory.hy_sdk_runtime import (
|
||||
AmbiguousMemoryIdError,
|
||||
SdkClientHandle,
|
||||
builtin_retrieval_snapshot,
|
||||
close_all_shared_runtimes,
|
||||
hold_stores_offline,
|
||||
release_stores,
|
||||
@@ -104,6 +104,7 @@ from one_memory.hy_sdk_types import (
|
||||
HySdkError,
|
||||
HySdkMemoryPage,
|
||||
HySdkMemoryRecord,
|
||||
HySdkProfileVersions,
|
||||
HySdkRawHit,
|
||||
HySdkRawPage,
|
||||
HySdkRawRecord,
|
||||
@@ -111,8 +112,10 @@ from one_memory.hy_sdk_types import (
|
||||
HySdkUserProfile,
|
||||
LibraryHit,
|
||||
library_keyword_hits,
|
||||
parse_sdk_lane_search,
|
||||
parse_sdk_raw_search,
|
||||
parse_sdk_search,
|
||||
raw_turn_messages,
|
||||
)
|
||||
from one_memory.imports.access import (
|
||||
AuthorizedSourceRoot,
|
||||
@@ -190,6 +193,7 @@ from one_memory.model_presets import seed_preset_profile
|
||||
from one_memory.model_probe import ModelProber, ModelProbeResult
|
||||
from one_memory.model_profiles import (
|
||||
ActiveModelProfileError,
|
||||
EmbeddingBaseUrlRequiredError,
|
||||
ModelCreationOperationConflictError,
|
||||
ModelCreationOperationLookupError,
|
||||
ModelCreationOperationReceipt,
|
||||
@@ -229,6 +233,14 @@ from one_memory.sdk_memory_qa import (
|
||||
)
|
||||
from one_memory.sdk_memory_runtime import SdkMemoryRuntime
|
||||
from one_memory.sdk_migration_log import error_chain_text, error_frames
|
||||
from one_memory.sdk_read_handle import ActiveProfileReadHandle
|
||||
from one_memory.sdk_search_options import (
|
||||
MEMORY_LAYERS,
|
||||
MEMORY_STATUSES,
|
||||
SdkSearchFilters,
|
||||
SearchLane,
|
||||
split_csv,
|
||||
)
|
||||
from one_memory.sdk_write_processor import SdkWriteCoordinator
|
||||
from one_memory.secrets import (
|
||||
KeyringSecretStore,
|
||||
@@ -236,6 +248,7 @@ from one_memory.secrets import (
|
||||
SecretStoreErrorCode,
|
||||
SecretStoreUnavailableError,
|
||||
)
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
from one_memory.service_levels import (
|
||||
ResourceLimit,
|
||||
ServiceLevel,
|
||||
@@ -398,6 +411,23 @@ class CreateImportJobRequest(BaseModel):
|
||||
operation: Literal["history", "sync"] = "history"
|
||||
|
||||
|
||||
class CreateImportJobV2Request(CreateImportJobRequest):
|
||||
"""`POST /api/v2/import-jobs`: v1's body plus whose lane it runs on."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
#: Only Hermes holds roots for a user other than `local`.
|
||||
user_id: UserId = DEFAULT_USER_ID
|
||||
|
||||
|
||||
class SourceSyncNowRequest(BaseModel):
|
||||
"""Optional body of `POST /api/v2/sources/{id}/sync`."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
user_id: UserId = DEFAULT_USER_ID
|
||||
|
||||
|
||||
class ConnectSourcesRequest(BaseModel):
|
||||
source_ids: tuple[str, ...] = Field(min_length=1, max_length=32)
|
||||
#: Whose grant this is. Only Hermes may connect a root for a user other
|
||||
@@ -453,13 +483,83 @@ class SdkUserProfileResponse(BaseModel):
|
||||
elapsed_ms: float = Field(ge=0)
|
||||
|
||||
|
||||
class SdkProfileVersionsResponse(HySdkProfileVersions):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
user_id: str
|
||||
|
||||
|
||||
class SdkRawMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
#: user / assistant (tool and system if the SDK kept them); "" if unnamed.
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class SdkRawItem(HySdkRawRecord):
|
||||
"""One SDK raw turn as `/api/v2/raw` lists it.
|
||||
|
||||
The SDK record's fields, plus what tracing an import needs: the turn's
|
||||
messages with their roles (parsed from the SDK's `[role]: text` lines),
|
||||
a `sha256:` hash of the stored content computed by Core (the SDK keeps
|
||||
none), and `created`, when the SDK stored it.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
messages: tuple[SdkRawMessage, ...] = ()
|
||||
#: The one role of a single-message turn; None for a mixed turn.
|
||||
role: str | None = None
|
||||
content_hash: str
|
||||
created: datetime | None = None
|
||||
|
||||
@classmethod
|
||||
def of(cls, record: HySdkRawRecord) -> "SdkRawItem":
|
||||
messages = tuple(
|
||||
SdkRawMessage(role=role, content=text)
|
||||
for role, text in raw_turn_messages(record.content)
|
||||
)
|
||||
roles = {message.role for message in messages}
|
||||
return cls(
|
||||
**record.model_dump(),
|
||||
messages=messages,
|
||||
role=next(iter(roles)) if len(roles) == 1 and "" not in roles else None,
|
||||
content_hash="sha256:" + hashlib.sha256(record.content.encode("utf-8")).hexdigest(),
|
||||
created=(
|
||||
datetime.fromtimestamp(record.gmt_created, tz=UTC)
|
||||
if record.gmt_created is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SdkRawPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
items: tuple[HySdkRawRecord, ...] = ()
|
||||
items: tuple[SdkRawItem, ...] = ()
|
||||
total: int = Field(ge=0)
|
||||
offset: int = Field(ge=0)
|
||||
limit: int = Field(ge=1, le=500)
|
||||
#: Set when a time filter was applied to this page in Core (the SDK list
|
||||
#: has none): `items` is the page's rows inside the range, `total` stays
|
||||
#: the SDK's count before that filter.
|
||||
time_filtered: bool = False
|
||||
|
||||
|
||||
class SdkLayerCounts(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
#: Every SDK layer, zero when empty.
|
||||
layers: dict[str, int]
|
||||
total: int = Field(ge=0)
|
||||
#: The statuses counted, or None for the defaults (active; raw: active+shadow).
|
||||
statuses: tuple[str, ...] | None = None
|
||||
|
||||
|
||||
#: Raw lifecycle states that are still a kept conversation. Pro marks a raw
|
||||
#: SHADOW once extraction succeeds; lite leaves it ACTIVE.
|
||||
RAW_STATUSES = ("active", "shadow")
|
||||
|
||||
|
||||
class MemoryProcessingBlock(BaseModel):
|
||||
@@ -604,8 +704,15 @@ class ManualExtractReceipt(BaseModel):
|
||||
batches_current: int
|
||||
|
||||
|
||||
class AskMemoryRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
class AskMemoryRequest(SdkSearchFilters):
|
||||
"""Ask over one user's memories.
|
||||
|
||||
The `SdkSearchFilters` fields (floors, rerank, source/session/agent/time
|
||||
and layer filters) reach the retrieval search. Scene, reader and search
|
||||
mode stay fixed: the answer cites chat memories read back by id.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=False)
|
||||
|
||||
question: str = Field(min_length=1, max_length=2_000)
|
||||
scope: str = Field(default="all-apps", min_length=1, max_length=512)
|
||||
@@ -922,6 +1029,16 @@ def create_app(
|
||||
)
|
||||
except sqlite3.Error as error:
|
||||
app.state.memory_visibility_error = str(error)
|
||||
else:
|
||||
# A home an earlier build migrated still carries the grant
|
||||
# triggers whose body reads a dropped table (#233).
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
memory_visibility.drop_stale_triggers,
|
||||
database_path,
|
||||
)
|
||||
except sqlite3.Error as error:
|
||||
app.state.memory_visibility_error = str(error)
|
||||
model_request_budget = (
|
||||
ModelRequestBudget(
|
||||
database_path,
|
||||
@@ -1953,7 +2070,11 @@ def create_app(
|
||||
dependencies=protected,
|
||||
)
|
||||
async def list_sources() -> list[SourceResponse]:
|
||||
detection = _detection(app)
|
||||
return await sources_for(DEFAULT_USER_ID)
|
||||
|
||||
async def sources_for(user_id: str) -> list[SourceResponse]:
|
||||
"""One user's sources. `local`'s are the App's; another user sees Hermes only."""
|
||||
detection = _detection(app) if user_id == DEFAULT_USER_ID else {}
|
||||
# Read in threads. The desktop polls this every few seconds, and both
|
||||
# reads below are synchronous SQLite — the totals across every source,
|
||||
# then a grant receipt per source. Made on the loop, as they were,
|
||||
@@ -1963,7 +2084,7 @@ def create_app(
|
||||
# The totals are also not counted again for each ask: counting is a
|
||||
# pass over every row, and answering this took up to 6.70s during a
|
||||
# catch-up, nearly all of it the same count repeated.
|
||||
totals_by_source = await app.state.source_totals.by_source(user_id="local")
|
||||
totals_by_source = await app.state.source_totals.by_source(user_id=user_id)
|
||||
# Status polling must not trigger history discovery: detect can walk an
|
||||
# entire host tree and recheck the grant in SQLite for every file, with
|
||||
# concurrent polls duplicating that work. Only the explicit detect
|
||||
@@ -1973,28 +2094,49 @@ def create_app(
|
||||
def describe() -> list[SourceResponse]:
|
||||
grant_store = _grant_store(app)
|
||||
config_store = _config_store(app)
|
||||
return [
|
||||
_source_response(
|
||||
registration,
|
||||
grant_store.receipt(user_id="local", source_id=source_id),
|
||||
config_store.sync_enabled(source_id),
|
||||
detection.get(source_id),
|
||||
runtime_settings.source_home,
|
||||
totals_by_source.get(source_id, (0, 0)),
|
||||
candidate_change_reason=app.state.source_candidate_store.change_reason(
|
||||
registration
|
||||
),
|
||||
authorization_state=grant_store.authorization_state(
|
||||
user_id="local",
|
||||
source_id=source_id,
|
||||
candidate=registration.candidate.path,
|
||||
),
|
||||
responses = []
|
||||
for source_id, registration in _catalog(app).items():
|
||||
if user_id != DEFAULT_USER_ID and source_id not in MULTI_USER_SOURCES:
|
||||
continue
|
||||
if user_id != DEFAULT_USER_ID:
|
||||
# Another user's Hermes is the profile root they were
|
||||
# granted, not the App's candidate.
|
||||
registration = _registration_at_grant(grant_store, registration, user_id)
|
||||
responses.append(
|
||||
_source_response(
|
||||
registration,
|
||||
grant_store.receipt(user_id=user_id, source_id=source_id),
|
||||
_lane_sync_enabled(config_store, lane_key(user_id, source_id)),
|
||||
detection.get(source_id),
|
||||
runtime_settings.source_home,
|
||||
totals_by_source.get(source_id, (0, 0)),
|
||||
candidate_change_reason=(
|
||||
app.state.source_candidate_store.change_reason(registration)
|
||||
if user_id == DEFAULT_USER_ID
|
||||
else "multi_user_grant"
|
||||
),
|
||||
authorization_state=grant_store.authorization_state(
|
||||
user_id=user_id,
|
||||
source_id=source_id,
|
||||
candidate=registration.candidate.path,
|
||||
),
|
||||
)
|
||||
)
|
||||
for source_id, registration in _catalog(app).items()
|
||||
]
|
||||
return responses
|
||||
|
||||
return await asyncio.to_thread(describe)
|
||||
|
||||
@app.get(
|
||||
"/api/v2/sources",
|
||||
response_model=list[SourceResponse],
|
||||
dependencies=strict,
|
||||
)
|
||||
async def list_sources_v2(
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
) -> list[SourceResponse]:
|
||||
"""`GET /api/v1/sources` for one user: `local` (the default) reads the App's."""
|
||||
return await sources_for(user_id)
|
||||
|
||||
@app.get(
|
||||
"/api/v1/adapters",
|
||||
dependencies=protected,
|
||||
@@ -2512,20 +2654,37 @@ def create_app(
|
||||
dependencies=protected,
|
||||
)
|
||||
async def create_import_job(request: CreateImportJobRequest) -> ImportJob:
|
||||
_require_registration(app, request.source_id)
|
||||
if request.source_id not in _supervisor(app).adapters:
|
||||
return await create_job_on(request.source_id, request.operation, DEFAULT_USER_ID)
|
||||
|
||||
async def create_job_on(
|
||||
source_id: str, operation: Literal["history", "sync"], user_id: str
|
||||
) -> ImportJob:
|
||||
"""Start an import on one user's lane of a source (`local`: the bare id)."""
|
||||
_require_registration(app, source_id)
|
||||
_require_multi_user_source(source_id, user_id)
|
||||
lane = lane_key(user_id, source_id)
|
||||
if lane not in _supervisor(app).adapters:
|
||||
raise HTTPException(status_code=409, detail="source_not_authorized")
|
||||
try:
|
||||
job = await asyncio.to_thread(
|
||||
_supervisor(app).create_job,
|
||||
request.source_id,
|
||||
operation=request.operation,
|
||||
lane,
|
||||
operation=operation,
|
||||
)
|
||||
except SourceAccessError as error:
|
||||
raise HTTPException(status_code=409, detail=error.code) from None
|
||||
_supervisor(app).enqueue(job.job_id)
|
||||
return job
|
||||
|
||||
@app.post(
|
||||
"/api/v2/import-jobs",
|
||||
response_model=ImportJob,
|
||||
dependencies=strict,
|
||||
)
|
||||
async def create_import_job_v2(request: CreateImportJobV2Request) -> ImportJob:
|
||||
"""`POST /api/v1/import-jobs` on one user's lane (Hermes only for another user)."""
|
||||
return await create_job_on(request.source_id, request.operation, request.user_id)
|
||||
|
||||
@app.post(
|
||||
"/api/v1/sources/connect",
|
||||
response_model=ConnectSourcesResponse,
|
||||
@@ -2775,14 +2934,55 @@ def create_app(
|
||||
dependencies=protected,
|
||||
)
|
||||
async def trigger_live_sync(source_id: str) -> ImportJob:
|
||||
return await sync_lane(source_id, DEFAULT_USER_ID)
|
||||
|
||||
async def sync_lane(source_id: str, user_id: str) -> ImportJob:
|
||||
_require_registration(app, source_id)
|
||||
_require_multi_user_source(source_id, user_id)
|
||||
# Asked for by hand, so it reports a job even when nothing was new;
|
||||
# `live_sync_unavailable` keeps meaning the source cannot be synced.
|
||||
job = await _live_sync(app).trigger_now(source_id)
|
||||
job = await _live_sync(app).trigger_now(lane_key(user_id, source_id))
|
||||
if job is None:
|
||||
raise HTTPException(status_code=409, detail="live_sync_unavailable")
|
||||
return job
|
||||
|
||||
@app.post(
|
||||
"/api/v2/sources/{source_id}/sync",
|
||||
response_model=ImportJob,
|
||||
dependencies=strict,
|
||||
)
|
||||
async def trigger_live_sync_v2(
|
||||
source_id: str,
|
||||
request: SourceSyncNowRequest | None = None,
|
||||
) -> ImportJob:
|
||||
"""`POST /api/v1/sources/{id}/sync` on one user's lane; the body is optional."""
|
||||
return await sync_lane(source_id, request.user_id if request else DEFAULT_USER_ID)
|
||||
|
||||
@app.get(
|
||||
"/api/v2/import-jobs",
|
||||
response_model=list[ImportJob],
|
||||
dependencies=strict,
|
||||
)
|
||||
def list_import_jobs_v2(
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
) -> list[ImportJob]:
|
||||
return list_import_jobs(user_id=user_id)
|
||||
|
||||
@app.get(
|
||||
"/api/v2/import-jobs/{job_id}",
|
||||
response_model=ImportJob,
|
||||
dependencies=strict,
|
||||
)
|
||||
def get_import_job_v2(
|
||||
job_id: str,
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
) -> ImportJob:
|
||||
"""One job; another user's job answers exactly like a missing one."""
|
||||
job = _require_job(app, job_id)
|
||||
if job.user_id != user_id:
|
||||
raise HTTPException(status_code=404, detail="job_not_found")
|
||||
return job
|
||||
|
||||
@app.get(
|
||||
"/api/v1/import-jobs/{job_id}",
|
||||
response_model=ImportJob,
|
||||
@@ -2863,6 +3063,8 @@ def create_app(
|
||||
def create_model_profile(request: ModelProfileInput) -> ModelProfile:
|
||||
try:
|
||||
return _model_profile_store(app).create(request, _secret_store(app))
|
||||
except EmbeddingBaseUrlRequiredError:
|
||||
raise HTTPException(422, "embedding_base_url_required") from None
|
||||
except ModelCreationOperationConflictError:
|
||||
raise HTTPException(409, "model_creation_operation_conflict") from None
|
||||
except ModelCreationOperationLookupError:
|
||||
@@ -2914,6 +3116,8 @@ def create_app(
|
||||
raise HTTPException(status_code=404, detail="model_profile_not_found") from None
|
||||
except ModelProfileConflictError:
|
||||
raise HTTPException(status_code=409, detail="model_profile_conflict") from None
|
||||
except EmbeddingBaseUrlRequiredError:
|
||||
raise HTTPException(status_code=422, detail="embedding_base_url_required") from None
|
||||
except SecretStoreUnavailableError as error:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
@@ -3141,6 +3345,11 @@ def create_app(
|
||||
window_id: str | None = Query(default=None, min_length=1, max_length=64),
|
||||
scene: Literal["all", "chat", "working", "coding"] = Query(default="all"),
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
layers: list[str] | None = Query(default=None), # noqa: B008
|
||||
layer: list[str] | None = Query(default=None), # noqa: B008
|
||||
exclude_sources: list[str] | None = Query(default=None), # noqa: B008
|
||||
session_ids: list[str] | None = Query(default=None), # noqa: B008
|
||||
agent_ids: list[str] | None = Query(default=None), # noqa: B008
|
||||
) -> HySdkMemoryPage:
|
||||
"""One page of memories across chat, working and coding.
|
||||
|
||||
@@ -3155,7 +3364,29 @@ def create_app(
|
||||
and are cut from the same matches, so pages never overlap or skip. A
|
||||
`window_id` that expired or was searched with other filters searches
|
||||
again and answers page 1 with `window_reset: true`.
|
||||
|
||||
`layers` (or `layer`), `statuses`, `exclude_sources`, `session_ids`
|
||||
and `agent_ids` may each repeat or be comma separated. Layers exist in
|
||||
the chat store only, so a layer filter lists the chat store (graph
|
||||
layers included) and refuses `scene=working|coding`. `l1_raw` belongs
|
||||
to `/api/v2/raw`.
|
||||
"""
|
||||
statuses = _library_values(statuses, MEMORY_STATUSES, "memory_status_unknown")
|
||||
chosen_layers = _library_values(
|
||||
[*(layers or ()), *(layer or ())], MEMORY_LAYERS, "memory_layer_unknown"
|
||||
)
|
||||
if chosen_layers and scene in {"working", "coding"}:
|
||||
raise HTTPException(status_code=422, detail="memory_layers_require_chat_scene")
|
||||
filters = {
|
||||
name: values
|
||||
for name, values in (
|
||||
("layers", chosen_layers),
|
||||
("exclude_sources", split_csv(exclude_sources)),
|
||||
("session_ids", split_csv(session_ids)),
|
||||
("agent_ids", split_csv(agent_ids)),
|
||||
)
|
||||
if values
|
||||
}
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
search_query = (query or "").strip()
|
||||
@@ -3176,16 +3407,20 @@ def create_app(
|
||||
window_id=window_id,
|
||||
scene=scene,
|
||||
user_id=user_id,
|
||||
filters=filters,
|
||||
)
|
||||
raw = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
user_id=user_id,
|
||||
scene=scene,
|
||||
# Layers are chat-store rows: `scene="all"` would still add
|
||||
# every working/coding experience to a layer-filtered page.
|
||||
scene=None if chosen_layers else scene,
|
||||
statuses=list(statuses),
|
||||
include_sources=[source] if source else None,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
order="desc",
|
||||
**filters,
|
||||
)
|
||||
page, skipped = HySdkMemoryPage.from_sdk_skipping_invalid(
|
||||
raw, limit=limit, offset=offset
|
||||
@@ -3197,6 +3432,51 @@ def create_app(
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_list_failed") from None
|
||||
|
||||
@app.get(
|
||||
"/api/v2/memory-library/layers",
|
||||
response_model=SdkLayerCounts,
|
||||
dependencies=strict,
|
||||
)
|
||||
async def sdk_memory_layers(
|
||||
statuses: list[str] | None = Query(default=None), # noqa: B008
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
) -> SdkLayerCounts:
|
||||
"""How many memories each SDK layer holds for one user.
|
||||
|
||||
One `list_memories(layers=[layer], limit=1)` per layer, read for its
|
||||
total only. Without `statuses` every layer counts its active rows,
|
||||
and `l1_raw` counts active and shadow raws: pro shadows a raw once
|
||||
it is extracted, and both are the user's kept conversations.
|
||||
"""
|
||||
chosen = _library_values(statuses or [], MEMORY_STATUSES, "memory_status_unknown")
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
|
||||
def count(layer: str) -> int:
|
||||
layer_statuses = chosen or (list(RAW_STATUSES) if layer == "l1_raw" else ["active"])
|
||||
raw = client.list_memories(
|
||||
user_id=user_id,
|
||||
layers=[layer],
|
||||
statuses=layer_statuses,
|
||||
limit=1,
|
||||
offset=0,
|
||||
order="desc",
|
||||
)
|
||||
return _list_total(raw)
|
||||
|
||||
counts = await asyncio.to_thread(
|
||||
lambda: {layer: count(layer) for layer in MEMORY_LAYERS}
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_list_failed") from None
|
||||
return SdkLayerCounts(
|
||||
layers=counts,
|
||||
total=sum(counts.values()),
|
||||
statuses=tuple(chosen) if chosen else None,
|
||||
)
|
||||
|
||||
@app.get(
|
||||
"/api/v2/memories/graph",
|
||||
response_model=MemoryGraphResponse,
|
||||
@@ -3207,13 +3487,30 @@ def create_app(
|
||||
offset: int = Query(default=0, ge=0),
|
||||
source: str | None = Query(default=None, min_length=1),
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
layers: list[str] | None = Query(default=None), # noqa: B008
|
||||
session_ids: list[str] | None = Query(default=None), # noqa: B008
|
||||
exclude_sources: list[str] | None = Query(default=None), # noqa: B008
|
||||
) -> MemoryGraphResponse:
|
||||
"""The memory map on SDK authority: SDK memories plus the Kuzu overlay.
|
||||
|
||||
Memories come from the public client; only the overlay reads Kuzu
|
||||
(`hy_graph_projection`, the visualization exception). Shape matches
|
||||
the legacy map so the same view draws both.
|
||||
|
||||
`layers` maps the chat store's chosen layers only (as on the library);
|
||||
`session_ids` and `exclude_sources` narrow the listed memories. Each
|
||||
may repeat or be comma separated. The overlay is not filtered.
|
||||
"""
|
||||
chosen_layers = _library_values(layers or [], MEMORY_LAYERS, "memory_layer_unknown")
|
||||
filters = {
|
||||
name: values
|
||||
for name, values in (
|
||||
("layers", chosen_layers),
|
||||
("session_ids", split_csv(session_ids)),
|
||||
("exclude_sources", split_csv(exclude_sources)),
|
||||
)
|
||||
if values
|
||||
}
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
# The map visualizes the active source scope, independently of the
|
||||
@@ -3221,17 +3518,20 @@ def create_app(
|
||||
raw = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
user_id=user_id,
|
||||
scene="all",
|
||||
scene=None if chosen_layers else "all",
|
||||
statuses=["active"],
|
||||
include_sources=[source] if source else None,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
order="desc",
|
||||
**filters,
|
||||
)
|
||||
page, skipped = HySdkMemoryPage.from_sdk_skipping_invalid(
|
||||
raw, limit=limit, offset=offset
|
||||
)
|
||||
_log_skipped_sdk_memories(skipped)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_list_failed") from None
|
||||
vdb = raw.get("vdb") if isinstance(raw, dict) else None
|
||||
@@ -3355,6 +3655,27 @@ def create_app(
|
||||
raise _sdk_http_error(error, "hy_memory_profile_read_failed") from None
|
||||
return SdkUserProfileResponse.model_validate(profile.model_dump())
|
||||
|
||||
@app.get(
|
||||
"/api/v2/user-profile/versions",
|
||||
response_model=SdkProfileVersionsResponse,
|
||||
dependencies=strict,
|
||||
)
|
||||
async def sdk_user_profile_versions(
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
) -> SdkProfileVersionsResponse:
|
||||
"""The long-term profile's L0 version chains (`get_user_basic_info`).
|
||||
|
||||
Each chain is one L0 head with its versions latest first, so a value
|
||||
that replaced a better one can be found and deleted by id.
|
||||
"""
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
value = await asyncio.to_thread(client.get_user_basic_info, user_id=user_id)
|
||||
versions = HySdkProfileVersions.from_sdk(value)
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_profile_read_failed") from None
|
||||
return SdkProfileVersionsResponse(user_id=user_id, **versions.model_dump())
|
||||
|
||||
@app.get(
|
||||
"/api/v2/memories/{memory_id}",
|
||||
response_model=HySdkMemoryRecord,
|
||||
@@ -3487,25 +3808,69 @@ def create_app(
|
||||
dependencies=strict,
|
||||
)
|
||||
async def sdk_raw_library(
|
||||
source: str | None = Query(default=None, min_length=1),
|
||||
source: list[str] | None = Query(default=None), # noqa: B008
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN),
|
||||
exclude_sources: list[str] | None = Query(default=None), # noqa: B008
|
||||
session_id: list[str] | None = Query(default=None), # noqa: B008
|
||||
statuses: list[str] | None = Query(default=None), # noqa: B008
|
||||
after: float | None = Query(default=None, ge=0),
|
||||
before: float | None = Query(default=None, ge=0),
|
||||
order: Literal["desc", "asc"] = Query(default="desc"),
|
||||
) -> SdkRawPage:
|
||||
"""One page of a user's SDK raw turns: what was imported, as stored.
|
||||
|
||||
The SDK lists raw only when asked for `layers=["l1_raw"]`, and only
|
||||
active rows unless told otherwise. Pro shadows a raw once extraction
|
||||
succeeds, so a raw page reads active and shadow (`statuses` narrows
|
||||
it). `source` and `session_id` (a conversation) may repeat or be comma
|
||||
separated. `after`/`before` (unix seconds, against the conversation
|
||||
time, else the stored time) have no SDK list filter: they cut this
|
||||
page in Core and mark it `time_filtered`.
|
||||
"""
|
||||
chosen_statuses = _library_values(
|
||||
statuses or [], MEMORY_STATUSES, "memory_status_unknown"
|
||||
) or list(RAW_STATUSES)
|
||||
sources = split_csv(source)
|
||||
excluded = split_csv(exclude_sources)
|
||||
sessions = split_csv(session_id)
|
||||
try:
|
||||
client = await _sdk_client_handle(app).client()
|
||||
value = await asyncio.to_thread(
|
||||
client.list_memories,
|
||||
user_id=user_id,
|
||||
layers=["l1_raw"],
|
||||
include_sources=[source] if source else None,
|
||||
statuses=chosen_statuses,
|
||||
include_sources=sources or None,
|
||||
exclude_sources=excluded or None,
|
||||
session_ids=sessions or None,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
order=order,
|
||||
)
|
||||
page = HySdkRawPage.from_memory_list(value, limit=limit, offset=offset)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise _sdk_http_error(error, "hy_memory_raw_list_failed") from None
|
||||
return SdkRawPage(items=page.raw, total=page.total, offset=offset, limit=limit)
|
||||
records = page.raw
|
||||
time_filtered = after is not None or before is not None
|
||||
if time_filtered:
|
||||
records = tuple(
|
||||
record
|
||||
for record in records
|
||||
if (at := record.memory_at or record.gmt_created) is not None
|
||||
and (after is None or at >= after)
|
||||
and (before is None or at < before)
|
||||
)
|
||||
return SdkRawPage(
|
||||
items=tuple(SdkRawItem.of(record) for record in records),
|
||||
total=page.total,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
time_filtered=time_filtered,
|
||||
)
|
||||
|
||||
@app.post(
|
||||
"/api/v2/memories/ask",
|
||||
@@ -3528,7 +3893,7 @@ def create_app(
|
||||
)
|
||||
try:
|
||||
answer = await _sdk_request_answerer(app).answer(
|
||||
request.question, user_id=request.user_id
|
||||
request.question, user_id=request.user_id, **_ask_filters(request)
|
||||
)
|
||||
except MemoryAnswerError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from None
|
||||
@@ -3569,7 +3934,7 @@ def create_app(
|
||||
final: dict[str, object] | None = None
|
||||
try:
|
||||
async for event in answerer.answer_stream(
|
||||
request.question, user_id=request.user_id
|
||||
request.question, user_id=request.user_id, **_ask_filters(request)
|
||||
):
|
||||
if event.get("type") == "final":
|
||||
final = event
|
||||
@@ -3614,11 +3979,15 @@ def create_app(
|
||||
await _require_external_recall_scope(
|
||||
app, http_request, request.scope, request.workspace_path
|
||||
)
|
||||
# A caller's pass-through search options need the SDK target, which
|
||||
# carries them; without any, recall is the call it always was.
|
||||
options = request if request.search_overrides() else None
|
||||
target = (
|
||||
_recall_target(app)
|
||||
if request.user_id == DEFAULT_USER_ID
|
||||
else _sdk_recall_target(app, request.user_id)
|
||||
if request.user_id == DEFAULT_USER_ID and options is None
|
||||
else _sdk_recall_target(app, request.user_id, options)
|
||||
)
|
||||
raw_options = {"filters": options} if options is not None else {}
|
||||
access = external_access(http_request.headers)
|
||||
receipt = await recall_with_fallback(
|
||||
request,
|
||||
@@ -3627,7 +3996,12 @@ def create_app(
|
||||
has_memories=lambda: _library_has_memories(app, request.user_id),
|
||||
raw_sources=lambda: _recall_raw_sources(app, request, access, started),
|
||||
search_raw=lambda query, limit, sources: _search_sdk_raw(
|
||||
app, query, limit=limit, sources=sources, user_id=request.user_id
|
||||
app,
|
||||
query,
|
||||
limit=limit,
|
||||
sources=sources,
|
||||
user_id=request.user_id,
|
||||
**raw_options,
|
||||
),
|
||||
)
|
||||
event_store = getattr(app.state, "call_event_store", None)
|
||||
@@ -3917,6 +4291,11 @@ async def _require_external_recall_scope(
|
||||
access.require(tool, frozenset({scope}))
|
||||
|
||||
|
||||
def _ask_filters(request: AskMemoryRequest) -> dict[str, SdkSearchFilters]:
|
||||
"""The answerer's `filters` argument, only when the caller set any."""
|
||||
return {"filters": request} if request.search_overrides() else {}
|
||||
|
||||
|
||||
def _sdk_request_answerer(app: FastAPI) -> SdkMemoryAnswerer:
|
||||
return SdkMemoryAnswerer(
|
||||
client_handle=_sdk_client_handle(app),
|
||||
@@ -4093,6 +4472,25 @@ def _active_adapters(
|
||||
return adapters
|
||||
|
||||
|
||||
def _registration_at_grant(
|
||||
grant_store: SourceGrantStore,
|
||||
registration: SourceRegistration,
|
||||
user_id: str,
|
||||
) -> SourceRegistration:
|
||||
"""The registration with its candidate at `user_id`'s granted root, if any."""
|
||||
for receipt, root_path in grant_store.grants():
|
||||
if receipt.user_id == user_id and receipt.source_id == registration.descriptor.source_id:
|
||||
return replace(
|
||||
registration,
|
||||
candidate=replace(
|
||||
registration.candidate,
|
||||
path=Path(root_path),
|
||||
kind=RootCandidateKind.USER_SELECTED,
|
||||
),
|
||||
)
|
||||
return registration
|
||||
|
||||
|
||||
def _source_response(
|
||||
registration: SourceRegistration,
|
||||
receipt: SourceGrantReceipt | None,
|
||||
@@ -4177,12 +4575,21 @@ def _require_registration(app: FastAPI, source_id: str) -> SourceRegistration:
|
||||
return registration
|
||||
|
||||
|
||||
def _require_multi_user_source(source_id: str, user_id: str) -> None:
|
||||
"""Refuse another user's lane of a source that holds `local`'s roots only."""
|
||||
if user_id != DEFAULT_USER_ID and source_id not in MULTI_USER_SOURCES:
|
||||
raise HTTPException(status_code=422, detail="user_id_requires_multi_user_source")
|
||||
|
||||
|
||||
def _diagnostic_request_event(method: str, request_path: str) -> str | None:
|
||||
if method == "GET" or request_path in {"/api/v1/health", "/api/v1/runtime/identity"}:
|
||||
return None
|
||||
families = (
|
||||
("/api/v1/import", "import.request"),
|
||||
("/api/v1/sources", "source.request"),
|
||||
("/api/v2/import", "import.request"),
|
||||
("/api/v2/sources", "source.request"),
|
||||
("/api/v2/evolve", "memory.request"),
|
||||
("/api/v1/model", "model.request"),
|
||||
("/api/v1/memor", "memory.request"),
|
||||
("/api/v2/context", "recall.request"),
|
||||
@@ -4762,23 +5169,33 @@ async def _recall_model_ready(app: FastAPI) -> bool:
|
||||
return profile is not None and bool(profile.capabilities.model_available)
|
||||
|
||||
|
||||
def _sdk_recall_target(app: FastAPI, user_id: str) -> SdkRecallTarget:
|
||||
"""Recall over one user's SDK memories; `get` results are kept only if theirs."""
|
||||
def _sdk_recall_target(
|
||||
app: FastAPI,
|
||||
user_id: str,
|
||||
options: SdkSearchFilters | None = None,
|
||||
) -> SdkRecallTarget:
|
||||
"""Recall over one user's SDK memories; `get` results are kept only if theirs.
|
||||
|
||||
With a caller's pass-through `options`, the search carries them and a
|
||||
working/coding hit is read back from its own namespace.
|
||||
"""
|
||||
handle = _sdk_client_handle(app)
|
||||
namespaces: dict[str, dict[str, str]] = {}
|
||||
|
||||
async def owned(memory_id: str) -> HySdkMemoryRecord | None:
|
||||
memory = await _get_sdk_memory(handle, memory_id)
|
||||
memory = await _get_sdk_memory(handle, memory_id, **namespaces.get(memory_id, {}))
|
||||
return memory if memory is not None and owns(memory.user_id, user_id) else None
|
||||
|
||||
return SdkRecallTarget(
|
||||
semantic_search=lambda query: _search_sdk_memories(
|
||||
handle,
|
||||
query,
|
||||
user_id=user_id,
|
||||
agent_id="*",
|
||||
),
|
||||
get_memory=owned,
|
||||
)
|
||||
async def search(query: str) -> tuple[object, ...]:
|
||||
hits = await _search_sdk_memories(
|
||||
handle, query, user_id=user_id, agent_id="*", options=options
|
||||
)
|
||||
for hit in hits:
|
||||
if isinstance(hit, SemanticMemoryHit) and hit.variant is not None:
|
||||
namespaces[hit.memory_id] = {"kind": hit.kind, "variant": hit.variant}
|
||||
return hits
|
||||
|
||||
return SdkRecallTarget(semantic_search=search, get_memory=owned)
|
||||
|
||||
|
||||
async def _library_has_memories(app: FastAPI, user_id: str = DEFAULT_USER_ID) -> bool:
|
||||
@@ -4846,33 +5263,57 @@ async def _search_sdk_raw(
|
||||
limit: int,
|
||||
sources: frozenset[str] | None,
|
||||
user_id: str = DEFAULT_USER_ID,
|
||||
filters: SdkSearchFilters | None = None,
|
||||
) -> tuple[HySdkRawHit, ...]:
|
||||
"""The imported conversations nearest the query: the SDK raw layer."""
|
||||
"""The imported conversations nearest the query: the SDK raw layer.
|
||||
|
||||
`filters` are a caller's pass-through options: its floor and row filters
|
||||
apply, and its `include_sources` only narrow `sources`, never widen them.
|
||||
"""
|
||||
from one_memory.context_routes import raw_lane_overrides
|
||||
|
||||
include = sorted(sources) if sources else None
|
||||
if filters is not None and filters.include_sources is not None:
|
||||
include = [
|
||||
source
|
||||
for source in dict.fromkeys(filters.include_sources)
|
||||
if sources is None or source in sources
|
||||
]
|
||||
if not include:
|
||||
return ()
|
||||
client = await _sdk_client_handle(app).client()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
query,
|
||||
user_ids=[user_id],
|
||||
agent_ids=[],
|
||||
include_sources=sorted(sources) if sources else None,
|
||||
limit=limit,
|
||||
min_score=0.0,
|
||||
schema_min_score=0.0,
|
||||
schema_limit=0,
|
||||
intention_limit=0,
|
||||
scene="chat",
|
||||
reader=HY_MEMORY_READER,
|
||||
layers=["l1_raw"],
|
||||
**{
|
||||
"user_ids": [user_id],
|
||||
"agent_ids": [],
|
||||
"include_sources": include,
|
||||
"limit": limit,
|
||||
"min_score": 0.0,
|
||||
"schema_min_score": 0.0,
|
||||
"schema_limit": 0,
|
||||
"intention_limit": 0,
|
||||
"scene": "chat",
|
||||
"reader": HY_MEMORY_READER,
|
||||
"layers": ["l1_raw"],
|
||||
**(raw_lane_overrides(filters) if filters is not None else {}),
|
||||
},
|
||||
)
|
||||
return parse_sdk_raw_search(raw)
|
||||
hits = parse_sdk_raw_search(raw)
|
||||
if filters is None:
|
||||
return hits
|
||||
return tuple(hit for hit in hits if filters.passes_floor("raw", hit.score))
|
||||
|
||||
|
||||
def _sdk_client_handle(app: FastAPI) -> SdkClientHandle:
|
||||
handle = cast(SdkClientHandle | None, app.state.sdk_client_handle)
|
||||
if handle is None:
|
||||
handle = SdkClientHandle(
|
||||
profile=builtin_retrieval_snapshot(),
|
||||
api_key=None,
|
||||
# Reads open the embedding space writes fill: the active profile's,
|
||||
# the built-in one only when none is active (#242).
|
||||
handle = ActiveProfileReadHandle(
|
||||
profiles=_model_profile_store(app),
|
||||
secrets=_secret_store(app),
|
||||
data_dir=cast(Settings, app.state.settings).data_dir,
|
||||
)
|
||||
app.state.sdk_client_handle = handle
|
||||
@@ -4944,6 +5385,30 @@ def _library_search_windows(app: FastAPI) -> LibrarySearchWindows:
|
||||
return cast(LibrarySearchWindows, windows)
|
||||
|
||||
|
||||
def _library_values(values: list[str], allowed: tuple[str, ...], code: str) -> list[str]:
|
||||
"""Repeated or comma-separated query values, each one the SDK knows."""
|
||||
chosen = split_csv(values)
|
||||
if any(value not in allowed for value in chosen):
|
||||
raise HTTPException(status_code=422, detail=code)
|
||||
return chosen
|
||||
|
||||
|
||||
def _list_total(raw: object) -> int:
|
||||
"""A chat-store `list_memories` reply's total: its vector and graph buckets."""
|
||||
if not isinstance(raw, dict):
|
||||
raise HySdkReadError("hy_memory_list_shape_unknown")
|
||||
total = 0
|
||||
for name in ("vdb", "graph"):
|
||||
bucket = raw.get(name)
|
||||
if bucket is None:
|
||||
continue
|
||||
value = bucket.get("total") if isinstance(bucket, dict) else None
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
||||
raise HySdkReadError("hy_memory_list_shape_unknown")
|
||||
total += value
|
||||
return total
|
||||
|
||||
|
||||
async def _library_search_page(
|
||||
app: FastAPI,
|
||||
client: Any,
|
||||
@@ -4956,6 +5421,7 @@ async def _library_search_page(
|
||||
window_id: str | None,
|
||||
scene: str = "all",
|
||||
user_id: str = DEFAULT_USER_ID,
|
||||
filters: dict[str, list[str]] | None = None,
|
||||
) -> HySdkMemoryPage:
|
||||
"""One page of a keyword search, cut from a cached result window.
|
||||
|
||||
@@ -4965,11 +5431,16 @@ async def _library_search_page(
|
||||
contain every word, and caches those matches under a `window_id`. Every
|
||||
page of that search is a slice of the same matches, read fresh from the
|
||||
SDK so edits and deletes show. The whole library is never listed in Core.
|
||||
|
||||
`filters` (layers, exclude_sources, session_ids, agent_ids) reach the SDK
|
||||
search and are part of the window's identity. A layer filter keeps chat
|
||||
matches only: the SDK applies layers to the chat lane alone.
|
||||
"""
|
||||
windows = _library_search_windows(app)
|
||||
filters = filters or {}
|
||||
# The user is part of the window's identity: one user's `window_id` must
|
||||
# never cut a page from another user's matches.
|
||||
key = window_key(query, source, statuses, scene, user_id)
|
||||
key = window_key(query, source, statuses, scene, user_id, filters)
|
||||
window = windows.get(window_id, key) if window_id else None
|
||||
window_reset = window_id is not None and window is None
|
||||
if window is None:
|
||||
@@ -4980,7 +5451,7 @@ async def _library_search_page(
|
||||
scene="all",
|
||||
rerank=False,
|
||||
user_ids=[user_id],
|
||||
agent_ids=[],
|
||||
agent_ids=filters.get("agent_ids", []),
|
||||
include_sources=[source] if source else None,
|
||||
limit=fetch,
|
||||
min_score=0.0,
|
||||
@@ -4992,8 +5463,11 @@ async def _library_search_page(
|
||||
# recall-time lane the list never shows.
|
||||
intention_limit=0,
|
||||
reader=HY_MEMORY_READER,
|
||||
**{name: values for name, values in filters.items() if name != "agent_ids"},
|
||||
)
|
||||
hits, held = library_keyword_hits(search_raw, query)
|
||||
if "layers" in filters:
|
||||
hits = tuple(hit for hit in hits if hit.kind == "chat")
|
||||
if scene != "all":
|
||||
# One scene: keep that scene's matches from the same fused window.
|
||||
hits = tuple(
|
||||
@@ -5036,22 +5510,33 @@ async def _search_sdk_memories(
|
||||
user_id: str,
|
||||
agent_id: str,
|
||||
limit: int = 12,
|
||||
options: SdkSearchFilters | None = None,
|
||||
) -> tuple[object, ...]:
|
||||
client = await handle.client()
|
||||
overrides = options.search_overrides() if options is not None else {}
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
query,
|
||||
user_ids=[user_id],
|
||||
agent_ids=[] if agent_id == "*" else [agent_id],
|
||||
limit=limit,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=limit,
|
||||
intention_limit=limit,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
**{
|
||||
"user_ids": [user_id],
|
||||
"agent_ids": [] if agent_id == "*" else [agent_id],
|
||||
"limit": limit,
|
||||
"min_score": sdk_relevance_floor(),
|
||||
"schema_min_score": sdk_relevance_floor(),
|
||||
"schema_limit": limit,
|
||||
"intention_limit": limit,
|
||||
"scene": "normal",
|
||||
"reader": HY_MEMORY_READER,
|
||||
**overrides,
|
||||
},
|
||||
)
|
||||
if options is None or not overrides:
|
||||
return parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
return tuple(
|
||||
hit
|
||||
for hit in parse_sdk_lane_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
if options.passes_floor(cast(SearchLane, hit.lane), hit.score)
|
||||
)
|
||||
return parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
|
||||
|
||||
def _sdk_http_error(error: Exception, fallback: str) -> HTTPException:
|
||||
|
||||
@@ -584,6 +584,7 @@ def _sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
|
||||
}
|
||||
else:
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
from one_memory.sdk_data_plane_migration import (
|
||||
MemoryAuthority,
|
||||
SdkDataPlaneMigrationStore,
|
||||
@@ -595,14 +596,14 @@ def _sdk_migration_status(options: Sequence[str], stream: TextIO) -> int:
|
||||
|
||||
authority = SdkDataPlaneMigrationStore(database_path).authority()
|
||||
if authority is MemoryAuthority.SDK:
|
||||
profile = read_active_profile(database_path)
|
||||
profile = read_active_profile(database_path) or builtin_retrieval_snapshot()
|
||||
leftovers = bool(
|
||||
legacy_tables_present(database_path)
|
||||
or abandoned_staging_present(
|
||||
data_dir=settings.data_dir,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
)
|
||||
or (profile is not None and legacy_vector_collections(settings.data_dir, profile))
|
||||
or legacy_vector_collections(settings.data_dir, profile)
|
||||
)
|
||||
state = "required" if leftovers else "not_required"
|
||||
else:
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from one_memory.sdk_search_options import SdkSearchOptions
|
||||
from one_memory.user_scope import DEFAULT_USER_ID, UserId
|
||||
|
||||
ContextKind = Literal["raw", "memory"]
|
||||
@@ -49,12 +50,17 @@ class ContextSearchRequest(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class SdkContextSearchRequest(ContextSearchRequest):
|
||||
class SdkContextSearchRequest(ContextSearchRequest, SdkSearchOptions):
|
||||
"""`POST /api/v2/context/search`: the three SDK recall lanes are set apart.
|
||||
|
||||
`limit` is the normal lane's top-k and the cap on returned items;
|
||||
`schema_limit` and `intention_limit` are the SDK's schema (L6) and
|
||||
proactive intention (L7) lanes. A caller's values are passed through.
|
||||
|
||||
The `SdkSearchOptions` fields reach the memory lane's `client.search`;
|
||||
the raw lane takes the floor and the source, session, agent and time
|
||||
filters (never wider than the caller's raw authority). Absent, both
|
||||
lanes make the call they always made.
|
||||
"""
|
||||
|
||||
limit: int = Field(default=10, ge=1, le=20, strict=True)
|
||||
|
||||
@@ -24,14 +24,57 @@ from one_memory.hy_runtime_config import HY_MEMORY_READER
|
||||
from one_memory.hy_sdk_runtime import sdk_relevance_floor
|
||||
from one_memory.hy_sdk_types import (
|
||||
HySdkMemoryRecord,
|
||||
parse_sdk_lane_search,
|
||||
parse_sdk_raw_search,
|
||||
parse_sdk_search,
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.sdk_search_options import SdkSearchFilters, SearchLane
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
from one_memory.user_scope import owns
|
||||
|
||||
CALL_EVENT_RESPONSE_BUDGET_MS = 250
|
||||
|
||||
#: Pass-through filters the raw lane honours. Scene, layers, reader and the
|
||||
#: schema/working floors describe memory lanes; the raw lane is always the
|
||||
#: chat store's L1 raw layer.
|
||||
_RAW_LANE_FILTERS = ("created_after", "session_ids", "agent_ids", "exclude_sources", "rerank")
|
||||
|
||||
|
||||
def raw_lane_sources(
|
||||
allowed: Sequence[str] | frozenset[str] | None,
|
||||
requested: Sequence[str] | None,
|
||||
) -> list[str] | None:
|
||||
"""The raw lane's source filter: the caller's, never wider than its authority.
|
||||
|
||||
`allowed` empty or None means every source (the call Core always made).
|
||||
None back means nothing is left to search.
|
||||
"""
|
||||
allowed_list = list(allowed or ())
|
||||
if requested is None:
|
||||
return allowed_list
|
||||
chosen = [
|
||||
source for source in dict.fromkeys(requested) if not allowed_list or source in allowed_list
|
||||
]
|
||||
return chosen or None
|
||||
|
||||
|
||||
def raw_lane_overrides(value: SdkSearchFilters) -> dict[str, object]:
|
||||
"""The caller's raw-lane overrides: its floor and its row filters."""
|
||||
overrides = value.filter_kwargs()
|
||||
found = {name: overrides[name] for name in _RAW_LANE_FILTERS if name in overrides}
|
||||
floor = value.lane_floor("raw")
|
||||
if floor is not None:
|
||||
found["min_score"] = floor
|
||||
return found
|
||||
|
||||
|
||||
def hit_namespace(hit: SemanticMemoryHit) -> dict[str, str]:
|
||||
"""The `get` namespace of a hit: nothing for chat, kind and variant otherwise."""
|
||||
if hit.kind in {"experience", "fact"} and hit.variant is not None:
|
||||
return {"kind": hit.kind, "variant": hit.variant}
|
||||
return {}
|
||||
|
||||
|
||||
def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
"""SDK-native context: no Evidence/Governance ids or fabricated revisions."""
|
||||
@@ -71,22 +114,30 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
async def raw_lane() -> tuple[SdkRawContextItem, ...]:
|
||||
if "raw" not in value.kinds or not authority.raw_allowed:
|
||||
return ()
|
||||
sources = raw_lane_sources(authority.raw_sources, value.include_sources)
|
||||
if sources is None:
|
||||
return ()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
value.query,
|
||||
user_ids=[value.user_id],
|
||||
agent_ids=[],
|
||||
include_sources=list(authority.raw_sources or ()),
|
||||
limit=value.limit,
|
||||
min_score=0.0,
|
||||
schema_min_score=0.0,
|
||||
schema_limit=0,
|
||||
intention_limit=0,
|
||||
scene="chat",
|
||||
reader=HY_MEMORY_READER,
|
||||
layers=["l1_raw"],
|
||||
**{
|
||||
"user_ids": [value.user_id],
|
||||
"agent_ids": [],
|
||||
"include_sources": sources,
|
||||
"limit": value.limit,
|
||||
"min_score": 0.0,
|
||||
"schema_min_score": 0.0,
|
||||
"schema_limit": 0,
|
||||
"intention_limit": 0,
|
||||
"scene": "chat",
|
||||
"reader": HY_MEMORY_READER,
|
||||
"layers": ["l1_raw"],
|
||||
**raw_lane_overrides(value),
|
||||
},
|
||||
)
|
||||
hits = tuple(
|
||||
hit for hit in parse_sdk_raw_search(raw) if value.passes_floor("raw", hit.score)
|
||||
)
|
||||
hits = parse_sdk_raw_search(raw)
|
||||
return tuple(
|
||||
SdkRawContextItem(
|
||||
raw_id=hit.raw_id,
|
||||
@@ -106,23 +157,37 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
async def memory_lane() -> tuple[SdkMemoryContextItem, ...]:
|
||||
if "memory" not in value.kinds or not authority.memory_allowed:
|
||||
return ()
|
||||
overrides = value.search_overrides()
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
value.query,
|
||||
user_ids=[value.user_id],
|
||||
agent_ids=[],
|
||||
limit=value.limit,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=value.schema_limit,
|
||||
intention_limit=value.intention_limit,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
**{
|
||||
"user_ids": [value.user_id],
|
||||
"agent_ids": [],
|
||||
"limit": value.limit,
|
||||
"min_score": sdk_relevance_floor(),
|
||||
"schema_min_score": sdk_relevance_floor(),
|
||||
"schema_limit": value.schema_limit,
|
||||
"intention_limit": value.intention_limit,
|
||||
"scene": "normal",
|
||||
"reader": HY_MEMORY_READER,
|
||||
**overrides,
|
||||
},
|
||||
)
|
||||
hits = (
|
||||
tuple(
|
||||
hit
|
||||
for hit in parse_sdk_lane_search(
|
||||
raw, error_code="hy_memory_search_shape_unknown"
|
||||
)
|
||||
if value.passes_floor(cast(SearchLane, hit.lane), hit.score)
|
||||
)
|
||||
if overrides
|
||||
else parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
)
|
||||
hits = parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
items: list[SdkMemoryContextItem] = []
|
||||
for hit in hits:
|
||||
record = await asyncio.to_thread(client.get, hit.memory_id)
|
||||
record = await asyncio.to_thread(client.get, hit.memory_id, **hit_namespace(hit))
|
||||
memory = None if record is None else HySdkMemoryRecord.from_sdk(record)
|
||||
# `get` takes no user: a hit is kept only if it is the caller's.
|
||||
if (
|
||||
|
||||
@@ -217,6 +217,33 @@ def builtin_retrieval_snapshot() -> ModelProfileSnapshot:
|
||||
)
|
||||
|
||||
|
||||
def retrieval_view(profile: ModelProfileSnapshot) -> ModelProfileSnapshot:
|
||||
"""The read side of `profile`: its embedding space, and no generation.
|
||||
|
||||
Reads must open the collection writes fill (#242), and the collection is
|
||||
named by the embedding only. Generation stays the built-in placeholder, so
|
||||
a read never needs the generation model. A remote embedding still needs
|
||||
the profile's API key, which `build_sdk_config` uses as the embedding key.
|
||||
"""
|
||||
provider = profile.embedding_provider_kind or profile.provider_kind
|
||||
endpoint: str | None = None
|
||||
if provider is not ProviderKind.LOCAL_ONNX:
|
||||
endpoint = profile.embedding_base_url
|
||||
if not endpoint:
|
||||
raise HySdkRuntimeError("hy_sdk_model_endpoint_missing")
|
||||
return builtin_retrieval_snapshot().model_copy(
|
||||
update={
|
||||
"profile_id": profile.profile_id,
|
||||
"revision": profile.revision,
|
||||
"embedding_provider_kind": provider,
|
||||
"embedding_model": profile.embedding_model,
|
||||
"embedding_base_url": endpoint,
|
||||
"embedding_dimensions": profile.embedding_dimensions,
|
||||
"has_api_key": profile.has_api_key,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def sdk_relevance_floor() -> float:
|
||||
raw = os.environ.get("ONE_MEMORY_HY_RELEVANCE_FLOOR", "").strip()
|
||||
if not raw:
|
||||
@@ -249,10 +276,8 @@ def build_sdk_config(
|
||||
embedding_key = None
|
||||
dimensions = profile.embedding_dimensions or BUILTIN_EMBEDDING_DIMENSIONS
|
||||
else:
|
||||
embedding_endpoint = _openai_endpoint(
|
||||
embedding_provider,
|
||||
profile.embedding_base_url or profile.base_url,
|
||||
)
|
||||
# Never the generation endpoint: the collection name hashes this one.
|
||||
embedding_endpoint = _openai_endpoint(embedding_provider, profile.embedding_base_url)
|
||||
embedding_key = "ollama" if embedding_provider is ProviderKind.OLLAMA else api_key
|
||||
if embedding_provider is ProviderKind.OPENAI_COMPATIBLE and not embedding_key:
|
||||
raise HySdkRuntimeError("hy_sdk_embedding_api_key_missing")
|
||||
@@ -467,7 +492,7 @@ def vector_collection_bases(profile: ModelProfileSnapshot) -> tuple[str, tuple[s
|
||||
endpoint, dimensions = None, profile.embedding_dimensions or BUILTIN_EMBEDDING_DIMENSIONS
|
||||
legacy_endpoint = "local-onnx"
|
||||
else:
|
||||
endpoint = _openai_endpoint(provider, profile.embedding_base_url or profile.base_url)
|
||||
endpoint = _openai_endpoint(provider, profile.embedding_base_url)
|
||||
dimensions = profile.embedding_dimensions
|
||||
legacy_endpoint = endpoint
|
||||
legacy_model = profile.embedding_model or "bge-small-zh-v1.5"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Literal, NamedTuple, cast
|
||||
@@ -391,6 +392,8 @@ class HySdkRawRecord(BaseModel):
|
||||
memory_at: int | None = None
|
||||
gmt_created: int | None = None
|
||||
custom: dict[str, object] = Field(default_factory=dict)
|
||||
#: active, or shadow once pro extracted it; None when the SDK gave none.
|
||||
status: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_sdk(cls, raw: object) -> HySdkRawRecord:
|
||||
@@ -417,11 +420,34 @@ class HySdkRawRecord(BaseModel):
|
||||
raw.get("gmt_created") if isinstance(raw.get("gmt_created"), int) else None
|
||||
),
|
||||
custom=dict(custom),
|
||||
status=str(raw["status"]) if isinstance(raw.get("status"), str) else None,
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise HySdkReadError("hy_memory_raw_shape_unknown") from error
|
||||
|
||||
|
||||
_RAW_TURN_LINE = re.compile(r"^\[(user|assistant|tool|system)\]:\s?(.*)$")
|
||||
|
||||
|
||||
def raw_turn_messages(content: str) -> tuple[tuple[str, str], ...]:
|
||||
"""The `(role, text)` messages of one SDK raw turn.
|
||||
|
||||
The SDK stores a turn as `[user]: …\\n[assistant]: …`; a line without a
|
||||
role prefix continues the message above it. Content that names no role
|
||||
at all is one message of unknown role (`""`).
|
||||
"""
|
||||
messages: list[list[str]] = []
|
||||
for line in content.splitlines():
|
||||
match = _RAW_TURN_LINE.match(line)
|
||||
if match is not None:
|
||||
messages.append([match.group(1), match.group(2)])
|
||||
elif messages:
|
||||
messages[-1][1] += "\n" + line
|
||||
else:
|
||||
messages.append(["", line])
|
||||
return tuple((role, text) for role, text in messages)
|
||||
|
||||
|
||||
class HySdkRawHit(HySdkRawRecord):
|
||||
score: float = Field(ge=0, le=1)
|
||||
|
||||
@@ -516,6 +542,96 @@ class HySdkUserProfile(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class HySdkProfileVersion(BaseModel):
|
||||
"""One version on an L0 basic-info chain."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
memory_id: str
|
||||
content: str
|
||||
#: Conversation time as the SDK renders it on a chain (epoch or text).
|
||||
memory_at: int | str | None = None
|
||||
gmt_created: int | None = None
|
||||
#: The chain head: the version the profile shows now.
|
||||
current: bool = False
|
||||
|
||||
|
||||
class HySdkProfileChain(BaseModel):
|
||||
"""One L0 basic-info head and its versions, latest first."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
memory_id: str
|
||||
content: str
|
||||
status: str
|
||||
memory_at: int | None = None
|
||||
gmt_created: int | None = None
|
||||
#: The structured fields this head holds (`basic_info_kv`).
|
||||
basic_info: dict[str, object] = Field(default_factory=dict)
|
||||
versions: tuple[HySdkProfileVersion, ...]
|
||||
|
||||
|
||||
class HySdkProfileVersions(BaseModel):
|
||||
"""`get_user_basic_info`: the long-term profile's L0 version chains."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
request_id: str
|
||||
chains: tuple[HySdkProfileChain, ...]
|
||||
elapsed_ms: float = Field(ge=0)
|
||||
|
||||
@classmethod
|
||||
def from_sdk(cls, raw: object) -> HySdkProfileVersions:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError("hy_memory_profile_versions_shape_unknown")
|
||||
request_id = raw.get("request_id")
|
||||
rows = raw.get("memories")
|
||||
elapsed_ms = raw.get("elapsed_ms")
|
||||
if (
|
||||
not isinstance(request_id, str)
|
||||
or not isinstance(rows, list)
|
||||
or not isinstance(elapsed_ms, int | float)
|
||||
or isinstance(elapsed_ms, bool)
|
||||
):
|
||||
raise HySdkReadError("hy_memory_profile_versions_shape_unknown")
|
||||
chains: list[HySdkProfileChain] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, Mapping) or not isinstance(row.get("memory_id"), str):
|
||||
raise HySdkReadError("hy_memory_profile_versions_shape_unknown")
|
||||
head_id = str(row["memory_id"])
|
||||
chain = row.get("evolution_chain")
|
||||
entries = chain if isinstance(chain, list) and chain else [{**row, "node_id": head_id}]
|
||||
versions = tuple(
|
||||
HySdkProfileVersion(
|
||||
memory_id=str(entry.get("node_id") or entry.get("memory_id")),
|
||||
content=str(entry.get("content") or ""),
|
||||
memory_at=(
|
||||
entry.get("memory_at")
|
||||
if isinstance(entry.get("memory_at"), int | str)
|
||||
and not isinstance(entry.get("memory_at"), bool)
|
||||
else None
|
||||
),
|
||||
gmt_created=_epoch(entry.get("gmt_created")),
|
||||
current=str(entry.get("node_id") or entry.get("memory_id")) == head_id,
|
||||
)
|
||||
for entry in entries
|
||||
if isinstance(entry, Mapping) and (entry.get("node_id") or entry.get("memory_id"))
|
||||
)
|
||||
basic_info = row.get("basic_info_kv")
|
||||
chains.append(
|
||||
HySdkProfileChain(
|
||||
memory_id=head_id,
|
||||
content=str(row.get("content") or ""),
|
||||
status=str(row.get("status") or "active"),
|
||||
memory_at=_epoch(row.get("memory_at")),
|
||||
gmt_created=_epoch(row.get("gmt_created")),
|
||||
basic_info=dict(basic_info) if isinstance(basic_info, Mapping) else {},
|
||||
versions=versions,
|
||||
)
|
||||
)
|
||||
return cls(request_id=request_id, chains=tuple(chains), elapsed_ms=float(elapsed_ms))
|
||||
|
||||
|
||||
def parse_sdk_search(raw: object, *, error_code: str) -> tuple[SemanticMemoryHit, ...]:
|
||||
rows = _reply_rows(raw, error_code)
|
||||
found: list[SemanticMemoryHit] = []
|
||||
@@ -542,6 +658,120 @@ def parse_sdk_search(raw: object, *, error_code: str) -> tuple[SemanticMemoryHit
|
||||
return tuple(found)
|
||||
|
||||
|
||||
def parse_sdk_lane_search(raw: object, *, error_code: str) -> tuple[SemanticMemoryHit, ...]:
|
||||
"""Every scene's search reply as hits that name their lane and namespace.
|
||||
|
||||
`parse_sdk_search` reads the chat reply only and forgets which lane a hit
|
||||
came from. A caller may now pick the scene and a floor per lane, so this
|
||||
reads all three reply shapes:
|
||||
|
||||
- chat: `memories` as `{schema, proactive, normal}` (or one list, normal);
|
||||
- working/coding: `facts` and `experiences` of that variant;
|
||||
- all: one fused `memories` list whose rows carry `scene`, `kind` and
|
||||
their own lane's score (`lane_score`). The fused `score` is a rank
|
||||
fusion (1/(60+rank)), not a relevance, so the lane score is reported.
|
||||
|
||||
Highest score first, first occurrence of an id kept.
|
||||
"""
|
||||
if not isinstance(raw, Mapping):
|
||||
raise HySdkReadError(error_code)
|
||||
scene = raw.get("scene")
|
||||
found: list[SemanticMemoryHit] = []
|
||||
if scene == "all":
|
||||
rows = raw.get("memories")
|
||||
if not isinstance(rows, list) or any(not isinstance(row, Mapping) for row in rows):
|
||||
raise HySdkReadError(error_code)
|
||||
for row in rows:
|
||||
hit = _lane_hit(row, scene=str(row.get("scene") or "chat"), score_key="lane_score")
|
||||
if hit is not None:
|
||||
found.append(hit)
|
||||
elif scene in ("productivity", "working", "coding"):
|
||||
variant = raw.get("variant") if raw.get("variant") in {"working", "coding"} else None
|
||||
lane = variant or (scene if scene in {"working", "coding"} else "working")
|
||||
for key, kind in (("facts", "fact"), ("experiences", "experience")):
|
||||
rows = raw.get(key) or []
|
||||
if not isinstance(rows, list) or any(not isinstance(row, Mapping) for row in rows):
|
||||
raise HySdkReadError(error_code)
|
||||
for row in rows:
|
||||
hit = _lane_hit({**row, "kind": kind}, scene=lane, score_key="score")
|
||||
if hit is not None:
|
||||
found.append(hit)
|
||||
else:
|
||||
memories = raw.get("memories")
|
||||
if isinstance(memories, Mapping):
|
||||
if set(memories) != {"schema", "proactive", "normal"}:
|
||||
raise HySdkReadError(error_code)
|
||||
channels = tuple(memories.items())
|
||||
elif isinstance(memories, list):
|
||||
channels = (("normal", memories),)
|
||||
else:
|
||||
raise HySdkReadError(error_code)
|
||||
for lane, rows in channels:
|
||||
if not isinstance(rows, list) or any(not isinstance(row, Mapping) for row in rows):
|
||||
raise HySdkReadError(error_code)
|
||||
for row in rows:
|
||||
hit = _lane_hit(row, scene="chat", score_key="score", lane=str(lane))
|
||||
if hit is not None:
|
||||
found.append(hit)
|
||||
ordered: list[SemanticMemoryHit] = []
|
||||
seen: set[tuple[str, str, str | None]] = set()
|
||||
for hit in sorted(found, key=lambda value: value.score, reverse=True):
|
||||
identity = (hit.memory_id, hit.kind, hit.variant)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
ordered.append(hit)
|
||||
return tuple(ordered)
|
||||
|
||||
|
||||
def _lane_hit(
|
||||
row: Mapping[str, object],
|
||||
*,
|
||||
scene: str,
|
||||
score_key: str,
|
||||
lane: str | None = None,
|
||||
) -> SemanticMemoryHit | None:
|
||||
kind = str(row.get("kind")) if row.get("kind") in {"experience", "fact"} else "chat"
|
||||
if kind == "chat":
|
||||
memory_id = row.get("memory_id") or row.get("id")
|
||||
variant = None
|
||||
hit_lane = lane or "normal"
|
||||
else:
|
||||
id_key = "experience_id" if kind == "experience" else "fact_id"
|
||||
memory_id = row.get("memory_id") or row.get(id_key)
|
||||
variant = scene if scene in {"working", "coding"} else None
|
||||
if variant is None:
|
||||
return None
|
||||
hit_lane = variant
|
||||
if not isinstance(memory_id, str) or not memory_id:
|
||||
return None
|
||||
value = row.get(score_key)
|
||||
if not isinstance(value, int | float) or isinstance(value, bool):
|
||||
value = row.get("score")
|
||||
score = (
|
||||
max(0.0, min(1.0, float(value)))
|
||||
if isinstance(value, int | float) and not isinstance(value, bool)
|
||||
else 0.0
|
||||
)
|
||||
if kind != "chat":
|
||||
return SemanticMemoryHit(
|
||||
memory_id=memory_id, score=score, lane=hit_lane, kind=kind, variant=variant
|
||||
)
|
||||
content = row.get("content")
|
||||
member_ids, formatting_complete = _formatted_member_ids(row)
|
||||
return SemanticMemoryHit(
|
||||
memory_id=memory_id,
|
||||
score=score,
|
||||
formatted_content=(
|
||||
content.strip() if isinstance(content, str) and content.strip() else None
|
||||
),
|
||||
member_ids=member_ids,
|
||||
sdk_formatted=True,
|
||||
formatting_complete=formatting_complete,
|
||||
lane=hit_lane,
|
||||
)
|
||||
|
||||
|
||||
class LibraryHit(NamedTuple):
|
||||
"""One keyword match in a library search window, with its namespace."""
|
||||
|
||||
|
||||
@@ -28,7 +28,10 @@ from one_memory.hy_sdk_types import LibraryHit
|
||||
DEFAULT_WINDOW_TTL_SECONDS = 300.0
|
||||
DEFAULT_MAX_WINDOWS = 32
|
||||
|
||||
WindowKey = tuple[str, str | None, tuple[str, ...], str, str]
|
||||
#: (query, source, statuses, scene, user, other filters as sorted name/values pairs)
|
||||
WindowKey = tuple[
|
||||
str, str | None, tuple[str, ...], str, str, tuple[tuple[str, tuple[str, ...]], ...]
|
||||
]
|
||||
|
||||
|
||||
def window_key(
|
||||
@@ -37,8 +40,12 @@ def window_key(
|
||||
statuses: list[str],
|
||||
scene: str = "all",
|
||||
user_id: str = "local",
|
||||
filters: dict[str, list[str]] | None = None,
|
||||
) -> WindowKey:
|
||||
return (query, source, tuple(sorted(set(statuses))), scene, user_id)
|
||||
extra = tuple(
|
||||
sorted((name, tuple(sorted(set(values)))) for name, values in (filters or {}).items())
|
||||
)
|
||||
return (query, source, tuple(sorted(set(statuses))), scene, user_id, extra)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -160,7 +160,7 @@ def pending(db: sqlite3.Connection, *, limit: int = 100) -> list[str]:
|
||||
|
||||
|
||||
def _trigger_sql() -> list[tuple[str, str]]:
|
||||
"""(table it depends on, statement) for each trigger."""
|
||||
"""(table it is on, statement) for each trigger."""
|
||||
return [
|
||||
(
|
||||
"evidence",
|
||||
@@ -303,9 +303,61 @@ def install_triggers(db: sqlite3.Connection) -> None:
|
||||
for table, statement in _trigger_sql():
|
||||
if table not in present:
|
||||
continue
|
||||
if not _body_tables(statement) <= present:
|
||||
# SQLite resolves a trigger body only when it fires, so a body
|
||||
# naming a dropped table fails the write it rides on (#233: the
|
||||
# SDK cutover drops `governed_memory_evidence`, and every grant
|
||||
# write then failed). Not installed, and an old copy is removed.
|
||||
cursor.execute(f"DROP TRIGGER IF EXISTS {_trigger_name(statement)}")
|
||||
continue
|
||||
cursor.execute(statement)
|
||||
|
||||
|
||||
def _trigger_name(statement: str) -> str:
|
||||
return statement.split("IF NOT EXISTS", 1)[1].split()[0]
|
||||
|
||||
|
||||
def _body_tables(statement: str) -> set[str]:
|
||||
"""Tables the trigger body reads or writes besides the one it is on."""
|
||||
_head, _begin, body = statement.partition("BEGIN")
|
||||
words = body.replace("(", " ").replace(")", " ").split()
|
||||
return {
|
||||
name
|
||||
for keyword, name in zip(words, words[1:], strict=False)
|
||||
if keyword.upper() in {"FROM", "JOIN", "INTO"}
|
||||
}
|
||||
|
||||
|
||||
def drop_stale_triggers(database_path: Path) -> tuple[str, ...]:
|
||||
"""Drop this module's triggers whose body names a table that is gone.
|
||||
|
||||
Startup on SDK authority does not install the cache (it is read only on
|
||||
legacy authority), but a home migrated by an earlier build still carries
|
||||
the triggers it installed. One whose body reads a dropped table fails
|
||||
every write to the table it is on, so it is dropped here. Idempotent.
|
||||
"""
|
||||
with main_database(database_path).write() as db, db:
|
||||
cursor = db.cursor()
|
||||
cursor.row_factory = sqlite3.Row
|
||||
present = {
|
||||
str(row["name"])
|
||||
for row in cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
}
|
||||
installed = {
|
||||
str(row["name"])
|
||||
for row in cursor.execute("SELECT name FROM sqlite_master WHERE type='trigger'")
|
||||
}
|
||||
dropped = tuple(
|
||||
name
|
||||
for _table, statement in _trigger_sql()
|
||||
if (name := _trigger_name(statement)) in installed
|
||||
and not _body_tables(statement) <= present
|
||||
)
|
||||
for name in dropped:
|
||||
cursor.execute(f"DROP TRIGGER IF EXISTS {name}")
|
||||
return dropped
|
||||
|
||||
|
||||
def startup(database_path: Path) -> None:
|
||||
"""Idempotent: create the derived tables and triggers. No version change.
|
||||
|
||||
|
||||
@@ -564,7 +564,7 @@ def _embedding_request(
|
||||
secret: str | None,
|
||||
) -> tuple[str, str, dict[str, str], dict[str, object]]:
|
||||
provider = profile.embedding_provider_kind or profile.provider_kind
|
||||
base_url = (profile.embedding_base_url or profile.base_url or "").rstrip("/")
|
||||
base_url = (profile.embedding_base_url or "").rstrip("/")
|
||||
if provider is ProviderKind.OLLAMA:
|
||||
return (
|
||||
"POST",
|
||||
|
||||
@@ -97,7 +97,6 @@ class ModelProfileInput(BaseModel):
|
||||
)
|
||||
embedding_provider, resolved_embedding_base_url = resolve_embedding_endpoint(
|
||||
provider_kind=self.provider_kind,
|
||||
base_url=base_url,
|
||||
embedding_model=embedding_model,
|
||||
embedding_provider_kind=embedding_provider_kind,
|
||||
embedding_base_url=embedding_base_url,
|
||||
@@ -107,6 +106,12 @@ class ModelProfileInput(BaseModel):
|
||||
ProviderKind.ANTHROPIC,
|
||||
}:
|
||||
raise ValueError("embedding provider must be OpenAI-compatible, Ollama, or local ONNX")
|
||||
if (
|
||||
embedding_provider is not ProviderKind.LOCAL_ONNX
|
||||
and resolved_embedding_base_url is None
|
||||
):
|
||||
# `ModelProfileStore.create` refuses it with its own code.
|
||||
return self
|
||||
if (
|
||||
embedding_provider is not ProviderKind.LOCAL_ONNX
|
||||
and resolved_embedding_base_url is not None
|
||||
@@ -308,11 +313,12 @@ class ModelProfileStore:
|
||||
f"""
|
||||
UPDATE {table}
|
||||
SET embedding_provider_kind = provider_kind,
|
||||
embedding_base_url = base_url
|
||||
embedding_base_url = COALESCE(embedding_base_url, base_url)
|
||||
WHERE embedding_model IS NOT NULL
|
||||
AND embedding_provider_kind IS NULL
|
||||
"""
|
||||
)
|
||||
_pin_embedding_endpoints(connection, table)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO model_profile_versions (
|
||||
@@ -355,11 +361,11 @@ class ModelProfileStore:
|
||||
)
|
||||
embedding_provider_kind, embedding_base_url = resolve_embedding_endpoint(
|
||||
provider_kind=value.provider_kind,
|
||||
base_url=base_url,
|
||||
embedding_model=defaulted_model,
|
||||
embedding_provider_kind=defaulted_provider,
|
||||
embedding_base_url=defaulted_base_url,
|
||||
)
|
||||
require_embedding_base_url(embedding_provider_kind, embedding_base_url)
|
||||
try:
|
||||
with self._connect(writing=True) as connection:
|
||||
connection.execute(
|
||||
@@ -612,7 +618,6 @@ class ModelProfileStore:
|
||||
)
|
||||
embedding_provider_kind, embedding_base_url = resolve_embedding_endpoint(
|
||||
provider_kind=current.provider_kind,
|
||||
base_url=base_url,
|
||||
embedding_model=embedding_model,
|
||||
embedding_provider_kind=(
|
||||
patch.embedding_provider_kind
|
||||
@@ -627,6 +632,7 @@ class ModelProfileStore:
|
||||
)
|
||||
if embedding_provider_kind in {ProviderKind.HY_DEFAULT, ProviderKind.ANTHROPIC}:
|
||||
raise ValueError("embedding provider must be OpenAI-compatible, Ollama, or local ONNX")
|
||||
require_embedding_base_url(embedding_provider_kind, embedding_base_url)
|
||||
if (
|
||||
embedding_provider_kind is not ProviderKind.LOCAL_ONNX
|
||||
and embedding_base_url is not None
|
||||
@@ -1167,21 +1173,77 @@ def attach_disable_thinking(
|
||||
def resolve_embedding_endpoint(
|
||||
*,
|
||||
provider_kind: ProviderKind,
|
||||
base_url: str | None,
|
||||
embedding_model: str | None,
|
||||
embedding_provider_kind: ProviderKind | None,
|
||||
embedding_base_url: str | None,
|
||||
) -> tuple[ProviderKind | None, str | None]:
|
||||
"""The embedding provider and the endpoint exactly as given.
|
||||
|
||||
The endpoint never follows the generation `base_url`: the collection name
|
||||
hashes it, so a proxy or region edit of the generation endpoint moved
|
||||
writes to a new, empty collection, and providers without embeddings were
|
||||
sent embedding requests. `require_embedding_base_url` refuses a remote
|
||||
embedding without one.
|
||||
"""
|
||||
if embedding_model is None:
|
||||
return None, None
|
||||
if embedding_provider_kind is ProviderKind.LOCAL_ONNX:
|
||||
return ProviderKind.LOCAL_ONNX, None
|
||||
if embedding_provider_kind is None:
|
||||
return provider_kind, embedding_base_url or base_url
|
||||
return (
|
||||
embedding_provider_kind,
|
||||
embedding_base_url or DEFAULT_BASE_URLS.get(embedding_provider_kind),
|
||||
)
|
||||
return embedding_provider_kind or provider_kind, embedding_base_url
|
||||
|
||||
|
||||
class EmbeddingBaseUrlRequiredError(ValueError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("embedding_base_url_required")
|
||||
|
||||
|
||||
def require_embedding_base_url(provider: ProviderKind | None, base_url: str | None) -> None:
|
||||
"""A remote embedding must name its endpoint; the UI may offer the generation one."""
|
||||
if provider not in {None, ProviderKind.LOCAL_ONNX} and not base_url:
|
||||
raise EmbeddingBaseUrlRequiredError()
|
||||
|
||||
|
||||
def pinned_embedding_base_url(
|
||||
*,
|
||||
provider_kind: ProviderKind | str,
|
||||
base_url: str | None,
|
||||
embedding_model: str | None,
|
||||
embedding_provider_kind: ProviderKind | str | None,
|
||||
embedding_base_url: str | None,
|
||||
) -> str | None:
|
||||
"""The endpoint a stored profile's embedding resolved to before it was pinned.
|
||||
|
||||
The Runtime used `embedding_base_url or base_url`, and the provider default
|
||||
only when both were empty (no collection could have existed then). Writing
|
||||
that value keeps the endpoint, and so the collection name, byte-identical.
|
||||
"""
|
||||
provider = ProviderKind(embedding_provider_kind or provider_kind)
|
||||
if embedding_model is None or embedding_base_url or provider is ProviderKind.LOCAL_ONNX:
|
||||
return embedding_base_url
|
||||
return base_url or DEFAULT_BASE_URLS.get(provider)
|
||||
|
||||
|
||||
def _pin_embedding_endpoints(connection: sqlite3.Connection, table: str) -> None:
|
||||
"""Write the resolved endpoint into every remote embedding left without one."""
|
||||
rows = connection.execute(
|
||||
f"SELECT rowid, provider_kind, base_url, embedding_model, embedding_provider_kind "
|
||||
f"FROM {table} WHERE embedding_model IS NOT NULL AND embedding_base_url IS NULL "
|
||||
"AND COALESCE(embedding_provider_kind, provider_kind) != 'local_onnx'"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
connection.execute(
|
||||
f"UPDATE {table} SET embedding_base_url = ? WHERE rowid = ?",
|
||||
(
|
||||
pinned_embedding_base_url(
|
||||
provider_kind=row["provider_kind"],
|
||||
base_url=row["base_url"],
|
||||
embedding_model=row["embedding_model"],
|
||||
embedding_provider_kind=row["embedding_provider_kind"],
|
||||
embedding_base_url=None,
|
||||
),
|
||||
row["rowid"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def validate_model_base_url(value: str) -> None:
|
||||
|
||||
@@ -14,6 +14,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.hy_sdk_types import HySdkMemoryRecord
|
||||
from one_memory.memory_governance import GovernedMemory, MemoryStatus
|
||||
from one_memory.sdk_search_options import SdkSearchOptions
|
||||
from one_memory.semantic_deadline import (
|
||||
SemanticMemoryHit,
|
||||
SemanticSearch,
|
||||
@@ -50,7 +51,15 @@ class TruncationBudget(StrEnum):
|
||||
TIMEOUT = "timeout_budget"
|
||||
|
||||
|
||||
class RecallRequest(BaseModel):
|
||||
class RecallRequest(SdkSearchOptions):
|
||||
"""`POST /api/v1/recall`.
|
||||
|
||||
The `SdkSearchOptions` fields, `schema_limit` and `intention_limit` pass
|
||||
through to the SDK memory search (and the floor and row filters to the
|
||||
raw fallback). Absent, recall makes the search it always made: the
|
||||
relevance floor, schema and intention lanes sized to `limit`.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
query: str = Field(min_length=1, max_length=2_000)
|
||||
@@ -70,6 +79,17 @@ class RecallRequest(BaseModel):
|
||||
timeout_ms: int = Field(default=2_000, ge=50, le=30_000)
|
||||
#: Whose memories to recall. Omitted, it is `local`, the App's library.
|
||||
user_id: UserId = DEFAULT_USER_ID
|
||||
#: The SDK's schema (L6) and proactive intention (L7) lanes.
|
||||
schema_limit: int | None = Field(default=None, ge=0, le=20)
|
||||
intention_limit: int | None = Field(default=None, ge=0, le=20)
|
||||
|
||||
def search_overrides(self) -> Mapping[str, object]:
|
||||
values = dict(super().search_overrides())
|
||||
if self.schema_limit is not None:
|
||||
values["schema_limit"] = self.schema_limit
|
||||
if self.intention_limit is not None:
|
||||
values["intention_limit"] = self.intention_limit
|
||||
return values
|
||||
|
||||
|
||||
class RecallHit(BaseModel):
|
||||
|
||||
@@ -6,7 +6,7 @@ import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from contextlib import aclosing
|
||||
from typing import Any, Literal, Protocol
|
||||
from typing import Any, Literal, Protocol, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
@@ -14,7 +14,7 @@ from pydantic import BaseModel, ConfigDict
|
||||
from one_memory.http_client import create_async_client
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_READER
|
||||
from one_memory.hy_sdk_runtime import sdk_relevance_floor
|
||||
from one_memory.hy_sdk_types import HySdkMemoryRecord, parse_sdk_search
|
||||
from one_memory.hy_sdk_types import HySdkMemoryRecord, parse_sdk_lane_search, parse_sdk_search
|
||||
from one_memory.memory_qa import (
|
||||
MemoryAnswerError,
|
||||
_DeferredSecretStore,
|
||||
@@ -31,6 +31,7 @@ from one_memory.model_profiles import (
|
||||
ProviderKind,
|
||||
attach_disable_thinking,
|
||||
)
|
||||
from one_memory.sdk_search_options import SdkSearchFilters, SearchLane
|
||||
from one_memory.secrets import SecretStore, SecretStoreUnavailableError
|
||||
from one_memory.user_scope import owns
|
||||
|
||||
@@ -96,22 +97,39 @@ class SdkMemoryAnswerer:
|
||||
question: str,
|
||||
*,
|
||||
user_id: str = "local",
|
||||
filters: SdkSearchFilters | None = None,
|
||||
) -> tuple[SdkAskMemory, ...]:
|
||||
"""The user's active memories nearest the question.
|
||||
|
||||
`filters` are a caller's pass-through search options; absent, the
|
||||
search is the one ask always made.
|
||||
"""
|
||||
client = await self.client_handle.client()
|
||||
overrides = filters.search_overrides() if filters is not None else {}
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
question,
|
||||
user_ids=[user_id],
|
||||
agent_ids=[],
|
||||
limit=12,
|
||||
min_score=sdk_relevance_floor(),
|
||||
schema_min_score=sdk_relevance_floor(),
|
||||
schema_limit=12,
|
||||
intention_limit=12,
|
||||
scene="normal",
|
||||
reader=HY_MEMORY_READER,
|
||||
**{
|
||||
"user_ids": [user_id],
|
||||
"agent_ids": [],
|
||||
"limit": 12,
|
||||
"min_score": sdk_relevance_floor(),
|
||||
"schema_min_score": sdk_relevance_floor(),
|
||||
"schema_limit": 12,
|
||||
"intention_limit": 12,
|
||||
"scene": "normal",
|
||||
"reader": HY_MEMORY_READER,
|
||||
**overrides,
|
||||
},
|
||||
)
|
||||
hits = parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
if filters is not None and overrides:
|
||||
hits = tuple(
|
||||
hit
|
||||
for hit in parse_sdk_lane_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
if filters.passes_floor(cast(SearchLane, hit.lane), hit.score)
|
||||
)
|
||||
else:
|
||||
hits = parse_sdk_search(raw, error_code="hy_memory_search_shape_unknown")
|
||||
found: list[SdkAskMemory] = []
|
||||
seen: set[str] = set()
|
||||
for hit in hits:
|
||||
@@ -134,8 +152,9 @@ class SdkMemoryAnswerer:
|
||||
question: str,
|
||||
*,
|
||||
user_id: str = "local",
|
||||
filters: SdkSearchFilters | None = None,
|
||||
) -> SdkMemoryAnswer:
|
||||
candidates = await self.retrieve(question, user_id=user_id)
|
||||
candidates = await self.retrieve(question, user_id=user_id, filters=filters)
|
||||
if not candidates:
|
||||
return SdkMemoryAnswer(answer=_NO_GROUNDING, memories=(), mode="empty")
|
||||
profile = await self._active_profile()
|
||||
@@ -166,8 +185,9 @@ class SdkMemoryAnswerer:
|
||||
question: str,
|
||||
*,
|
||||
user_id: str = "local",
|
||||
filters: SdkSearchFilters | None = None,
|
||||
) -> AsyncIterator[dict[str, object]]:
|
||||
candidates = await self.retrieve(question, user_id=user_id)
|
||||
candidates = await self.retrieve(question, user_id=user_id, filters=filters)
|
||||
await self._require_current(candidates)
|
||||
yield {"type": "recall", "memories": candidates}
|
||||
if not candidates:
|
||||
|
||||
@@ -122,7 +122,9 @@ class SdkMemoryRuntime:
|
||||
while True:
|
||||
await asyncio.sleep(RAW_RETENTION_CHECK_SECONDS)
|
||||
try:
|
||||
removed = await self.prune_expired_raw()
|
||||
# Its own task, so this pass's use of the read client ends
|
||||
# with it instead of with this loop (see `sdk_read_handle`).
|
||||
removed = await asyncio.create_task(self.prune_expired_raw())
|
||||
if removed:
|
||||
logger.info("sdk_raw_retention_deleted count=%d", removed)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -23,6 +23,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.contracts import (
|
||||
@@ -51,6 +52,7 @@ from one_memory.sdk_migration_bundle import inspect_sdk_migration_bundle
|
||||
from one_memory.sdk_migration_verify import (
|
||||
MemoryReadbackFailed,
|
||||
ReadbackReport,
|
||||
expected_memory_ids,
|
||||
legacy_memory_ids_from,
|
||||
legacy_memory_ids_from_bundle,
|
||||
read_active_profile,
|
||||
@@ -61,6 +63,8 @@ from one_memory.sdk_vector_reattach import (
|
||||
ReattachedCollection,
|
||||
legacy_vector_collections,
|
||||
reattach_legacy_vectors,
|
||||
stored_memory_ids,
|
||||
vectors_belong_to_another_space,
|
||||
)
|
||||
from one_memory.target_schema import drop_legacy_tables, legacy_tables_present
|
||||
|
||||
@@ -457,13 +461,17 @@ def resume_legacy_cleanup(*, data_dir: Path, database_path: Path) -> tuple[str,
|
||||
bundle_id=marker.backup_bundle_id,
|
||||
)
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_NAMESPACE
|
||||
from one_memory.memory_visibility import drop_stale_triggers
|
||||
|
||||
recover_abandoned_staging(
|
||||
data_dir=data_dir,
|
||||
database_path=database_path,
|
||||
namespace=HY_MEMORY_NAMESPACE,
|
||||
)
|
||||
return drop_legacy_tables(database_path)
|
||||
dropped = drop_legacy_tables(database_path)
|
||||
# The visibility cache's grant triggers read a table just dropped (#233).
|
||||
drop_stale_triggers(database_path)
|
||||
return dropped
|
||||
|
||||
|
||||
MIGRATION_PREFIX = "sdk-authority-v2"
|
||||
@@ -700,13 +708,14 @@ def run_startup_migration(
|
||||
step["authority"] = authority.value
|
||||
log.migration_id = authority_migration
|
||||
with log.step("preflight.profile_read") as step:
|
||||
profile = read_active_profile(database_path)
|
||||
step["profile_present"] = profile is not None
|
||||
active = read_active_profile(database_path)
|
||||
step["profile_present"] = active is not None
|
||||
# No active profile means no generation model, not no embedding: the
|
||||
# vectors are the built-in model's, which is also what the Runtime reads.
|
||||
profile = active if active is not None else builtin_retrieval_snapshot()
|
||||
if authority is MemoryAuthority.SDK:
|
||||
with log.step("preflight.leftovers_scan") as step:
|
||||
vectors_waiting = profile is not None and bool(
|
||||
legacy_vector_collections(data_dir, profile)
|
||||
)
|
||||
vectors_waiting = bool(legacy_vector_collections(data_dir, profile))
|
||||
leftovers = bool(legacy_tables_present(database_path)) or abandoned_staging_present(
|
||||
data_dir=data_dir, namespace=namespace
|
||||
)
|
||||
@@ -717,7 +726,6 @@ def run_startup_migration(
|
||||
reattached: tuple[ReattachedCollection, ...] = ()
|
||||
readback: ReadbackReport | None = None
|
||||
if vectors_waiting:
|
||||
assert profile is not None
|
||||
progress("vectors")
|
||||
with log.step("vectors.reattach") as step:
|
||||
reattached = reattach_legacy_vectors(data_dir, profile)
|
||||
@@ -769,12 +777,12 @@ def run_startup_migration(
|
||||
step["chat"] = len(legacy.chat)
|
||||
step["productivity"] = len(legacy.productivity)
|
||||
step["users"] = len(legacy.users)
|
||||
if profile is None and not legacy.empty:
|
||||
# Without the embedding profile the Runtime's collections cannot be
|
||||
# named, so neither reattached nor read back. Legacy stays intact.
|
||||
log.check("preflight.model_profile_check", "sdk_migration_model_profile_required")
|
||||
raise DataPlaneMigrationError("sdk_migration_model_profile_required")
|
||||
log.check("preflight.model_profile_check", None)
|
||||
if active is None and not legacy.empty and vectors_belong_to_another_space(data_dir, profile):
|
||||
# The memories live in the space of a profile that is no longer active,
|
||||
# so the built-in model cannot read them. Legacy stays intact.
|
||||
log.check("preflight.vector_space_check", "sdk_migration_vector_profile_missing")
|
||||
raise DataPlaneMigrationError("sdk_migration_vector_profile_missing")
|
||||
log.check("preflight.vector_space_check", None)
|
||||
with capture_fence(data_dir):
|
||||
# Builds through 19 left a staged SDK copy beside the original store.
|
||||
# Settling it before the bundle means the bundle holds the original
|
||||
@@ -830,17 +838,23 @@ def run_startup_migration(
|
||||
step.update(preparation.as_document())
|
||||
reattached = ()
|
||||
readback = None
|
||||
if profile is not None:
|
||||
progress("vectors")
|
||||
with log.step("vectors.reattach") as step:
|
||||
reattached = reattach_legacy_vectors(data_dir, profile)
|
||||
step["collections"] = len(reattached)
|
||||
if not legacy.empty:
|
||||
assert profile is not None
|
||||
# Before reattach touches the store, and after the staging
|
||||
# recovery above has settled which store is the original.
|
||||
with log.step("verify.stored_ids") as step:
|
||||
expected, missing = expected_memory_ids(legacy, stored_memory_ids(data_dir))
|
||||
step["chat_expected"] = len(expected.chat)
|
||||
step["chat_missing_before_migration"] = missing
|
||||
progress("vectors")
|
||||
with log.step("vectors.reattach") as step:
|
||||
reattached = reattach_legacy_vectors(data_dir, profile)
|
||||
step["collections"] = len(reattached)
|
||||
if not expected.empty:
|
||||
progress("verify")
|
||||
with log.step("verify.readback") as step:
|
||||
with client_for(profile) as client:
|
||||
readback = verify_readback(client, legacy)
|
||||
readback = verify_readback(
|
||||
client, expected, chat_missing_before_migration=missing
|
||||
)
|
||||
step.update(readback.as_document())
|
||||
if not readback.passed:
|
||||
store.set_phase(migration_id, phase="readback_failed", state="blocked")
|
||||
|
||||
@@ -14,12 +14,12 @@ import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import closing, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.model_profiles import ModelProfileSnapshot
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, pinned_embedding_base_url
|
||||
from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
|
||||
|
||||
_PRODUCTIVITY_KINDS = frozenset({"coding", "working"})
|
||||
@@ -47,6 +47,8 @@ class ReadbackReport:
|
||||
chat_readable: int
|
||||
productivity_expected: int
|
||||
productivity_readable: int
|
||||
#: Confirmed in the mirror, gone from the SDK before this migration began.
|
||||
chat_missing_before_migration: int = 0
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
@@ -63,6 +65,7 @@ class ReadbackReport:
|
||||
"chat_readable": self.chat_readable,
|
||||
"productivity_expected": self.productivity_expected,
|
||||
"productivity_readable": self.productivity_readable,
|
||||
"chat_missing_before_migration": self.chat_missing_before_migration,
|
||||
"passed": self.passed,
|
||||
}
|
||||
|
||||
@@ -115,6 +118,27 @@ def read_legacy_memory_ids(connection: sqlite3.Connection) -> LegacyMemoryIds:
|
||||
)
|
||||
|
||||
|
||||
def expected_memory_ids(
|
||||
legacy: LegacyMemoryIds, stored: frozenset[str]
|
||||
) -> tuple[LegacyMemoryIds, int]:
|
||||
"""What the readback must find, and how many confirmed chats it need not.
|
||||
|
||||
A 1.0.4 library can hold confirmed rows whose SDK node the SDK had already
|
||||
merged or deleted: no migration could read those back (658 confirmed, 657
|
||||
in the store, and the whole upgrade failed). So chat is held to what the
|
||||
store held before this run, and anything in there must still read back.
|
||||
|
||||
If none of the confirmed chats is in the store, that is not history: the
|
||||
store is empty or is the wrong one. Nothing is excused then, so the
|
||||
readback still fails on every id instead of checking nothing just before
|
||||
the mirror is dropped.
|
||||
"""
|
||||
chat = tuple(memory_id for memory_id in legacy.chat if memory_id in stored)
|
||||
if not chat:
|
||||
return legacy, 0
|
||||
return replace(legacy, chat=chat), len(legacy.chat) - len(chat)
|
||||
|
||||
|
||||
def legacy_memory_ids_from(database_path: Path) -> LegacyMemoryIds:
|
||||
with main_database(database_path).read() as connection:
|
||||
return read_legacy_memory_ids(connection)
|
||||
@@ -160,7 +184,14 @@ def read_active_profile(database_path: Path) -> ModelProfileSnapshot | None:
|
||||
model=row["model"],
|
||||
embedding_model=row["embedding_model"],
|
||||
embedding_provider_kind=row["embedding_provider_kind"],
|
||||
embedding_base_url=row["embedding_base_url"],
|
||||
# Read-only, beside a Core that may not have pinned it yet.
|
||||
embedding_base_url=pinned_embedding_base_url(
|
||||
provider_kind=row["provider_kind"],
|
||||
base_url=row["base_url"],
|
||||
embedding_model=row["embedding_model"],
|
||||
embedding_provider_kind=row["embedding_provider_kind"],
|
||||
embedding_base_url=row["embedding_base_url"],
|
||||
),
|
||||
embedding_dimensions=row["embedding_dimensions"],
|
||||
timeout_seconds=row["timeout_seconds"],
|
||||
max_concurrency=row["max_concurrency"],
|
||||
@@ -179,14 +210,18 @@ def runtime_sdk_client(
|
||||
from one_memory.model_profiles import ModelProfileStore
|
||||
from one_memory.secrets import KeyringSecretStore, SecretStoreUnavailableError
|
||||
|
||||
try:
|
||||
secret = ModelProfileStore(database_path).resolve_snapshot_secret(
|
||||
profile.profile_id, profile.revision, KeyringSecretStore()
|
||||
)
|
||||
except SecretStoreUnavailableError:
|
||||
raise DataPlaneMigrationError("sdk_migration_secure_storage_unavailable") from None
|
||||
if profile.has_api_key and secret is None:
|
||||
raise DataPlaneMigrationError("sdk_migration_model_secret_unavailable")
|
||||
secret: str | None = None
|
||||
# Without a key there is nothing to resolve, and the built-in retrieval
|
||||
# snapshot has no stored version to look one up by.
|
||||
if profile.has_api_key:
|
||||
try:
|
||||
secret = ModelProfileStore(database_path).resolve_snapshot_secret(
|
||||
profile.profile_id, profile.revision, KeyringSecretStore()
|
||||
)
|
||||
except SecretStoreUnavailableError:
|
||||
raise DataPlaneMigrationError("sdk_migration_secure_storage_unavailable") from None
|
||||
if secret is None:
|
||||
raise DataPlaneMigrationError("sdk_migration_model_secret_unavailable")
|
||||
lease = acquire_client(profile, api_key=secret, data_dir=data_dir)
|
||||
try:
|
||||
yield lease.client
|
||||
@@ -194,7 +229,9 @@ def runtime_sdk_client(
|
||||
lease.close()
|
||||
|
||||
|
||||
def verify_readback(client: Any, legacy: LegacyMemoryIds) -> ReadbackReport:
|
||||
def verify_readback(
|
||||
client: Any, legacy: LegacyMemoryIds, *, chat_missing_before_migration: int = 0
|
||||
) -> ReadbackReport:
|
||||
listed: set[str] = set()
|
||||
for user_id in legacy.users or ("local",):
|
||||
offset = 0
|
||||
@@ -228,5 +265,6 @@ def verify_readback(client: Any, legacy: LegacyMemoryIds) -> ReadbackReport:
|
||||
chat_readable=sum(1 for memory_id in legacy.chat if readable(memory_id)),
|
||||
productivity_expected=len(legacy.productivity),
|
||||
productivity_readable=sum(1 for memory_id in legacy.productivity if readable(memory_id)),
|
||||
chat_missing_before_migration=chat_missing_before_migration,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""The read client, kept in the embedding space the active profile writes to.
|
||||
|
||||
Every read used to open the built-in 512-dim collection while writes followed
|
||||
the active profile, so a custom embedding read an empty space (#242). This
|
||||
handle opens the active profile's `retrieval_view` instead, and the built-in
|
||||
space only when no profile is active.
|
||||
|
||||
A profile change builds a new client and retires the old one; a retired client
|
||||
is closed once no task still uses it, as `SdkWriteCoordinator` does for writes
|
||||
(#218). A use is the task that asked for the client: callers keep their
|
||||
`client = await handle.client()` shape, and the use ends with that task (one
|
||||
HTTP request, one retention pass).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from one_memory.hy_sdk_runtime import (
|
||||
BUILTIN_RETRIEVAL_PROFILE_ID,
|
||||
SdkClientHandle,
|
||||
builtin_retrieval_snapshot,
|
||||
retrieval_view,
|
||||
)
|
||||
from one_memory.hy_sdk_types import HySdkRuntimeError
|
||||
from one_memory.model_profiles import ModelProfileStore, ProviderKind
|
||||
from one_memory.secrets import SecretStore, SecretStoreUnavailableError
|
||||
|
||||
_BUILTIN = (BUILTIN_RETRIEVAL_PROFILE_ID, 0)
|
||||
|
||||
|
||||
class ActiveProfileReadHandle(SdkClientHandle):
|
||||
"""An `SdkClientHandle` whose profile follows the active one."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
profiles: ModelProfileStore,
|
||||
secrets: SecretStore,
|
||||
data_dir: Path,
|
||||
) -> None:
|
||||
super().__init__(builtin_retrieval_snapshot(), api_key=None, data_dir=data_dir)
|
||||
self.profiles = profiles
|
||||
self.secrets = secrets
|
||||
self.data_dir = data_dir
|
||||
self._handles: dict[tuple[str, int], SdkClientHandle] = {}
|
||||
#: Tasks in flight per handle; a retired handle closes at zero.
|
||||
self._uses: dict[SdkClientHandle, int] = {}
|
||||
self._retired: set[SdkClientHandle] = set()
|
||||
self._closing: set[asyncio.Task[None]] = set()
|
||||
self._handle_lock = asyncio.Lock()
|
||||
|
||||
async def client(self) -> Any:
|
||||
identity = await asyncio.to_thread(self.profiles.active_identity) or _BUILTIN
|
||||
async with self._handle_lock:
|
||||
handle = self._handles.get(identity)
|
||||
if handle is None:
|
||||
# Only on a change of profile: resolving a key can reach the
|
||||
# system keychain, so it is not done on every read.
|
||||
handle = await asyncio.to_thread(self._build, identity)
|
||||
self._handles[identity] = handle
|
||||
self._retired.update(value for key, value in self._handles.items() if key != identity)
|
||||
self._handles = {identity: handle}
|
||||
unused = {value for value in self._retired if not self._uses.get(value)}
|
||||
self._retired -= unused
|
||||
self._hold(handle)
|
||||
await asyncio.gather(*(value.close() for value in unused))
|
||||
return await handle.client()
|
||||
|
||||
async def close(self) -> None:
|
||||
async with self._handle_lock:
|
||||
handles = (*self._handles.values(), *self._retired)
|
||||
self._handles, self._retired, self._uses = {}, set(), {}
|
||||
await asyncio.gather(*(handle.close() for handle in handles), *self._closing)
|
||||
|
||||
def _hold(self, handle: SdkClientHandle) -> None:
|
||||
"""Count a use of `handle` until the asking task finishes."""
|
||||
task = asyncio.current_task()
|
||||
if task is None: # pragma: no cover - always called from a coroutine
|
||||
return
|
||||
self._uses[handle] = self._uses.get(handle, 0) + 1
|
||||
task.add_done_callback(lambda _task: self._release(handle))
|
||||
|
||||
def _release(self, handle: SdkClientHandle) -> None:
|
||||
remaining = self._uses.get(handle, 0) - 1
|
||||
if remaining > 0:
|
||||
self._uses[handle] = remaining
|
||||
return
|
||||
self._uses.pop(handle, None)
|
||||
if handle in self._retired:
|
||||
self._retired.discard(handle)
|
||||
closing = asyncio.get_running_loop().create_task(handle.close())
|
||||
self._closing.add(closing)
|
||||
closing.add_done_callback(self._closing.discard)
|
||||
|
||||
def _build(self, identity: tuple[str, int]) -> SdkClientHandle:
|
||||
if identity == _BUILTIN:
|
||||
return SdkClientHandle(
|
||||
builtin_retrieval_snapshot(), api_key=None, data_dir=self.data_dir
|
||||
)
|
||||
profile = self.profiles.snapshot(*identity)
|
||||
view = retrieval_view(profile)
|
||||
secret: str | None = None
|
||||
# Only a remote OpenAI-compatible embedding sends the shared key.
|
||||
if view.embedding_provider_kind is ProviderKind.OPENAI_COMPATIBLE:
|
||||
try:
|
||||
secret = self.profiles.resolve_snapshot_secret(*identity, self.secrets)
|
||||
except SecretStoreUnavailableError:
|
||||
raise HySdkRuntimeError("secure_storage_unavailable") from None
|
||||
if secret is None:
|
||||
raise HySdkRuntimeError("model_secret_unavailable")
|
||||
return SdkClientHandle(view, api_key=secret, data_dir=self.data_dir)
|
||||
@@ -0,0 +1,179 @@
|
||||
"""SDK search parameters a host may pass through to `client.search`.
|
||||
|
||||
`POST /api/v2/context/search`, `POST /api/v1/recall` and the v2 ask routes
|
||||
used to fix every SDK search knob, so a host that wanted a stricter relevance
|
||||
floor filtered the answer itself. These fields are the public `search`
|
||||
parameters of hy-memory 2.0.9 that are safe to hand a caller. Every field is
|
||||
optional: absent, the route keeps exactly the call it made before.
|
||||
|
||||
Never exposed: `user_ids` (a route searches its one scoped `user_id`),
|
||||
`embed_override` and `query_embedding` (they bypass the configured embedder),
|
||||
and `request_id` (Core's own correlation id).
|
||||
|
||||
A caller's floor is also applied to what Core returns, not only handed to the
|
||||
SDK: a hit below the floor that names its lane is dropped (`passes_floor`).
|
||||
With `min_score` alone that floor covers every lane, so `min_score=0.9`
|
||||
answers only results scoring at least 0.9.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictBool, model_validator
|
||||
|
||||
#: The SDK's `MemoryLayer` values (hy-memory 2.0.9).
|
||||
MemoryLayerName = Literal[
|
||||
"l0_basic_info",
|
||||
"l1_raw",
|
||||
"l2_fact",
|
||||
"l3_summary",
|
||||
"l4_identity",
|
||||
"l5_knowledge",
|
||||
"l6_schema",
|
||||
"l7_intention",
|
||||
]
|
||||
MEMORY_LAYERS: tuple[str, ...] = (
|
||||
"l0_basic_info",
|
||||
"l1_raw",
|
||||
"l2_fact",
|
||||
"l3_summary",
|
||||
"l4_identity",
|
||||
"l5_knowledge",
|
||||
"l6_schema",
|
||||
"l7_intention",
|
||||
)
|
||||
#: The SDK's `MemoryStatus` values.
|
||||
MEMORY_STATUSES: tuple[str, ...] = (
|
||||
"active",
|
||||
"superseded",
|
||||
"negated",
|
||||
"conflicted",
|
||||
"shadow",
|
||||
"archived",
|
||||
)
|
||||
SearchScene = Literal["chat", "working", "coding", "all"]
|
||||
#: Which recall lane produced a hit, for the floor that applies to it.
|
||||
SearchLane = Literal["normal", "schema", "proactive", "working", "coding", "raw"]
|
||||
|
||||
Score = Annotated[float, Field(ge=0.0, le=1.0)]
|
||||
SourceName = Annotated[str, Field(min_length=1, max_length=128)]
|
||||
ScopeId = Annotated[str, Field(min_length=1, max_length=512)]
|
||||
|
||||
|
||||
class SdkSearchFilters(BaseModel):
|
||||
"""Filters and floors every search route accepts (recall, context, ask)."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
#: Chat lane floor; with nothing more specific, every lane's.
|
||||
min_score: Score | None = None
|
||||
#: Schema (L6) lane floor; defaults to `min_score` when only that is set.
|
||||
schema_min_score: Score | None = None
|
||||
#: False skips the SDK's reranking stages for this call.
|
||||
rerank: StrictBool | None = None
|
||||
include_sources: tuple[SourceName, ...] | None = Field(default=None, max_length=64)
|
||||
exclude_sources: tuple[SourceName, ...] | None = Field(default=None, max_length=64)
|
||||
#: Unix seconds: only memories created at or after it.
|
||||
created_after: float | None = Field(default=None, ge=0)
|
||||
session_ids: tuple[ScopeId, ...] | None = Field(default=None, max_length=64)
|
||||
agent_ids: tuple[ScopeId, ...] | None = Field(default=None, max_length=64)
|
||||
#: Only these layers (pro/ultra; lite always searches raw).
|
||||
layers: tuple[MemoryLayerName, ...] | None = Field(default=None, min_length=1, max_length=8)
|
||||
|
||||
def filter_kwargs(self) -> dict[str, object]:
|
||||
"""The SDK keyword arguments this request sets, and only those."""
|
||||
values: dict[str, object] = {}
|
||||
for name in (
|
||||
"min_score",
|
||||
"schema_min_score",
|
||||
"rerank",
|
||||
"created_after",
|
||||
):
|
||||
value = getattr(self, name)
|
||||
if value is not None:
|
||||
values[name] = value
|
||||
for name in ("include_sources", "exclude_sources", "session_ids", "agent_ids", "layers"):
|
||||
value = getattr(self, name)
|
||||
if value is not None:
|
||||
values[name] = list(dict.fromkeys(value))
|
||||
if self.min_score is not None and self.schema_min_score is None:
|
||||
values["schema_min_score"] = self.min_score
|
||||
return values
|
||||
|
||||
def lane_floor(self, lane: SearchLane) -> float | None:
|
||||
"""The caller's floor for one lane, or None when the caller set none."""
|
||||
if lane == "schema":
|
||||
return self.schema_min_score if self.schema_min_score is not None else self.min_score
|
||||
return self.min_score
|
||||
|
||||
def passes_floor(self, lane: SearchLane, score: float) -> bool:
|
||||
floor = self.lane_floor(lane)
|
||||
return floor is None or score >= floor
|
||||
|
||||
def search_overrides(self) -> Mapping[str, object]:
|
||||
return self.filter_kwargs()
|
||||
|
||||
|
||||
class SdkSearchOptions(SdkSearchFilters):
|
||||
"""Every SDK search knob a recall or context caller may set."""
|
||||
|
||||
#: working/coding fact-layer floor (SDK default 0.30).
|
||||
working_fact_min_score: Score | None = None
|
||||
#: working/coding experience-layer floor (SDK default fact floor + 0.10).
|
||||
working_experience_min_score: Score | None = None
|
||||
reader: Literal["legacy", "hybrid"] | None = None
|
||||
#: chat (the default), working, coding, or all three fused.
|
||||
scene: SearchScene | None = None
|
||||
#: agentic spends one model call to classify and rewrite the query.
|
||||
search_mode: Literal["default", "agentic"] | None = None
|
||||
#: working/coding lanes only; `branch` needs `workspace_id`.
|
||||
workspace_id: ScopeId | None = None
|
||||
branch: ScopeId | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def branch_needs_workspace(self) -> SdkSearchOptions:
|
||||
if self.branch is not None and self.workspace_id is None:
|
||||
raise ValueError("branch requires workspace_id")
|
||||
return self
|
||||
|
||||
def search_overrides(self) -> Mapping[str, object]:
|
||||
values = self.filter_kwargs()
|
||||
for name in (
|
||||
"working_fact_min_score",
|
||||
"working_experience_min_score",
|
||||
"reader",
|
||||
"scene",
|
||||
"search_mode",
|
||||
"workspace_id",
|
||||
"branch",
|
||||
):
|
||||
value = getattr(self, name)
|
||||
if value is not None:
|
||||
values[name] = value
|
||||
if self.min_score is not None and self.working_fact_min_score is None:
|
||||
values["working_fact_min_score"] = self.min_score
|
||||
if self.min_score is not None and self.working_experience_min_score is None:
|
||||
values["working_experience_min_score"] = self.min_score
|
||||
return values
|
||||
|
||||
def lane_floor(self, lane: SearchLane) -> float | None:
|
||||
if lane in {"working", "coding"}:
|
||||
explicit = [
|
||||
value
|
||||
for value in (self.working_fact_min_score, self.working_experience_min_score)
|
||||
if value is not None
|
||||
]
|
||||
# Facts and experiences share one score field on a fused hit, so
|
||||
# the looser of the two floors is the one Core can enforce.
|
||||
return min(explicit) if explicit else self.min_score
|
||||
return super().lane_floor(lane)
|
||||
|
||||
|
||||
def split_csv(values: list[str] | None) -> list[str]:
|
||||
"""Query values given repeated (`a&a`) or comma separated (`a,b`)."""
|
||||
found: list[str] = []
|
||||
for value in values or ():
|
||||
found.extend(part.strip() for part in value.split(",") if part.strip())
|
||||
return list(dict.fromkeys(found))
|
||||
@@ -9,6 +9,7 @@ one. Runtime code only reaches vectors through the SDK. This module runs from
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import closing
|
||||
@@ -21,6 +22,8 @@ from one_memory.model_profiles import ModelProfileSnapshot
|
||||
from one_memory.sdk_data_plane_migration import DataPlaneMigrationError
|
||||
|
||||
_BATCH = 500
|
||||
#: Every name a MemHub build gave an embedding space's base collection.
|
||||
_SPACE_NAME = re.compile(r"^(?:memhub_[0-9a-f]{16}|one_memory_[0-9a-f]{12})(?:_|$)")
|
||||
|
||||
|
||||
def open_offline_chroma(directory: Path) -> Any:
|
||||
@@ -51,13 +54,20 @@ def _vector_directory(data_dir: Path) -> Path:
|
||||
return data_dir / "hy-memory" / "governed-v2" / "vector"
|
||||
|
||||
|
||||
def _stored_names(data_dir: Path) -> tuple[str, ...]:
|
||||
def _stored_names(data_dir: Path, *, filled: bool = False) -> tuple[str, ...]:
|
||||
"""Collection names; with `filled`, only those holding at least one row."""
|
||||
database = _vector_directory(data_dir) / "chroma.sqlite3"
|
||||
if not database.is_file():
|
||||
return ()
|
||||
with closing(sqlite3.connect(f"file:{database}?mode=ro", uri=True)) as connection:
|
||||
try:
|
||||
rows = connection.execute("SELECT name FROM collections").fetchall()
|
||||
rows = connection.execute(
|
||||
"SELECT DISTINCT collections.name FROM collections "
|
||||
"JOIN segments ON segments.collection = collections.id "
|
||||
"JOIN embeddings ON embeddings.segment_id = segments.id"
|
||||
if filled
|
||||
else "SELECT name FROM collections"
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
return ()
|
||||
return tuple(sorted(str(row[0]) for row in rows))
|
||||
@@ -80,6 +90,59 @@ def legacy_vector_collections(
|
||||
return tuple(pairs)
|
||||
|
||||
|
||||
def vectors_belong_to_another_space(data_dir: Path, profile: ModelProfileSnapshot) -> bool:
|
||||
"""Vectors exist, but only in embedding spaces `profile` does not name.
|
||||
|
||||
Counted rows, not names: every Runtime opens its read collection with
|
||||
`get_or_create`, so an empty collection under the expected base proves
|
||||
nothing. Read-only, so a preflight can ask before anything changes.
|
||||
"""
|
||||
current, legacy_bases = vector_collection_bases(profile)
|
||||
known = (current, *legacy_bases)
|
||||
filled = _stored_names(data_dir, filled=True)
|
||||
return not any(
|
||||
name == base or name.startswith(f"{base}_") for name in filled for base in known
|
||||
) and any(_SPACE_NAME.match(name) for name in filled)
|
||||
|
||||
|
||||
def stored_memory_ids(data_dir: Path) -> frozenset[str]:
|
||||
"""Every id in the vector store as found, across all embedding spaces.
|
||||
|
||||
All of them on purpose. A memory stranded under a space the Runtime will
|
||||
not read must count as present, so the readback then fails on it instead
|
||||
of excusing it. Both the point id and the SDK's `node_id` are kept: the
|
||||
SDK derives a point id from a node id that is not a UUID.
|
||||
|
||||
Callers must have stopped every SDK writer. A store that cannot be read
|
||||
raises: "unknown" must never be taken for "absent".
|
||||
"""
|
||||
if not (_vector_directory(data_dir) / "chroma.sqlite3").is_file():
|
||||
return frozenset()
|
||||
found: set[str] = set()
|
||||
try:
|
||||
client = open_offline_chroma(_vector_directory(data_dir))
|
||||
try:
|
||||
for collection in client.list_collections():
|
||||
offset = 0
|
||||
while True:
|
||||
page = collection.get(include=["metadatas"], limit=_BATCH, offset=offset)
|
||||
ids = list(page["ids"])
|
||||
if not ids:
|
||||
break
|
||||
found.update(ids)
|
||||
found.update(
|
||||
str(metadata["node_id"])
|
||||
for metadata in page["metadatas"] or ()
|
||||
if metadata and metadata.get("node_id")
|
||||
)
|
||||
offset += len(ids)
|
||||
finally:
|
||||
client.close()
|
||||
except Exception as error:
|
||||
raise DataPlaneMigrationError("sdk_migration_vector_store_unreadable") from error
|
||||
return frozenset(found)
|
||||
|
||||
|
||||
def _records(collection: Any) -> Iterator[dict[str, Any]]:
|
||||
offset = 0
|
||||
while True:
|
||||
|
||||
@@ -27,6 +27,11 @@ class SemanticMemoryHit:
|
||||
member_ids: tuple[str, ...] = ()
|
||||
sdk_formatted: bool = False
|
||||
formatting_complete: bool = False
|
||||
#: The recall lane that produced the hit (`sdk_search_options.SearchLane`)
|
||||
#: and the namespace to read it back from. Defaults are a chat memory.
|
||||
lane: str = "normal"
|
||||
kind: str = "chat"
|
||||
variant: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
|
||||
@@ -248,6 +248,7 @@
|
||||
| `test_schema_gate.py` | `contract` | 覆盖 fresh database records the build schema、future schema refuses to open、legacy database without a ledger is left untouched 场景。 |
|
||||
| `test_schema_migrations.py` | `contract` | 覆盖 evidence store adds source ordinal to previous schema、import store migrates previous job and error schema、old active job index lets a sync sit beside paused history 等 4 个场景。 |
|
||||
| `test_schema_ownership_gate.py` | `contract` | A1-03/A1-04: schema ownership, and the hard part of telling DDL from reads. |
|
||||
| `test_sdk_api_surface.py` | `contract` | The SDK surface MemHub exposes: search pass-through, layers, raw, v2 aliases. |
|
||||
| `test_sdk_client_construction.py` | `contract` | SDK clients of one home are constructed one at a time. |
|
||||
| `test_sdk_data_plane_migration.py` | `contract` | SDK-authority migration ledger, aliases and atomic cutover marker. |
|
||||
| `test_sdk_memory_qa.py` | `contract` | 覆盖 sdk ask propagates sdk search failure without legacy fallback、sdk ask revalidates memory before returning extract、sdk ask uses native sdk memory without governance fields 场景。 |
|
||||
@@ -311,4 +312,4 @@
|
||||
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
|
||||
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
|
||||
|
||||
共 306 个测试文件。
|
||||
共 307 个测试文件。
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
|
||||
EXPECTED_TEST_FILE_COUNT = 306
|
||||
EXPECTED_TEST_FILES_SHA256 = "e616dcf9fea288bf15a4445045f7b4278517f3304091c04fffd55be80308d481"
|
||||
EXPECTED_TEST_FILE_COUNT = 307
|
||||
EXPECTED_TEST_FILES_SHA256 = "96a9ed11e358791723fb5ad827f72da2e9caeb673841119a5d9856a2d76fd5b1"
|
||||
|
||||
# Pure transformations, validation models, parity checks and state machines.
|
||||
# This list is intentionally small: uncertain files default to contract.
|
||||
|
||||
@@ -24,7 +24,8 @@ def model_snapshot(
|
||||
model="memory-model",
|
||||
embedding_model="embedding-model",
|
||||
embedding_provider_kind=embedding_provider,
|
||||
embedding_base_url=embedding_base_url,
|
||||
# Stored profiles always name it; see `pinned_embedding_base_url`.
|
||||
embedding_base_url=embedding_base_url or base_url,
|
||||
embedding_dimensions=768,
|
||||
timeout_seconds=30,
|
||||
max_concurrency=1,
|
||||
|
||||
@@ -175,7 +175,13 @@ def test_sdk_migrate_reports_each_step_then_the_result(tmp_path, monkeypatch) ->
|
||||
|
||||
assert code == 0, output.getvalue()
|
||||
lines = [json.loads(line) for line in output.getvalue().splitlines()]
|
||||
assert [line["step"] for line in lines[:-1]] == ["bundle", "prepare", "cutover", "cleanup"]
|
||||
assert [line["step"] for line in lines[:-1]] == [
|
||||
"bundle",
|
||||
"prepare",
|
||||
"vectors",
|
||||
"cutover",
|
||||
"cleanup",
|
||||
]
|
||||
assert lines[-1]["state"] == "cutover"
|
||||
assert lines[-1]["bundle_id"] == f"{lines[-1]['migration_id']}-pre-cutover"
|
||||
assert not (tmp_path / ".sdk-migration-capture-fence.json").exists()
|
||||
|
||||
@@ -26,6 +26,7 @@ def snapshot() -> ModelProfileSnapshot:
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="memory-model",
|
||||
embedding_model="embedding-model",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
embedding_dimensions=768,
|
||||
timeout_seconds=30,
|
||||
max_concurrency=1,
|
||||
|
||||
@@ -66,6 +66,7 @@ def _grounded_setup(tmp_path, *, memory_id: str = "memory-real-data"):
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="test-model",
|
||||
embedding_model="test-embedding",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -263,6 +264,7 @@ def test_answer_uses_only_grounded_confirmed_memory_citations(tmp_path) -> None:
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="test-model",
|
||||
embedding_model="test-embedding",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -468,6 +470,7 @@ def test_answer_rejects_model_citation_outside_retrieved_scope(tmp_path) -> None
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="test-model",
|
||||
embedding_model="test-embedding",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -530,6 +533,7 @@ def _compatible_gateway_setup(tmp_path):
|
||||
base_url="https://api.example.invalid/standard/v1",
|
||||
model="deepseek-v4-pro",
|
||||
embedding_model="text-embedding",
|
||||
embedding_base_url="https://api.example.invalid/standard/v1",
|
||||
api_key="probe-key",
|
||||
),
|
||||
secrets,
|
||||
|
||||
@@ -186,6 +186,7 @@ def _setup(tmp_path: Path, *, model: str = "available") -> _Setup:
|
||||
base_url="https://memory-qa.example.invalid/v1",
|
||||
model="synthetic-generation",
|
||||
embedding_model="synthetic-embedding",
|
||||
embedding_base_url="https://memory-qa.example.invalid/v1",
|
||||
api_key=SecretStr(_SECRET),
|
||||
),
|
||||
secrets,
|
||||
|
||||
@@ -25,6 +25,7 @@ def _stores(tmp_path):
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="hy-bot-phi4",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
|
||||
@@ -83,6 +83,7 @@ def test_task_freezes_active_profile_revision_and_is_idempotent(tmp_path) -> Non
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -240,6 +241,7 @@ def test_task_extraction_checkpoint_is_idempotent_and_recoverable(tmp_path) -> N
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -293,6 +295,7 @@ def test_task_waits_for_model_and_retries_with_same_snapshot(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -337,6 +340,7 @@ def test_blocked_stages_name_the_stuck_stage_and_reason(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -433,6 +437,7 @@ def test_task_duration_estimate_ignores_time_the_app_was_closed(tmp_path) -> Non
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -537,6 +542,7 @@ def test_running_task_is_recovered_after_worker_restart(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -572,6 +578,7 @@ def test_superseded_profile_cancels_only_claimable_tasks(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="phi",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -601,6 +608,7 @@ def test_superseded_profile_cancels_only_claimable_tasks(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -630,6 +638,7 @@ def test_superseded_ingest_policy_cancels_only_old_conversation_tasks(tmp_path)
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -676,6 +685,7 @@ def test_task_rejects_inactive_profile_and_request_id_reuse(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -1700,6 +1710,7 @@ def _profile_and_tasks(tmp_path) -> tuple[MemoryTaskStore, ModelProfile]:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
@@ -2058,6 +2069,7 @@ def test_a_failed_task_below_the_ceiling_is_still_claimable(tmp_path) -> None:
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
),
|
||||
|
||||
@@ -55,6 +55,7 @@ def library(tmp_path):
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="hy-bot-phi4",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -611,3 +612,58 @@ def test_granting_a_source_repeatedly_does_not_raise(library):
|
||||
|
||||
with main_database_read(database) as db:
|
||||
assert memory.memory_id in pending(db)
|
||||
|
||||
|
||||
def test_grant_writes_still_work_after_the_cutover_drops_the_link_table(library):
|
||||
"""#233: the grant triggers read `governed_memory_evidence`.
|
||||
|
||||
The SDK cutover drops it, and SQLite resolves a trigger body only when it
|
||||
fires, so every grant, revoke and connect then failed with "no such
|
||||
table". A trigger whose body names a missing table is not installed, and
|
||||
the copy an earlier startup installed is dropped.
|
||||
"""
|
||||
database, *_ = library
|
||||
from one_memory.imports.candidates import RootCandidateKind
|
||||
from one_memory.imports.grants import SourceGrantStore
|
||||
from one_memory.memory_visibility import drop_stale_triggers
|
||||
|
||||
startup(database)
|
||||
with sqlite3.connect(database) as db:
|
||||
db.execute("DROP TABLE governed_memory_evidence")
|
||||
|
||||
def grant_triggers() -> set[str]:
|
||||
with sqlite3.connect(database) as db:
|
||||
return {
|
||||
str(row[0])
|
||||
for row in db.execute(
|
||||
"SELECT name FROM sqlite_master "
|
||||
"WHERE type='trigger' AND tbl_name='source_grants'"
|
||||
)
|
||||
}
|
||||
|
||||
assert grant_triggers() == {
|
||||
"memory_visibility_dirty_on_grants",
|
||||
"memory_visibility_dirty_on_grants_update",
|
||||
}
|
||||
# The `evidence` triggers read the dropped table too; the cutover drops
|
||||
# `evidence` as well, which takes them with it.
|
||||
assert {
|
||||
"memory_visibility_dirty_on_grants",
|
||||
"memory_visibility_dirty_on_grants_update",
|
||||
} <= set(drop_stale_triggers(database))
|
||||
assert grant_triggers() == set()
|
||||
assert drop_stale_triggers(database) == ()
|
||||
# Installing again on this shape skips them instead of bringing them back.
|
||||
startup(database)
|
||||
assert grant_triggers() == set()
|
||||
|
||||
folder = database.parent / "cursor"
|
||||
folder.mkdir(exist_ok=True)
|
||||
store = SourceGrantStore(database, home=database.parent)
|
||||
store.grant(
|
||||
user_id="local",
|
||||
source_id="cursor",
|
||||
root=folder,
|
||||
candidate_kind=RootCandidateKind.DEFAULT,
|
||||
)
|
||||
assert store.revoke(user_id="local", source_id="cursor").revoked_at is not None
|
||||
@@ -51,6 +51,7 @@ def library(tmp_path):
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="hy-bot-phi4",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
|
||||
@@ -28,6 +28,7 @@ def test_probe_still_measures_embedding_when_structured_output_fails(tmp_path) -
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="plain-model",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -63,6 +64,7 @@ def test_probe_records_the_real_embedding_width_on_the_profile(tmp_path) -> None
|
||||
base_url="http://127.0.0.1:11434",
|
||||
model="qwen3:8b",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -119,6 +121,7 @@ def test_openai_compatible_probe_checks_model_and_structured_output(tmp_path) ->
|
||||
base_url="https://models.example.com/v1",
|
||||
model="memory-model",
|
||||
embedding_model="embedding-model",
|
||||
embedding_base_url="https://models.example.com/v1",
|
||||
api_key=SecretStr(API_KEY),
|
||||
),
|
||||
secrets,
|
||||
@@ -260,6 +263,7 @@ def test_probe_reports_missing_model_without_sending_generation(tmp_path) -> Non
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="memory-model",
|
||||
embedding_model="embedding-model",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
|
||||
@@ -50,6 +50,7 @@ def test_model_profile_api_never_returns_key_and_enforces_revision(tmp_path) ->
|
||||
"base_url": "https://models.example.com/v1",
|
||||
"model": "memory-model",
|
||||
"embedding_model": "embedding-model",
|
||||
"embedding_base_url": "https://models.example.com/v1",
|
||||
"api_key": API_KEY,
|
||||
},
|
||||
)
|
||||
@@ -134,6 +135,7 @@ def test_model_profile_probe_route_returns_only_machine_results(tmp_path) -> Non
|
||||
"base_url": "https://models.example.com/v1",
|
||||
"model": "memory-model",
|
||||
"embedding_model": "embedding-model",
|
||||
"embedding_base_url": "https://models.example.com/v1",
|
||||
"api_key": API_KEY,
|
||||
},
|
||||
)
|
||||
@@ -244,6 +246,7 @@ def test_activation_survives_a_corpus_already_imported(tmp_path) -> None:
|
||||
"base_url": "https://models.example.com/v1",
|
||||
"model": "memory-model",
|
||||
"embedding_model": "embedding-model",
|
||||
"embedding_base_url": "https://models.example.com/v1",
|
||||
"api_key": API_KEY,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -198,6 +198,7 @@ def test_active_profile_can_be_deleted_without_changing_other_profiles(tmp_path)
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="other",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -208,6 +209,7 @@ def test_active_profile_can_be_deleted_without_changing_other_profiles(tmp_path)
|
||||
base_url="https://models.example.com/v1",
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="https://models.example.com/v1",
|
||||
api_key=SecretStr(API_KEY),
|
||||
),
|
||||
secrets,
|
||||
@@ -349,6 +351,7 @@ def _probed(store, secrets, *, structured: bool, generates: bool = True):
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="qwen",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
@@ -394,3 +397,102 @@ def test_a_model_that_cannot_generate_still_cannot_be_turned_on(tmp_path) -> Non
|
||||
|
||||
with pytest.raises(ModelProfileUnavailableError):
|
||||
store.activate(profile.profile_id, secrets)
|
||||
|
||||
|
||||
def _space(store: ModelProfileStore, profile_id: str) -> str:
|
||||
"""The embedding space's collection base, as the Runtime names it."""
|
||||
from one_memory.hy_sdk_runtime import vector_collection_bases
|
||||
|
||||
identity = (profile_id, store.require(profile_id, InMemorySecretStore()).revision)
|
||||
return vector_collection_bases(store.snapshot(*identity))[0]
|
||||
|
||||
|
||||
def test_a_remote_embedding_must_name_its_endpoint(tmp_path) -> None:
|
||||
"""The endpoint no longer follows the generation `base_url`."""
|
||||
from one_memory.model_profiles import EmbeddingBaseUrlRequiredError
|
||||
|
||||
store = ModelProfileStore(tmp_path / "one-memory.sqlite3")
|
||||
secrets = InMemorySecretStore()
|
||||
unnamed = ModelProfileInput(
|
||||
name="Remote",
|
||||
provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
base_url="https://models.example.com/v1",
|
||||
model="qwen",
|
||||
embedding_model="text-embedding-3-small",
|
||||
api_key=SecretStr(API_KEY),
|
||||
)
|
||||
with pytest.raises(EmbeddingBaseUrlRequiredError):
|
||||
store.create(unnamed, secrets)
|
||||
assert store.list(secrets) == []
|
||||
|
||||
named = store.create(
|
||||
unnamed.model_copy(update={"embedding_base_url": "https://models.example.com/v1"}),
|
||||
secrets,
|
||||
)
|
||||
with pytest.raises(EmbeddingBaseUrlRequiredError):
|
||||
store.update(
|
||||
named.profile_id,
|
||||
ModelProfilePatch(
|
||||
expected_revision=named.revision,
|
||||
embedding_provider_kind=ProviderKind.OLLAMA,
|
||||
embedding_base_url="",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
|
||||
|
||||
def test_stored_profiles_get_the_endpoint_they_resolved_to_and_keep_their_space(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""An older build stored remote embeddings with a NULL endpoint and resolved
|
||||
it to the generation `base_url` at runtime. Opening the store writes that
|
||||
endpoint in once, so the collection, and the memories in it, stay put; a
|
||||
later edit of the generation endpoint no longer moves them.
|
||||
"""
|
||||
from one_memory.hy_sdk_runtime import vector_collection_bases
|
||||
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
store = ModelProfileStore(database)
|
||||
secrets = InMemorySecretStore()
|
||||
profile = store.create(
|
||||
ModelProfileInput(
|
||||
name="Remote",
|
||||
provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
base_url="https://gateway.example.com/v1",
|
||||
model="qwen",
|
||||
embedding_model="text-embedding-3-small",
|
||||
embedding_base_url="https://gateway.example.com/v1",
|
||||
embedding_dimensions=1536,
|
||||
api_key=SecretStr(API_KEY),
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
identity = (profile.profile_id, profile.revision)
|
||||
# The space the older build's Runtime resolved: `embedding_base_url or base_url`.
|
||||
before = vector_collection_bases(store.snapshot(*identity))[0]
|
||||
with sqlite3.connect(database) as db:
|
||||
# The shape an older build stored.
|
||||
db.execute("UPDATE model_profiles SET embedding_base_url=NULL")
|
||||
db.execute("UPDATE model_profile_versions SET embedding_base_url=NULL")
|
||||
|
||||
migrated = ModelProfileStore(database)
|
||||
assert migrated.snapshot(*identity).embedding_base_url == "https://gateway.example.com/v1"
|
||||
assert _space(migrated, profile.profile_id) == before
|
||||
# Idempotent: opening again changes nothing.
|
||||
assert _space(ModelProfileStore(database), profile.profile_id) == before
|
||||
with sqlite3.connect(database) as db:
|
||||
assert db.execute(
|
||||
"SELECT COUNT(*) FROM model_profile_versions WHERE embedding_base_url IS NULL"
|
||||
).fetchone() == (0,)
|
||||
|
||||
edited = migrated.update(
|
||||
profile.profile_id,
|
||||
ModelProfilePatch(
|
||||
expected_revision=profile.revision,
|
||||
base_url="https://gateway.example.com/eu/v1",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
assert edited.base_url == "https://gateway.example.com/eu/v1"
|
||||
assert edited.embedding_base_url == "https://gateway.example.com/v1"
|
||||
assert _space(migrated, profile.profile_id) == before
|
||||
@@ -650,6 +650,75 @@ def test_two_hermes_grants_each_read_their_own_root_into_separate_batches(tmp_pa
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_a_migrated_home_with_the_old_grant_triggers_can_connect_and_revoke(tmp_path) -> None:
|
||||
"""#233: a home an earlier build migrated kept two `source_grants` triggers
|
||||
whose body reads the dropped `governed_memory_evidence`. Every grant write
|
||||
then failed, so connecting and revoking answered 503. Startup drops them.
|
||||
"""
|
||||
stale = """
|
||||
CREATE TRIGGER memory_visibility_dirty_on_grants{suffix}
|
||||
AFTER {event} ON source_grants
|
||||
BEGIN
|
||||
INSERT INTO memory_visibility_dirty (memory_id)
|
||||
SELECT DISTINCT link.memory_id
|
||||
FROM governed_memory_evidence link
|
||||
JOIN evidence e ON e.evidence_id = link.evidence_id
|
||||
WHERE e.source_id = NEW.source_id
|
||||
ON CONFLICT(memory_id) DO NOTHING;
|
||||
END;"""
|
||||
|
||||
async def scenario():
|
||||
home = tmp_path / "home"
|
||||
_hermes, wife = _hermes_homes(home)
|
||||
app = application(tmp_path, home=home)
|
||||
async with app.router.lifespan_context(app):
|
||||
database = app.state.settings.data_dir / "one-memory.sqlite3"
|
||||
with sqlite3.connect(database) as db:
|
||||
db.execute("CREATE TABLE IF NOT EXISTS memory_visibility_dirty (memory_id TEXT)")
|
||||
db.execute(stale.format(suffix="", event="INSERT"))
|
||||
db.execute(stale.format(suffix="_update", event="UPDATE"))
|
||||
|
||||
app = application(tmp_path, home=home)
|
||||
async with app.router.lifespan_context(app), client_for(app) as client:
|
||||
location = (await client.post("/api/v1/sources/hermes/location")).json()
|
||||
local = await client.post(
|
||||
"/api/v1/sources/connect",
|
||||
json={
|
||||
"source_ids": ["hermes"],
|
||||
"request_id": str(uuid4()),
|
||||
"source_roots": {"hermes": location["root_id"]},
|
||||
},
|
||||
)
|
||||
assert local.status_code == 200, local.text
|
||||
profile = await client.post(
|
||||
"/api/v1/sources/connect",
|
||||
json={
|
||||
"source_ids": ["hermes"],
|
||||
"request_id": str(uuid4()),
|
||||
"user_id": "wife",
|
||||
"root": str(wife),
|
||||
},
|
||||
)
|
||||
assert profile.status_code == 200, profile.text
|
||||
for user_id in ("wife", "local"):
|
||||
revoked = await client.post(
|
||||
"/api/v1/sources/hermes/revoke", params={"user_id": user_id}
|
||||
)
|
||||
assert revoked.status_code == 200, revoked.text
|
||||
assert revoked.json()["revoked_at"] is not None
|
||||
with sqlite3.connect(database) as db:
|
||||
assert (
|
||||
db.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master "
|
||||
"WHERE type='trigger' AND tbl_name='source_grants'"
|
||||
).fetchone()[0]
|
||||
== 0
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_only_hermes_takes_another_user_or_a_custom_root(tmp_path) -> None:
|
||||
async def scenario():
|
||||
|
||||
@@ -40,6 +40,7 @@ def library(tmp_path):
|
||||
provider_kind=ProviderKind.OLLAMA,
|
||||
model="hy-bot-phi4",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_base_url="http://127.0.0.1:11434",
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,744 @@
|
||||
"""The SDK surface MemHub exposes: search pass-through, layers, raw, v2 aliases.
|
||||
|
||||
P2-1 (R8): `context/search`, `recall` and the v2 ask routes pass the public
|
||||
`client.search` knobs through, validated, and keep a caller's floor on what
|
||||
they return. P2-2 (R9): the library honours `layers`, `statuses`, `scene` and
|
||||
the source/session/agent filters, and counts memories per layer. P2-4: raw
|
||||
lists the SDK's L1 raw turns, active and shadow. P2-3: v2 aliases of the
|
||||
source and import routes, per user. Profile versions read the L0 chain.
|
||||
|
||||
Every route stays scoped to its one `user_id`; the stand-in client records
|
||||
each call so a test can see exactly what reached the SDK.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import sqlite3
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from pydantic import SecretStr
|
||||
|
||||
from one_memory.api import create_app
|
||||
from one_memory.config import Settings
|
||||
from one_memory.imports.sources.registry import build_source_catalog
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
TOKEN = "sdk-surface-test"
|
||||
|
||||
CHAT_ROWS = {
|
||||
"local": [
|
||||
("m-high", "local likes green tea", 0.95, "l2_fact", "hermes"),
|
||||
("m-mid", "local walks at dawn", 0.6, "l2_fact", "codex"),
|
||||
],
|
||||
"wife": [("w-high", "wife likes coffee", 0.92, "l2_fact", "hermes")],
|
||||
}
|
||||
RAW_ROWS = {
|
||||
"local": [
|
||||
{
|
||||
"memory_id": "raw-1",
|
||||
"content": "[user]: 我叫费天翔\n[assistant]: 好的,记住了。",
|
||||
"status": "shadow",
|
||||
"session_id": "s-1",
|
||||
"source": "hermes",
|
||||
"memory_at": 1_790_000_000,
|
||||
"gmt_created": 1_790_000_100,
|
||||
},
|
||||
{
|
||||
"memory_id": "raw-2",
|
||||
"content": "[user]: 你好",
|
||||
"status": "active",
|
||||
"session_id": "s-2",
|
||||
"source": "codex",
|
||||
"memory_at": 1_790_100_000,
|
||||
"gmt_created": 1_790_100_100,
|
||||
},
|
||||
],
|
||||
"wife": [
|
||||
{
|
||||
"memory_id": "raw-w",
|
||||
"content": "[user]: I am the wife",
|
||||
"status": "active",
|
||||
"session_id": "s-w",
|
||||
"source": "hermes",
|
||||
"memory_at": 1_790_000_000,
|
||||
"gmt_created": 1_790_000_000,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _chat(user_id: str, memory_id: str, content: str, layer: str, source: str) -> dict:
|
||||
return {
|
||||
"memory_id": memory_id,
|
||||
"content": content,
|
||||
"layer": layer,
|
||||
"status": "active",
|
||||
"user_id": user_id,
|
||||
"agent_id": "all-apps",
|
||||
"session_id": "s-1",
|
||||
"source": source,
|
||||
"tags": [],
|
||||
"gmt_created": 1_790_000_000,
|
||||
}
|
||||
|
||||
|
||||
class RecordingSdk:
|
||||
"""The public client surface, recording every call's keyword arguments."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
def _record(self, name: str, kwargs: dict) -> None:
|
||||
self.calls.append((name, dict(kwargs)))
|
||||
|
||||
def last(self, name: str) -> dict:
|
||||
return [kwargs for called, kwargs in self.calls if called == name][-1]
|
||||
|
||||
def all(self, name: str) -> list[dict]:
|
||||
return [kwargs for called, kwargs in self.calls if called == name]
|
||||
|
||||
# The SDK answers rows below a caller's floor in some lanes (a schema
|
||||
# floor, a fused rank score), so Core must filter what it returns.
|
||||
def search(self, query, **kwargs):
|
||||
self._record("search", {"query": query, **kwargs})
|
||||
[user_id] = kwargs["user_ids"]
|
||||
if kwargs.get("layers") == ["l1_raw"]:
|
||||
hits = [
|
||||
{**row, "user_id": user_id, "agent_id": "all-apps", "score": score}
|
||||
for row, score in zip(RAW_ROWS.get(user_id, []), (0.97, 0.3), strict=False)
|
||||
]
|
||||
return {"request_id": "s", "memories": hits}
|
||||
rows = [
|
||||
{**_chat(user_id, mid, text, layer, source), "score": score}
|
||||
for mid, text, score, layer, source in CHAT_ROWS.get(user_id, [])
|
||||
]
|
||||
scene = kwargs.get("scene")
|
||||
if scene == "all":
|
||||
fused = [
|
||||
{**row, "scene": "chat", "kind": "chat", "lane_score": row["score"], "score": 0.016}
|
||||
for row in rows
|
||||
]
|
||||
return {"request_id": "s", "scene": "all", "memories": fused, "lanes": {}, "errors": {}}
|
||||
if scene in {"working", "coding"}:
|
||||
return {
|
||||
"request_id": "s",
|
||||
"scene": "productivity",
|
||||
"variant": scene,
|
||||
"facts": [
|
||||
{"fact_id": "f-1", "statement": "use uv", "score": 0.91, "user_id": user_id}
|
||||
],
|
||||
"experiences": [],
|
||||
}
|
||||
return {
|
||||
"request_id": "s",
|
||||
"scene": "normal",
|
||||
"memories": {"normal": rows, "schema": [], "proactive": []},
|
||||
}
|
||||
|
||||
def get(self, memory_id, kind=None, variant=None):
|
||||
self._record("get", {"memory_id": memory_id, "kind": kind, "variant": variant})
|
||||
if kind == "fact":
|
||||
return {
|
||||
"kind": "fact",
|
||||
"variant": variant,
|
||||
"fact_id": memory_id,
|
||||
"statement": "use uv",
|
||||
"status": "active",
|
||||
"user_id": "local",
|
||||
}
|
||||
for user_id, rows in CHAT_ROWS.items():
|
||||
for mid, text, _score, layer, source in rows:
|
||||
if mid == memory_id:
|
||||
return {**_chat(user_id, mid, text, layer, source), "kind": "chat"}
|
||||
return None
|
||||
|
||||
def list_memories(self, **kwargs):
|
||||
self._record("list_memories", kwargs)
|
||||
user_id = kwargs["user_id"]
|
||||
layers = kwargs.get("layers")
|
||||
if layers == ["l1_raw"]:
|
||||
statuses = set(kwargs.get("statuses") or ["active"])
|
||||
rows = [
|
||||
{**row, "layer": "l1_raw", "user_id": user_id, "agent_id": "all-apps"}
|
||||
for row in RAW_ROWS.get(user_id, [])
|
||||
if row["status"] in statuses
|
||||
and (
|
||||
not kwargs.get("include_sources") or row["source"] in kwargs["include_sources"]
|
||||
)
|
||||
and (not kwargs.get("session_ids") or row["session_id"] in kwargs["session_ids"])
|
||||
]
|
||||
return {"vdb": {"memories": rows, "total": len(rows)}}
|
||||
rows = [_chat(user_id, *row[:2], *row[3:]) for row in CHAT_ROWS.get(user_id, [])]
|
||||
if layers:
|
||||
rows = [row for row in rows if row["layer"] in layers]
|
||||
if "l0_basic_info" in layers:
|
||||
rows.append(_chat(user_id, "l0-1", "name is Fei", "l0_basic_info", "hermes"))
|
||||
if kwargs.get("scene") == "all":
|
||||
return {
|
||||
"scene": "all",
|
||||
"memories": [{**row, "scene": "chat", "kind": "chat"} for row in rows],
|
||||
"total": len(rows),
|
||||
"totals": {"chat": len(rows), "working": 0, "coding": 0},
|
||||
"errors": {},
|
||||
}
|
||||
return {"vdb": {"memories": rows, "total": len(rows)}}
|
||||
|
||||
def memory_stats(self, *, user_id, statuses=None):
|
||||
return {"total": 0, "request_id": "stats", "elapsed_ms": 1.0}
|
||||
|
||||
def get_user_basic_info(self, *, user_id):
|
||||
self._record("get_user_basic_info", {"user_id": user_id})
|
||||
return {
|
||||
"request_id": f"basic-{user_id}",
|
||||
"elapsed_ms": 1.5,
|
||||
"memories": [
|
||||
{
|
||||
"memory_id": f"{user_id}-l0-head",
|
||||
"content": "The user's name is Fei.",
|
||||
"layer": "l0_basic_info",
|
||||
"status": "active",
|
||||
"memory_at": 1_790_000_100,
|
||||
"gmt_created": 1_790_000_100,
|
||||
"tags": [],
|
||||
"basic_info_kv": {"name": "Fei"},
|
||||
"evolution_chain": [
|
||||
{
|
||||
"node_id": f"{user_id}-l0-head",
|
||||
"content": "The user's name is Fei.",
|
||||
"memory_at": "2026-10-31 10:00",
|
||||
"gmt_created": 1_790_000_100,
|
||||
"layer": "l0_basic_info",
|
||||
},
|
||||
{
|
||||
"node_id": f"{user_id}-l0-old",
|
||||
"content": "The user's name is 费天翔.",
|
||||
"memory_at": "2026-10-30 10:00",
|
||||
"gmt_created": 1_790_000_000,
|
||||
"layer": "l0_basic_info",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class Handle:
|
||||
def __init__(self, sdk: RecordingSdk) -> None:
|
||||
self.sdk = sdk
|
||||
|
||||
async def client(self) -> RecordingSdk:
|
||||
return self.sdk
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def application(tmp_path: Path, *, home: Path | None = None):
|
||||
settings = Settings(
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=home or tmp_path,
|
||||
api_token=SecretStr(TOKEN),
|
||||
imports_enabled=home is not None,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
)
|
||||
return create_app(
|
||||
settings,
|
||||
secret_store=InMemorySecretStore(),
|
||||
source_catalog=(
|
||||
build_source_catalog(home=home, environment={}, platform="darwin")
|
||||
if home is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def client_for(app) -> AsyncClient:
|
||||
return AsyncClient(
|
||||
transport=ASGITransport(app=app),
|
||||
base_url="http://test",
|
||||
headers={"Authorization": f"Bearer {TOKEN}"},
|
||||
)
|
||||
|
||||
|
||||
async def _model_ready(_app) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def run_with_sdk(tmp_path, monkeypatch, scenario) -> None:
|
||||
async def main() -> None:
|
||||
app = application(tmp_path)
|
||||
sdk = RecordingSdk()
|
||||
|
||||
@asynccontextmanager
|
||||
async def write_client():
|
||||
yield sdk
|
||||
|
||||
async with app.router.lifespan_context(app):
|
||||
app.state.sdk_client_handle = Handle(sdk)
|
||||
monkeypatch.setattr(app.state.sdk_write_coordinator, "use_client", write_client)
|
||||
monkeypatch.setattr("one_memory.api._recall_model_ready", _model_ready)
|
||||
async with client_for(app) as client:
|
||||
await scenario(app, sdk, client)
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
PASS_THROUGH = {
|
||||
"min_score": 0.5,
|
||||
"schema_min_score": 0.6,
|
||||
"working_fact_min_score": 0.3,
|
||||
"working_experience_min_score": 0.45,
|
||||
"schema_limit": 4,
|
||||
"intention_limit": 2,
|
||||
"rerank": False,
|
||||
"reader": "legacy",
|
||||
"scene": "chat",
|
||||
"search_mode": "agentic",
|
||||
"include_sources": ["hermes", "codex"],
|
||||
"exclude_sources": ["cursor"],
|
||||
"created_after": 1_780_000_000,
|
||||
"session_ids": ["s-1"],
|
||||
"agent_ids": ["all-apps"],
|
||||
"workspace_id": "ws-1",
|
||||
"branch": "main",
|
||||
"layers": ["l2_fact", "l0_basic_info"],
|
||||
}
|
||||
#: How each field reaches `client.search` (lists stay lists).
|
||||
EXPECTED_SEARCH = {**PASS_THROUGH}
|
||||
|
||||
|
||||
def _memory_search(sdk: RecordingSdk) -> dict:
|
||||
return [call for call in sdk.all("search") if call.get("layers") != ["l1_raw"]][-1]
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_every_context_search_option_reaches_the_sdk_and_absent_ones_keep_the_old_call(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
baseline = await client.post(
|
||||
"/api/v2/context/search", json={"query": "tea", "kinds": ["memory"]}
|
||||
)
|
||||
assert baseline.status_code == 200, baseline.text
|
||||
before = _memory_search(sdk)
|
||||
assert before["min_score"] == before["schema_min_score"] == 0.35
|
||||
assert before["scene"] == "normal" and before["reader"] == "hybrid"
|
||||
assert before["schema_limit"] == 3 and before["intention_limit"] == 1
|
||||
for absent in ("rerank", "search_mode", "layers", "session_ids", "created_after"):
|
||||
assert absent not in before
|
||||
|
||||
response = await client.post(
|
||||
"/api/v2/context/search",
|
||||
json={"query": "tea", "kinds": ["memory"], **PASS_THROUGH},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
sent = _memory_search(sdk)
|
||||
for name, value in EXPECTED_SEARCH.items():
|
||||
assert sent[name] == value, name
|
||||
assert sent["user_ids"] == ["local"]
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_every_recall_and_ask_option_reaches_the_sdk(tmp_path, monkeypatch) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
recalled = await client.post(
|
||||
"/api/v1/recall", json={"query": "tea", "user_id": "wife", **PASS_THROUGH}
|
||||
)
|
||||
assert recalled.status_code == 200, recalled.text
|
||||
sent = _memory_search(sdk)
|
||||
for name, value in EXPECTED_SEARCH.items():
|
||||
assert sent[name] == value, name
|
||||
assert sent["user_ids"] == ["wife"]
|
||||
|
||||
ask_options = {
|
||||
name: value
|
||||
for name, value in PASS_THROUGH.items()
|
||||
if name
|
||||
not in {
|
||||
"working_fact_min_score",
|
||||
"working_experience_min_score",
|
||||
"schema_limit",
|
||||
"intention_limit",
|
||||
"reader",
|
||||
"scene",
|
||||
"search_mode",
|
||||
"workspace_id",
|
||||
"branch",
|
||||
}
|
||||
}
|
||||
asked = await client.post(
|
||||
"/api/v2/memories/ask", json={"question": "tea?", "user_id": "wife", **ask_options}
|
||||
)
|
||||
assert asked.status_code == 200, asked.text
|
||||
sent = _memory_search(sdk)
|
||||
for name, value in ask_options.items():
|
||||
assert sent[name] == value, name
|
||||
assert sent["user_ids"] == ["wife"] and sent["scene"] == "normal"
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_min_score_is_enforced_on_what_core_returns(tmp_path, monkeypatch) -> None:
|
||||
"""Acceptance item 9: `min_score=0.9` answers only results scoring ≥ 0.9."""
|
||||
|
||||
async def scenario(app, sdk, client):
|
||||
context = await client.post(
|
||||
"/api/v2/context/search", json={"query": "tea", "min_score": 0.9}
|
||||
)
|
||||
assert context.status_code == 200, context.text
|
||||
items = context.json()["items"]
|
||||
assert items and all(item["score"] >= 0.9 for item in items)
|
||||
assert {item.get("memory_id") or item.get("raw_id") for item in items} == {
|
||||
"m-high",
|
||||
"raw-1",
|
||||
}
|
||||
raw_call = sdk.last("search") if sdk.last("search").get("layers") else None
|
||||
assert raw_call is None or raw_call["min_score"] == 0.9
|
||||
|
||||
# The fused scene ranks by 1/(60+rank); the lane's own score is the floor's.
|
||||
fused = await client.post(
|
||||
"/api/v2/context/search",
|
||||
json={"query": "tea", "kinds": ["memory"], "scene": "all", "min_score": 0.9},
|
||||
)
|
||||
assert [item["memory_id"] for item in fused.json()["items"]] == ["m-high"]
|
||||
assert fused.json()["items"][0]["score"] == 0.95
|
||||
|
||||
recalled = await client.post("/api/v1/recall", json={"query": "tea", "min_score": 0.9})
|
||||
assert recalled.status_code == 200, recalled.text
|
||||
assert [hit["memory_id"] for hit in recalled.json()["hits"]] == ["m-high"]
|
||||
|
||||
# A working-scene fact is read back from its own namespace.
|
||||
working = await client.post(
|
||||
"/api/v2/context/search",
|
||||
json={"query": "uv", "kinds": ["memory"], "scene": "working", "min_score": 0.9},
|
||||
)
|
||||
assert [item["memory_id"] for item in working.json()["items"]] == ["f-1"]
|
||||
assert sdk.last("get") == {"memory_id": "f-1", "kind": "fact", "variant": "working"}
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
@pytest.mark.parametrize(
|
||||
"extra",
|
||||
[
|
||||
{"min_score": 1.5},
|
||||
{"min_score": -0.1},
|
||||
{"schema_min_score": "high"},
|
||||
{"scene": "normal"},
|
||||
{"search_mode": "fast"},
|
||||
{"reader": "vector"},
|
||||
{"layers": ["l9_nothing"]},
|
||||
{"layers": []},
|
||||
{"branch": "main"},
|
||||
{"rerank": "no"},
|
||||
{"created_after": -1},
|
||||
{"user_ids": ["wife"]},
|
||||
{"embed_override": {"model": "x"}},
|
||||
{"query_embedding": [0.1]},
|
||||
{"request_id": "r-1"},
|
||||
],
|
||||
)
|
||||
def test_out_of_range_and_unexposed_search_fields_answer_422(tmp_path, extra) -> None:
|
||||
async def scenario():
|
||||
app = application(tmp_path)
|
||||
async with app.router.lifespan_context(app), client_for(app) as client:
|
||||
for path, body in (
|
||||
("/api/v2/context/search", {"query": "q"}),
|
||||
("/api/v1/recall", {"query": "q"}),
|
||||
("/api/v2/memories/ask", {"question": "q"}),
|
||||
):
|
||||
response = await client.post(path, json={**body, **extra})
|
||||
assert response.status_code == 422, (path, response.text)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_raw_lane_takes_the_callers_sources_only_inside_its_authority(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
response = await client.post(
|
||||
"/api/v2/context/search",
|
||||
json={
|
||||
"query": "tea",
|
||||
"kinds": ["raw"],
|
||||
"include_sources": ["hermes"],
|
||||
"session_ids": ["s-1"],
|
||||
"created_after": 1_700_000_000,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
raw_call = sdk.last("search")
|
||||
assert raw_call["layers"] == ["l1_raw"]
|
||||
assert raw_call["include_sources"] == ["hermes"]
|
||||
assert raw_call["session_ids"] == ["s-1"]
|
||||
assert raw_call["created_after"] == 1_700_000_000
|
||||
# Memory-lane knobs never reach the raw search.
|
||||
assert raw_call["scene"] == "chat" and raw_call["schema_limit"] == 0
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_library_honours_layers_statuses_and_filters(tmp_path, monkeypatch) -> None:
|
||||
"""Acceptance item 10: `layer=l0_basic_info` returns only L0 records."""
|
||||
|
||||
async def scenario(app, sdk, client):
|
||||
page = await client.get("/api/v2/memory-library", params={"layer": "l0_basic_info"})
|
||||
assert page.status_code == 200, page.text
|
||||
assert {memory["layer"] for memory in page.json()["memories"]} == {"l0_basic_info"}
|
||||
sent = sdk.last("list_memories")
|
||||
assert sent["layers"] == ["l0_basic_info"] and sent["scene"] is None
|
||||
|
||||
await client.get(
|
||||
"/api/v2/memory-library",
|
||||
params=[
|
||||
("layers", "l2_fact,l6_schema"),
|
||||
("layers", "l7_intention"),
|
||||
("statuses", "active,superseded"),
|
||||
("session_ids", "s-1"),
|
||||
("agent_ids", "all-apps"),
|
||||
("exclude_sources", "cursor"),
|
||||
("user_id", "wife"),
|
||||
],
|
||||
)
|
||||
sent = sdk.last("list_memories")
|
||||
assert sent["layers"] == ["l2_fact", "l6_schema", "l7_intention"]
|
||||
assert sent["statuses"] == ["active", "superseded"]
|
||||
assert sent["session_ids"] == ["s-1"] and sent["agent_ids"] == ["all-apps"]
|
||||
assert sent["exclude_sources"] == ["cursor"] and sent["user_id"] == "wife"
|
||||
|
||||
# Without a layer the page is what it always was.
|
||||
await client.get("/api/v2/memory-library")
|
||||
sent = sdk.last("list_memories")
|
||||
assert sent["scene"] == "all" and "layers" not in sent
|
||||
|
||||
searched = await client.get(
|
||||
"/api/v2/memory-library", params={"query": "tea", "layers": "l2_fact"}
|
||||
)
|
||||
assert searched.status_code == 200, searched.text
|
||||
assert sdk.last("search")["layers"] == ["l2_fact"]
|
||||
|
||||
for params in (
|
||||
{"layers": "l9_nothing"},
|
||||
{"statuses": "gone"},
|
||||
{"layers": "l2_fact", "scene": "working"},
|
||||
):
|
||||
refused = await client.get("/api/v2/memory-library", params=params)
|
||||
assert refused.status_code == 422, (params, refused.text)
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_layer_counts_read_each_layer_for_one_user(tmp_path, monkeypatch) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
counts = await client.get("/api/v2/memory-library/layers", params={"user_id": "wife"})
|
||||
assert counts.status_code == 200, counts.text
|
||||
body = counts.json()
|
||||
assert body["layers"]["l1_raw"] == 1
|
||||
assert body["layers"]["l2_fact"] == 1
|
||||
assert body["layers"]["l0_basic_info"] == 1
|
||||
assert body["total"] == sum(body["layers"].values())
|
||||
calls = sdk.all("list_memories")
|
||||
assert {call["user_id"] for call in calls} == {"wife"}
|
||||
[raw_call] = [call for call in calls if call["layers"] == ["l1_raw"]]
|
||||
assert raw_call["statuses"] == ["active", "shadow"]
|
||||
assert (
|
||||
await client.get("/api/v2/memory-library/layers", params={"layer": "x"})
|
||||
).status_code == 422
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_raw_lists_active_and_shadow_turns_with_roles_and_hashes(tmp_path, monkeypatch) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
page = await client.get("/api/v2/raw")
|
||||
assert page.status_code == 200, page.text
|
||||
body = page.json()
|
||||
assert body["total"] == 2 and len(body["items"]) == 2
|
||||
sent = sdk.last("list_memories")
|
||||
assert sent["layers"] == ["l1_raw"] and sent["statuses"] == ["active", "shadow"]
|
||||
first = next(item for item in body["items"] if item["raw_id"] == "raw-1")
|
||||
assert first["status"] == "shadow"
|
||||
assert [message["role"] for message in first["messages"]] == ["user", "assistant"]
|
||||
assert first["messages"][0]["content"] == "我叫费天翔"
|
||||
assert first["role"] is None
|
||||
expected = hashlib.sha256(first["content"].encode()).hexdigest()
|
||||
assert first["content_hash"] == f"sha256:{expected}"
|
||||
assert first["session_id"] == "s-1" and first["source"] == "hermes"
|
||||
assert first["created"].startswith("2026-")
|
||||
second = next(item for item in body["items"] if item["raw_id"] == "raw-2")
|
||||
assert second["role"] == "user"
|
||||
|
||||
await client.get(
|
||||
"/api/v2/raw",
|
||||
params=[("source", "hermes,codex"), ("session_id", "s-1"), ("order", "asc")],
|
||||
)
|
||||
sent = sdk.last("list_memories")
|
||||
assert sent["include_sources"] == ["hermes", "codex"]
|
||||
assert sent["session_ids"] == ["s-1"] and sent["order"] == "asc"
|
||||
|
||||
windowed = (await client.get("/api/v2/raw", params={"after": 1_790_050_000})).json()
|
||||
assert windowed["time_filtered"] is True
|
||||
assert [item["raw_id"] for item in windowed["items"]] == ["raw-2"]
|
||||
|
||||
wife = (await client.get("/api/v2/raw", params={"user_id": "wife"})).json()
|
||||
assert [item["raw_id"] for item in wife["items"]] == ["raw-w"]
|
||||
assert sdk.last("list_memories")["user_id"] == "wife"
|
||||
assert (await client.get("/api/v2/raw", params={"statuses": "bogus"})).status_code == 422
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_profile_versions_read_the_l0_chain_for_one_user(tmp_path, monkeypatch) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
response = await client.get("/api/v2/user-profile/versions", params={"user_id": "wife"})
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert body["user_id"] == "wife"
|
||||
assert sdk.last("get_user_basic_info") == {"user_id": "wife"}
|
||||
[chain] = body["chains"]
|
||||
assert chain["memory_id"] == "wife-l0-head"
|
||||
assert chain["basic_info"] == {"name": "Fei"}
|
||||
assert [(v["memory_id"], v["current"]) for v in chain["versions"]] == [
|
||||
("wife-l0-head", True),
|
||||
("wife-l0-old", False),
|
||||
]
|
||||
local = (await client.get("/api/v2/user-profile/versions")).json()
|
||||
assert local["user_id"] == "local"
|
||||
assert sdk.last("get_user_basic_info") == {"user_id": "local"}
|
||||
refused = await client.get("/api/v2/user-profile/versions", params={"owner": "wife"})
|
||||
assert refused.status_code == 422
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_graph_takes_layers_and_session_filters(tmp_path, monkeypatch) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
response = await client.get(
|
||||
"/api/v2/memories/graph",
|
||||
params={"layers": "l2_fact", "session_ids": "s-1", "user_id": "wife"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
sent = sdk.last("list_memories")
|
||||
assert sent["layers"] == ["l2_fact"] and sent["scene"] is None
|
||||
assert sent["session_ids"] == ["s-1"] and sent["user_id"] == "wife"
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
def _hermes_state_db(root: Path, text: str) -> None:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
connection = sqlite3.connect(root / "state.db")
|
||||
connection.execute(
|
||||
"CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT,"
|
||||
" content TEXT, timestamp REAL, tool_name TEXT, active INTEGER)"
|
||||
)
|
||||
connection.executemany(
|
||||
"INSERT INTO messages VALUES (?, ?, ?, ?, 1790000000, NULL, 1)",
|
||||
[(1, "session-1", "user", text), (2, "session-1", "assistant", "ok")],
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_v2_source_and_import_aliases_behave_like_v1_per_user(tmp_path) -> None:
|
||||
async def scenario():
|
||||
home = tmp_path / "home"
|
||||
hermes = home / ".hermes"
|
||||
wife = hermes / "profiles" / "wife"
|
||||
_hermes_state_db(hermes, "I am the husband")
|
||||
_hermes_state_db(wife, "I am the wife")
|
||||
app = application(tmp_path, home=home)
|
||||
async with app.router.lifespan_context(app), client_for(app) as client:
|
||||
location = (await client.post("/api/v1/sources/hermes/location")).json()
|
||||
local = await client.post(
|
||||
"/api/v1/sources/connect",
|
||||
json={
|
||||
"source_ids": ["hermes"],
|
||||
"request_id": str(uuid4()),
|
||||
"source_roots": {"hermes": location["root_id"]},
|
||||
"enable_sync": True,
|
||||
},
|
||||
)
|
||||
assert local.status_code == 200, local.text
|
||||
connected = await client.post(
|
||||
"/api/v1/sources/connect",
|
||||
json={
|
||||
"source_ids": ["hermes"],
|
||||
"request_id": str(uuid4()),
|
||||
"user_id": "wife",
|
||||
"root": str(wife),
|
||||
},
|
||||
)
|
||||
assert connected.status_code == 200, connected.text
|
||||
for job in (*local.json()["jobs"], *connected.json()["jobs"]):
|
||||
await app.state.import_supervisor.wait(job["job_id"])
|
||||
|
||||
v1 = (await client.get("/api/v1/sources")).json()
|
||||
v2 = (await client.get("/api/v2/sources")).json()
|
||||
assert v1 == v2
|
||||
wife_sources = (await client.get("/api/v2/sources", params={"user_id": "wife"})).json()
|
||||
assert [source["source_id"] for source in wife_sources] == ["hermes"]
|
||||
assert wife_sources[0]["authorized"] is True
|
||||
assert (await client.get("/api/v2/sources", params={"u": "x"})).status_code == 422
|
||||
|
||||
created = await client.post(
|
||||
"/api/v2/import-jobs",
|
||||
json={"source_id": "hermes", "operation": "sync", "user_id": "wife"},
|
||||
)
|
||||
assert created.status_code == 200, created.text
|
||||
wife_job = created.json()
|
||||
assert wife_job["user_id"] == "wife" and wife_job["source_id"] == "hermes"
|
||||
await app.state.import_supervisor.wait(wife_job["job_id"])
|
||||
refused = await client.post(
|
||||
"/api/v2/import-jobs",
|
||||
json={"source_id": "claude-code", "user_id": "wife"},
|
||||
)
|
||||
assert refused.status_code in {404, 422}
|
||||
unknown = await client.post(
|
||||
"/api/v2/import-jobs",
|
||||
json={"source_id": "hermes", "from_beginning": True},
|
||||
)
|
||||
assert unknown.status_code == 422
|
||||
|
||||
listed = (await client.get("/api/v2/import-jobs", params={"user_id": "wife"})).json()
|
||||
assert {job["user_id"] for job in listed} == {"wife"}
|
||||
assert (await client.get("/api/v2/import-jobs")).json() == (
|
||||
await client.get("/api/v1/import-jobs")
|
||||
).json()
|
||||
path = f"/api/v2/import-jobs/{wife_job['job_id']}"
|
||||
assert (await client.get(path)).status_code == 404
|
||||
own = await client.get(path, params={"user_id": "wife"})
|
||||
assert own.status_code == 200 and own.json()["job_id"] == wife_job["job_id"]
|
||||
|
||||
synced = await client.post("/api/v2/sources/hermes/sync", json={"user_id": "wife"})
|
||||
v1_synced = await client.post("/api/v1/sources/hermes/sync")
|
||||
assert synced.status_code == v1_synced.status_code, (synced.text, v1_synced.text)
|
||||
assert synced.status_code == 200
|
||||
assert synced.json()["user_id"] == "wife"
|
||||
assert v1_synced.json()["user_id"] == "local"
|
||||
other = await client.post("/api/v2/sources/claude-code/sync", json={"user_id": "wife"})
|
||||
assert other.status_code in {404, 422}
|
||||
assert (
|
||||
await client.post("/api/v2/sources/hermes/sync", json={"owner": "wife"})
|
||||
).status_code == 422
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -708,6 +708,7 @@ def test_cutover_waits_until_the_new_runtime_reads_every_chat_memory(store, acti
|
||||
"chat_readable": 1,
|
||||
"productivity_expected": 0,
|
||||
"productivity_readable": 0,
|
||||
"chat_missing_before_migration": 0,
|
||||
"passed": False,
|
||||
}
|
||||
assert store.authority() is MemoryAuthority.LEGACY
|
||||
@@ -774,18 +775,186 @@ def test_retired_coding_experiences_do_not_block_but_an_empty_store_does(store,
|
||||
assert (result.readback.productivity_expected, result.readback.productivity_readable) == (2, 1)
|
||||
|
||||
|
||||
def test_memories_without_a_model_profile_keep_legacy_authority(store, monkeypatch):
|
||||
def test_a_home_without_a_model_profile_migrates_on_the_builtin_embedding(store, monkeypatch):
|
||||
"""#234: no generation model was ever configured, and the upgrade stopped.
|
||||
|
||||
No profile means no LLM, not no embedding: the vectors are the built-in
|
||||
model's, the same space the Runtime reads with. They are reattached and
|
||||
read back through it, and the client is opened without resolving a key.
|
||||
"""
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot, vector_collection_bases
|
||||
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: None)
|
||||
home = store.database_path.parent
|
||||
builtin = builtin_retrieval_snapshot()
|
||||
_current, legacy_bases = vector_collection_bases(builtin)
|
||||
client = _chroma(home)
|
||||
_fill(client, f"{legacy_bases[0]}_512", ["m1", "m2"])
|
||||
del client
|
||||
_confirmed(store, "m1", "m2")
|
||||
opened: list[str] = []
|
||||
|
||||
def open_client(profile):
|
||||
opened.append(profile.profile_id)
|
||||
return _reading({"m1", "m2"})(profile)
|
||||
|
||||
result = run_startup_migration(
|
||||
data_dir=home,
|
||||
runtime_build="21",
|
||||
sdk_version="2.0.5",
|
||||
namespace="governed-v2",
|
||||
open_client=open_client,
|
||||
)
|
||||
|
||||
assert result.state == "cutover"
|
||||
assert opened == [builtin.profile_id]
|
||||
assert [item.legacy for item in result.reattached] == [f"{legacy_bases[0]}_512"]
|
||||
assert result.readback is not None and result.readback.passed
|
||||
assert store.authority() is MemoryAuthority.SDK
|
||||
|
||||
|
||||
def test_the_builtin_readback_client_resolves_no_secret(tmp_path, monkeypatch):
|
||||
"""The built-in snapshot has no stored version and no key to look up."""
|
||||
from one_memory import sdk_migration_verify
|
||||
from one_memory.hy_sdk_runtime import builtin_retrieval_snapshot
|
||||
|
||||
leased: list[tuple[str, object]] = []
|
||||
|
||||
class Lease:
|
||||
client = object()
|
||||
|
||||
def close(self) -> None:
|
||||
leased.append(("closed", None))
|
||||
|
||||
def acquire(profile, *, api_key, data_dir):
|
||||
leased.append((profile.profile_id, api_key))
|
||||
return Lease()
|
||||
|
||||
def no_lookup(*_args, **_kwargs):
|
||||
raise AssertionError("a key-less profile must not touch the secret store")
|
||||
|
||||
monkeypatch.setattr("one_memory.hy_sdk_runtime.acquire_client", acquire)
|
||||
monkeypatch.setattr(
|
||||
"one_memory.model_profiles.ModelProfileStore.resolve_snapshot_secret", no_lookup
|
||||
)
|
||||
with sdk_migration_verify.runtime_sdk_client(
|
||||
tmp_path / "missing.sqlite3", tmp_path, builtin_retrieval_snapshot()
|
||||
) as client:
|
||||
assert client is Lease.client
|
||||
assert leased == [("builtin-retrieval", None), ("closed", None)]
|
||||
|
||||
|
||||
def test_vectors_left_by_a_deleted_profile_stop_the_upgrade_by_name(store, monkeypatch):
|
||||
"""No active profile, and the vectors sit in another profile's space.
|
||||
|
||||
The built-in model cannot read them, so the old abort was right for this
|
||||
one shape; it now says what is actually wrong, before anything changes.
|
||||
"""
|
||||
from one_memory.hy_sdk_runtime import vector_collection_bases
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind
|
||||
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: None)
|
||||
deleted = ModelProfileSnapshot(
|
||||
profile_id="deleted",
|
||||
revision=3,
|
||||
provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
base_url="https://models.example/v1",
|
||||
model="chat",
|
||||
embedding_provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
embedding_base_url="https://models.example/v1",
|
||||
embedding_model="text-embedding-3-large",
|
||||
embedding_dimensions=1024,
|
||||
timeout_seconds=60,
|
||||
max_concurrency=1,
|
||||
context_window=None,
|
||||
has_api_key=True,
|
||||
)
|
||||
current, _legacy = vector_collection_bases(deleted)
|
||||
home = store.database_path.parent
|
||||
client = _chroma(home)
|
||||
_fill(client, f"{current}_1024", ["m1"])
|
||||
del client
|
||||
_confirmed(store, "m1")
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError, match="sdk_migration_model_profile_required"):
|
||||
with pytest.raises(DataPlaneMigrationError, match="sdk_migration_vector_profile_missing"):
|
||||
run_startup_migration(
|
||||
data_dir=store.database_path.parent,
|
||||
data_dir=home,
|
||||
runtime_build="21",
|
||||
sdk_version="2.0.5",
|
||||
namespace="governed-v2",
|
||||
open_client=_reading({"m1"}),
|
||||
)
|
||||
assert store.authority() is MemoryAuthority.LEGACY
|
||||
assert not (home / "backups").exists()
|
||||
|
||||
|
||||
def test_confirmed_chats_the_sdk_had_already_merged_do_not_fail_the_upgrade(store, active_profile):
|
||||
"""#229/#240: 658 confirmed chats, 657 in the store, and the upgrade failed.
|
||||
|
||||
A 1.0.4 library can hold confirmed rows whose SDK node had been merged or
|
||||
deleted before the upgrade; no migration can read those back. The check
|
||||
holds chat to what the store held before this run and counts the rest.
|
||||
"""
|
||||
from one_memory.hy_sdk_runtime import vector_collection_bases
|
||||
|
||||
home = store.database_path.parent
|
||||
stored = [f"chat-{index:03d}" for index in range(657)]
|
||||
current, _legacy = vector_collection_bases(active_profile)
|
||||
client = _chroma(home)
|
||||
_fill(client, f"{current}_512", stored)
|
||||
del client
|
||||
_confirmed(store, *stored, "chat-merged-before-upgrade")
|
||||
|
||||
result = run_startup_migration(
|
||||
data_dir=home,
|
||||
runtime_build="21",
|
||||
sdk_version="2.0.5",
|
||||
namespace="governed-v2",
|
||||
open_client=_reading(set(stored)),
|
||||
)
|
||||
|
||||
assert result.state == "cutover"
|
||||
assert result.readback is not None
|
||||
assert result.readback.as_document() == {
|
||||
"chat_expected": 657,
|
||||
"chat_readable": 657,
|
||||
"productivity_expected": 0,
|
||||
"productivity_readable": 0,
|
||||
"chat_missing_before_migration": 1,
|
||||
"passed": True,
|
||||
}
|
||||
assert store.authority() is MemoryAuthority.SDK
|
||||
|
||||
|
||||
def test_a_chat_in_the_store_before_the_upgrade_must_still_read_back(store, active_profile):
|
||||
"""Only what was already gone is excused; losing a stored memory still fails."""
|
||||
from one_memory.hy_sdk_runtime import vector_collection_bases
|
||||
from one_memory.sdk_migration_verify import MemoryReadbackFailed
|
||||
|
||||
home = store.database_path.parent
|
||||
current, _legacy = vector_collection_bases(active_profile)
|
||||
client = _chroma(home)
|
||||
_fill(client, f"{current}_512", ["m1", "m2"])
|
||||
del client
|
||||
_confirmed(store, "m1", "m2", "merged")
|
||||
|
||||
with pytest.raises(MemoryReadbackFailed) as caught:
|
||||
run_startup_migration(
|
||||
data_dir=home,
|
||||
runtime_build="21",
|
||||
sdk_version="2.0.5",
|
||||
namespace="governed-v2",
|
||||
open_client=_reading({"m1"}),
|
||||
)
|
||||
assert caught.value.report.as_document() == {
|
||||
"chat_expected": 2,
|
||||
"chat_readable": 1,
|
||||
"productivity_expected": 0,
|
||||
"productivity_readable": 0,
|
||||
"chat_missing_before_migration": 1,
|
||||
"passed": False,
|
||||
}
|
||||
assert store.authority() is MemoryAuthority.LEGACY
|
||||
|
||||
|
||||
def test_a_build_20_home_gets_its_vectors_back_and_is_checked_against_its_backup(
|
||||
@@ -1072,7 +1241,13 @@ def test_host_capture_is_fenced_for_the_whole_migration(store):
|
||||
)
|
||||
|
||||
assert result.state == "cutover"
|
||||
assert fenced == {"bundle": True, "prepare": True, "cutover": True, "cleanup": True}
|
||||
assert fenced == {
|
||||
"bundle": True,
|
||||
"prepare": True,
|
||||
"vectors": True,
|
||||
"cutover": True,
|
||||
"cleanup": True,
|
||||
}
|
||||
assert not sdk_migration_capture_fenced(home)
|
||||
|
||||
|
||||
|
||||
@@ -306,7 +306,7 @@ def test_the_runner_names_the_step_and_sub_step_that_failed(tmp_path, monkeypatc
|
||||
("step.finished", "preflight.profile_read"),
|
||||
("step.finished", "preflight.resumable_identity"),
|
||||
("step.finished", "preflight.legacy_ids_scan"),
|
||||
("step.finished", "preflight.model_profile_check"),
|
||||
("step.finished", "preflight.vector_space_check"),
|
||||
("step.finished", "prepare.recover_abandoned_staging"),
|
||||
("step.finished", "bundle.ensure"),
|
||||
("step.finished", "bundle.restore_point"),
|
||||
|
||||
@@ -463,11 +463,17 @@ def test_a_run_killed_after_its_restore_point_is_rolled_back_on_the_next_start(h
|
||||
|
||||
|
||||
def test_a_preflight_failure_changes_nothing_and_needs_no_restore(home, monkeypatch):
|
||||
def unreadable(_data_dir, _profile):
|
||||
raise DataPlaneMigrationError("sdk_migration_vector_profile_missing")
|
||||
|
||||
monkeypatch.setattr("one_memory.sdk_migration_runner.read_active_profile", lambda _db: None)
|
||||
monkeypatch.setattr(
|
||||
"one_memory.sdk_migration_runner.vectors_belong_to_another_space", unreadable
|
||||
)
|
||||
before = _state(home)
|
||||
report: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(DataPlaneMigrationError, match="sdk_migration_model_profile_required"):
|
||||
with pytest.raises(DataPlaneMigrationError, match="sdk_migration_vector_profile_missing"):
|
||||
_run(home, report=report)
|
||||
|
||||
assert report == {}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Model-free SDK search remains available without an active write profile."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -95,3 +96,158 @@ async def test_search_without_profile_uses_public_sdk_search() -> None:
|
||||
assert options["user_ids"] == ["local"]
|
||||
assert options["agent_ids"] == []
|
||||
assert options["scene"] == "normal"
|
||||
|
||||
|
||||
def _activate(profiles, secrets, *, dimensions: int, model: str = "chat"):
|
||||
from pydantic import SecretStr
|
||||
|
||||
from one_memory.model_profiles import ModelCapabilities, ModelProfileInput
|
||||
|
||||
profile = profiles.create(
|
||||
ModelProfileInput(
|
||||
name=f"embedding-{dimensions}",
|
||||
provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
base_url="https://models.example/v1",
|
||||
model=model,
|
||||
embedding_provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
embedding_base_url="https://models.example/v1",
|
||||
embedding_model="qwen3-embedding-4b",
|
||||
embedding_dimensions=dimensions,
|
||||
api_key=SecretStr("sk-test"),
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
profiles.record_capabilities(
|
||||
profile.profile_id,
|
||||
expected_revision=profile.revision,
|
||||
capabilities=ModelCapabilities(
|
||||
endpoint_reachable=True,
|
||||
model_available=True,
|
||||
structured_output=True,
|
||||
embedding_available=True,
|
||||
),
|
||||
secrets=secrets,
|
||||
)
|
||||
profiles.activate(profile.profile_id, secrets)
|
||||
return profiles.snapshot(profile.profile_id, profiles.active_identity()[1])
|
||||
|
||||
|
||||
class _Recorder:
|
||||
"""Stands in for `acquire_client`: records the collection each client opens."""
|
||||
|
||||
def __init__(self, data_dir: Path) -> None:
|
||||
self.data_dir = data_dir
|
||||
self.opened: list[tuple[str, int | None, str | None]] = []
|
||||
self.closed: list[str] = []
|
||||
|
||||
def __call__(self, profile, *, api_key, data_dir, http_transport_factory=None):
|
||||
from one_memory.hy_sdk_runtime import build_sdk_config
|
||||
|
||||
config = build_sdk_config(profile, api_key=api_key, data_dir=data_dir)
|
||||
name = config.vector_store.collection_name
|
||||
self.opened.append((name, config.vector_store.embedding_dims, api_key))
|
||||
return SimpleNamespace(
|
||||
client=SimpleNamespace(collection=name),
|
||||
close=lambda: self.closed.append(name),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_reads_open_the_collection_the_active_profile_writes(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""#242: a 2560-dim profile wrote `memhub_<sha>_2560`, reads opened `_512`."""
|
||||
from one_memory.hy_sdk_runtime import build_sdk_config
|
||||
from one_memory.model_profiles import ModelProfileStore
|
||||
from one_memory.sdk_read_handle import ActiveProfileReadHandle
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
recorder = _Recorder(tmp_path)
|
||||
monkeypatch.setattr("one_memory.hy_sdk_runtime.acquire_client", recorder)
|
||||
profiles = ModelProfileStore(tmp_path / "one-memory.sqlite3")
|
||||
secrets = InMemorySecretStore()
|
||||
handle = ActiveProfileReadHandle(profiles=profiles, secrets=secrets, data_dir=tmp_path)
|
||||
|
||||
builtin = build_sdk_config(builtin_retrieval_snapshot(), api_key=None, data_dir=tmp_path)
|
||||
assert (await handle.client()).collection == builtin.vector_store.collection_name
|
||||
|
||||
written = _activate(profiles, secrets, dimensions=2560)
|
||||
writes = build_sdk_config(written, api_key="sk-test", data_dir=tmp_path)
|
||||
|
||||
async def read() -> str:
|
||||
return str((await handle.client()).collection)
|
||||
|
||||
assert await asyncio.create_task(read()) == writes.vector_store.collection_name
|
||||
assert recorder.opened[-1] == (writes.vector_store.collection_name, 2560, "sk-test")
|
||||
assert writes.vector_store.collection_name != builtin.vector_store.collection_name
|
||||
await handle.close()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_a_profile_change_rebuilds_reads_without_closing_one_in_flight(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from one_memory.model_profiles import ModelProfileStore
|
||||
from one_memory.sdk_read_handle import ActiveProfileReadHandle
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
recorder = _Recorder(tmp_path)
|
||||
monkeypatch.setattr("one_memory.hy_sdk_runtime.acquire_client", recorder)
|
||||
profiles = ModelProfileStore(tmp_path / "one-memory.sqlite3")
|
||||
secrets = InMemorySecretStore()
|
||||
handle = ActiveProfileReadHandle(profiles=profiles, secrets=secrets, data_dir=tmp_path)
|
||||
_activate(profiles, secrets, dimensions=1024)
|
||||
|
||||
holding = asyncio.Event()
|
||||
finish = asyncio.Event()
|
||||
|
||||
async def slow_read() -> str:
|
||||
client = await handle.client()
|
||||
holding.set()
|
||||
await finish.wait()
|
||||
return str(client.collection)
|
||||
|
||||
in_flight = asyncio.create_task(slow_read())
|
||||
await holding.wait()
|
||||
[(old, _, _)] = recorder.opened
|
||||
|
||||
_activate(profiles, secrets, dimensions=2560, model="chat-2")
|
||||
|
||||
async def read() -> str:
|
||||
return str((await handle.client()).collection)
|
||||
|
||||
new = await asyncio.create_task(read())
|
||||
assert new != old
|
||||
# The read that started on the old space still has its client.
|
||||
assert recorder.closed == []
|
||||
|
||||
finish.set()
|
||||
assert await in_flight == old
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
assert recorder.closed == [old]
|
||||
await handle.close()
|
||||
assert recorder.closed == [old, new]
|
||||
|
||||
|
||||
def test_every_api_read_shares_the_active_profile_handle(tmp_path: Path) -> None:
|
||||
"""Library, search, recall, graph, profile, raw, ask and context all read
|
||||
through `_sdk_client_handle`, so it must be the active-profile handle."""
|
||||
from one_memory.api import _sdk_client_handle
|
||||
from one_memory.config import Settings
|
||||
from one_memory.model_profiles import ModelProfileStore
|
||||
from one_memory.sdk_read_handle import ActiveProfileReadHandle
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
state = SimpleNamespace(
|
||||
sdk_client_handle=None,
|
||||
model_profile_store=ModelProfileStore(tmp_path / "one-memory.sqlite3"),
|
||||
secret_store=InMemorySecretStore(),
|
||||
settings=Settings(_env_file=None, data_dir=tmp_path),
|
||||
)
|
||||
app = SimpleNamespace(state=state)
|
||||
|
||||
handle = _sdk_client_handle(app) # type: ignore[arg-type]
|
||||
|
||||
assert isinstance(handle, ActiveProfileReadHandle)
|
||||
assert _sdk_client_handle(app) is handle is state.sdk_client_handle # type: ignore[arg-type]
|
||||
@@ -133,6 +133,7 @@ const SAFE_STABLE_RUNTIME_ERROR_CODES = new Set([
|
||||
"sdk_migration_preflight_failed",
|
||||
"sdk_migration_memory_unreadable",
|
||||
"sdk_migration_model_profile_required",
|
||||
"sdk_migration_vector_profile_missing",
|
||||
"sdk_migration_model_secret_unavailable",
|
||||
"sdk_migration_secure_storage_unavailable",
|
||||
"sdk_migration_disk_space_insufficient",
|
||||
@@ -219,6 +220,8 @@ export const STABLE_RUNTIME_USER_MESSAGES: Record<string, string> = {
|
||||
"升级前核对发现,新版本还读不到你的全部记忆,所以没有切换数据源,旧数据保持原样可用。请把这个提示反馈给我们;原有记忆数据不会丢失。",
|
||||
sdk_migration_model_profile_required:
|
||||
"升级记忆数据格式需要先有可用的模型配置,用来核对新版本能否读回你的记忆。请在旧版本中完成模型配置后重启;你的记忆数据不会受到影响。",
|
||||
sdk_migration_vector_profile_missing:
|
||||
"你的记忆是用一个已删除的模型配置生成向量的,当前没有可用的模型配置能读回它们,所以没有升级。请在旧版本中重新添加那个模型配置并启用后重启;你的记忆数据不会受到影响。",
|
||||
sdk_migration_model_secret_unavailable:
|
||||
"模型密钥读不到(可能被系统钥匙串拒绝),无法核对新版本能否读回你的记忆。请允许 MemHub 访问钥匙串后重新启动;你的记忆数据不会受到影响。",
|
||||
sdk_migration_secure_storage_unavailable:
|
||||
|
||||
@@ -480,6 +480,30 @@ describe("ModelSettings", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("writes the prefilled embedding endpoint instead of leaving it to follow generation", async () => {
|
||||
profiles = [
|
||||
{
|
||||
...profile,
|
||||
embedding_provider_kind: "openai_compatible",
|
||||
embedding_model: "vector-old",
|
||||
embedding_base_url: null,
|
||||
},
|
||||
];
|
||||
render(<ModelSettings />);
|
||||
await userEvent.click(screen.getByRole("button", { name: "编辑配置" }));
|
||||
await userEvent.clear(screen.getByLabelText("向量模型名"));
|
||||
await userEvent.type(screen.getByLabelText("向量模型名"), "vector-new");
|
||||
await userEvent.click(
|
||||
screen.getByRole("button", { name: "高级:自行配置向量模型" }),
|
||||
);
|
||||
await userEvent.click(screen.getByRole("button", { name: "保存修改" }));
|
||||
expect(update).toHaveBeenCalledWith(profile.profile_id, {
|
||||
expected_revision: 1,
|
||||
embedding_model: "vector-new",
|
||||
embedding_base_url: profile.base_url,
|
||||
});
|
||||
});
|
||||
|
||||
it("explicitly switches an edited remote embedding to builtin without retaining its dimensions", async () => {
|
||||
profiles = [
|
||||
{
|
||||
|
||||
@@ -631,10 +631,9 @@ export function ModelSettings({
|
||||
patch.embedding_model = embeddingModel;
|
||||
if (embeddingProvider !== editing.embedding_provider_kind)
|
||||
patch.embedding_provider_kind = embeddingProvider;
|
||||
if (
|
||||
embeddingBaseUrl !==
|
||||
(editing.embedding_base_url ?? editing.base_url ?? "")
|
||||
)
|
||||
// Against the stored value only: the field is prefilled with the
|
||||
// generation endpoint, and Core no longer falls back to it.
|
||||
if (embeddingBaseUrl !== (editing.embedding_base_url ?? ""))
|
||||
patch.embedding_base_url = embeddingBaseUrl;
|
||||
} else if (
|
||||
embeddingModeChanged &&
|
||||
|
||||
@@ -173,10 +173,15 @@ export function OnboardingModelController({
|
||||
draft.embedding_mode === "local"
|
||||
? "bge-small-zh-v1.5"
|
||||
: draft.embedding_model.trim(),
|
||||
// Core no longer lets the embedding endpoint follow the generation
|
||||
// one, so an empty field is written as the endpoint it means.
|
||||
embedding_base_url:
|
||||
draft.embedding_mode === "local"
|
||||
? null
|
||||
: draft.embedding_base_url.trim() || null,
|
||||
: draft.embedding_base_url.trim() ||
|
||||
(draft.embedding_provider_kind === "ollama"
|
||||
? "http://127.0.0.1:11434"
|
||||
: draft.base_url.trim()),
|
||||
embedding_dimensions: draft.embedding_mode === "local" ? 512 : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -142,6 +142,16 @@ node apps/cli/dist/one-memory.cjs write status <request_id> --user-id wife
|
||||
- 不带 `--user-id`(或写 `--user-id local`)时请求里不出现 `user_id`,和以前完全一样。
|
||||
- 用户之间互相隔离:读、改、删另一个用户的记忆 ID 返回「未找到」;write 的 request id 也只属于提交它的用户。
|
||||
|
||||
### 画像版本(`profile --versions`)
|
||||
|
||||
```bash
|
||||
node apps/cli/dist/one-memory.cjs profile --versions # L0 画像版本链,最新在前,* 为当前版本
|
||||
node apps/cli/dist/one-memory.cjs profile --versions --user-id wife --json
|
||||
```
|
||||
|
||||
用于找出覆盖了正确值的错误版本(例如 `用户` / `The user`),再用
|
||||
`memories delete <memory-id> --yes` 删除;对应 `GET /api/v2/user-profile/versions`。
|
||||
|
||||
## 8. 召回(recall)
|
||||
|
||||
`recall` 两条路径(raw / memory)此前已真机验证。文本输出会标明每条是「原始对话摘录」
|
||||
|
||||
@@ -77,25 +77,33 @@ then:
|
||||
2. **prepare**: applies the management schema migrations, discards SDK staging
|
||||
left by builds through 19 (§5) and makes legacy queue work self-contained
|
||||
(§4);
|
||||
3. **vectors**: for the active embedding profile, renames every
|
||||
3. **vectors**: for the active embedding profile (the built-in model when no
|
||||
profile is active), renames every
|
||||
`one_memory_<old hash>…` collection to the current `memhub_<hash>…` name
|
||||
through Chroma's public client. A current collection that already holds
|
||||
writes is merged in first (newer record wins on a shared id); counts are
|
||||
checked after each step;
|
||||
4. **verify**: opens the public SDK client exactly as the Runtime will and reads
|
||||
the legacy confirmed memory ids back (`list_memories`, then `get`). Every
|
||||
chat memory must read back. Coding/Working may be fewer, because the SDK's
|
||||
Coding reconciler deletes experiences it retires, but not zero when the
|
||||
legacy mirror has some. Otherwise the run stops with
|
||||
`sdk_migration_memory_unreadable` and the counts, before anything
|
||||
irreversible; legacy authority and tables stay intact;
|
||||
confirmed chat memory that was in the vector store before this run (any
|
||||
collection, read before step 3) must read back. A confirmed chat the SDK had
|
||||
already merged or deleted before the upgrade cannot, and is only counted
|
||||
(`chat_missing_before_migration`); if none of them is in the store, none is
|
||||
excused. Coding/Working may be fewer, because the SDK's Coding reconciler
|
||||
deletes experiences it retires, but not zero when the legacy mirror has
|
||||
some. Otherwise the run stops with `sdk_migration_memory_unreadable` and
|
||||
the counts, before anything irreversible; legacy authority and tables stay
|
||||
intact;
|
||||
5. **cutover**: re-verifies the bundle, repeats §4, commits SDK authority;
|
||||
6. **cleanup**: drops the legacy tables (including the `evidence_search`
|
||||
index) and runs `VACUUM` (skipped if the disk is too full).
|
||||
|
||||
A home with legacy memories but no active model profile is not migrated
|
||||
(`sdk_migration_model_profile_required`): its collections can be neither named
|
||||
nor read back.
|
||||
A home with no active model profile has no generation model, but its vectors
|
||||
are the built-in embedding model's, the same space the Runtime reads with; it
|
||||
migrates through that space and no key is resolved. Only when its legacy
|
||||
memories have vectors solely under another embedding space (a profile since
|
||||
deleted) is it not migrated (`sdk_migration_vector_profile_missing`, in the
|
||||
preflight, before anything changes).
|
||||
|
||||
A home already on SDK authority still reports `required` while legacy-named
|
||||
vector collections exist for its profile (build 20 homes). `sdk-migrate` then
|
||||
|
||||
@@ -636,6 +636,16 @@ export interface SdkRawRecord {
|
||||
memory_at: number | null;
|
||||
gmt_created: number | null;
|
||||
custom: Record<string, unknown>;
|
||||
/** active, or shadow once pro extracted it. */
|
||||
status?: string | null;
|
||||
/** The turn's messages, parsed from the SDK's `[role]: text` lines. */
|
||||
messages?: { role: string; content: string }[];
|
||||
/** The one role of a single-message turn; null for a mixed turn. */
|
||||
role?: string | null;
|
||||
/** `sha256:<hex>` of the stored content, computed by Core. */
|
||||
content_hash?: string;
|
||||
/** ISO time the SDK stored the raw. */
|
||||
created?: string | null;
|
||||
}
|
||||
|
||||
export interface SdkRawPage {
|
||||
@@ -643,6 +653,37 @@ export interface SdkRawPage {
|
||||
total: number;
|
||||
limit: number;
|
||||
offset: number;
|
||||
/** A time range was applied to this page in Core (the SDK list has none). */
|
||||
time_filtered?: boolean;
|
||||
}
|
||||
|
||||
/** `GET /api/v2/memory-library/layers`: memories per SDK layer. */
|
||||
export interface SdkLayerCounts {
|
||||
layers: Record<string, number>;
|
||||
total: number;
|
||||
statuses: string[] | null;
|
||||
}
|
||||
|
||||
/** `GET /api/v2/user-profile/versions`: the L0 basic-info version chains. */
|
||||
export interface SdkProfileVersions {
|
||||
user_id: string;
|
||||
request_id: string;
|
||||
elapsed_ms: number;
|
||||
chains: {
|
||||
memory_id: string;
|
||||
content: string;
|
||||
status: string;
|
||||
memory_at: number | null;
|
||||
gmt_created: number | null;
|
||||
basic_info: Record<string, unknown>;
|
||||
versions: {
|
||||
memory_id: string;
|
||||
content: string;
|
||||
memory_at: number | string | null;
|
||||
gmt_created: number | null;
|
||||
current: boolean;
|
||||
}[];
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface FileImportReceipt {
|
||||
|
||||
@@ -1,642 +0,0 @@
|
||||
"""Re-embed into a new embedding space: vector stores, scan, diff and merge.
|
||||
|
||||
The switch model: writes are paused, every vector is computed again with the
|
||||
new model into a new space, the source is never touched. The graph half lives
|
||||
in ``test_reembed_graph.py``.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hy_memory import (
|
||||
EmbeddingDimensionMismatchError,
|
||||
MemoryConfig,
|
||||
ReembedCancelled,
|
||||
VectorSpace,
|
||||
diff_spaces,
|
||||
drop_vector_space,
|
||||
plan_reembed,
|
||||
reembed_text,
|
||||
reembed_vector_space,
|
||||
scan_vector_spaces,
|
||||
space_identity,
|
||||
vector_space_names,
|
||||
verify_reembed,
|
||||
)
|
||||
from hy_memory.config import EmbedderConfig
|
||||
from hy_memory.core.embed_service import EmbedService
|
||||
from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus
|
||||
from hy_memory.reembed import VECTOR_ROLES, parse_collection_name
|
||||
|
||||
|
||||
class FakeEmbedder:
|
||||
"""Deterministic vectors: same (model, text) -> same unit vector."""
|
||||
|
||||
def __init__(self, model: str, dims: int, *, wrong_width_after: int = -1):
|
||||
self.model = model
|
||||
self.dims = dims
|
||||
self.calls = []
|
||||
self.texts = 0
|
||||
self.wrong_width_after = wrong_width_after
|
||||
|
||||
def vector(self, text: str):
|
||||
seed = int(hashlib.sha256(f"{self.model}\0{text}".encode()).hexdigest()[:16], 16)
|
||||
rng = random.Random(seed)
|
||||
values = [rng.gauss(0.0, 1.0) for _ in range(self.dims)]
|
||||
norm = math.sqrt(sum(v * v for v in values)) or 1.0
|
||||
return [v / norm for v in values]
|
||||
|
||||
async def embed_batch(self, texts, use_cache=True):
|
||||
self.calls.append(list(texts))
|
||||
out = []
|
||||
for text in texts:
|
||||
self.texts += 1
|
||||
vector = self.vector(text)
|
||||
if 0 <= self.wrong_width_after < self.texts:
|
||||
vector = vector + [0.0]
|
||||
out.append(vector)
|
||||
return out
|
||||
|
||||
async def embed(self, text, use_cache=True):
|
||||
return self.vector(text)
|
||||
|
||||
async def embed_queued(self, text, use_cache=True):
|
||||
return self.vector(text)
|
||||
|
||||
def get_embedding_dims(self):
|
||||
return self.dims
|
||||
|
||||
|
||||
def _client(path):
|
||||
import chromadb
|
||||
return chromadb.PersistentClient(path=str(path))
|
||||
|
||||
|
||||
def _space(path, base, dims, model=None):
|
||||
return VectorSpace(base_name=base, dims=dims, persist_directory=str(path), model=model)
|
||||
|
||||
|
||||
def _custom_state():
|
||||
return {
|
||||
"memory_kind": "canonical_state",
|
||||
"state_key": "home_city",
|
||||
"state_value": "Berlin",
|
||||
"state_retrieval_text": "user lives in Berlin (home city)",
|
||||
}
|
||||
|
||||
|
||||
async def _seed(path, base, dims, *, model="old-model", users=("alice", "bob"), per_user=7):
|
||||
"""Write a realistic source space through the SDK's own stores."""
|
||||
from hy_memory.data.vector_store_chroma import ChromaVectorStore
|
||||
|
||||
old = FakeEmbedder(model, dims)
|
||||
cfg = MemoryConfig()
|
||||
cfg.vector_store.provider = "chroma"
|
||||
cfg.vector_store.persist_directory = str(path)
|
||||
cfg.vector_store.collection_name = base
|
||||
cfg.vector_store.embedding_dims = dims
|
||||
store = ChromaVectorStore(cfg)
|
||||
await store.initialize()
|
||||
nodes = []
|
||||
for user in users:
|
||||
for i in range(per_user):
|
||||
layer = [MemoryLayer.L1_RAW, MemoryLayer.L2_FACT, MemoryLayer.L3_SUMMARY][i % 3]
|
||||
node = MemoryNode(
|
||||
user_id=user, agent_id="a", session_id=f"s{i % 2}",
|
||||
layer=layer, content=f"{user} fact {i}",
|
||||
status=MemoryStatus.SUPERSEDED if i == 4 else MemoryStatus.ACTIVE,
|
||||
tags=["t1", f"t{i}"],
|
||||
custom={"k": i, "nested": {"x": [1, 2]}},
|
||||
)
|
||||
if i == 4:
|
||||
node.is_latest = False
|
||||
node.superseded_by = ["later"]
|
||||
if i == 5:
|
||||
node.custom = _custom_state()
|
||||
node.embedding = old.vector(node.content)
|
||||
nodes.append(node)
|
||||
await store.upsert_batch(nodes)
|
||||
entities = await store._get_entity_collection()
|
||||
entities.add(
|
||||
ids=["e1", "e2"],
|
||||
embeddings=[old.vector("Berlin"), old.vector("Dylan")],
|
||||
documents=["Berlin", "Dylan"],
|
||||
metadatas=[{"user_id": "alice", "label": "GPE"}, {"user_id": "bob", "label": "PERSON"}],
|
||||
)
|
||||
await store.close()
|
||||
client = _client(path)
|
||||
names = vector_space_names(_space(path, base, dims))
|
||||
for role, doc in (("coding_keys", "fix the flaky test"), ("working_facts_cv", "repo: uses uv")):
|
||||
coll = client.get_or_create_collection(names[role], metadata={"hnsw:space": "cosine"})
|
||||
coll.add(
|
||||
ids=[f"{role}:0"],
|
||||
embeddings=[old.vector(doc)],
|
||||
documents=[doc],
|
||||
metadatas=[{"user_id": "alice", "layer": "l1_raw",
|
||||
"custom": json.dumps({"coding_key_text": doc})}],
|
||||
)
|
||||
return names
|
||||
|
||||
|
||||
def _snapshot(path, names):
|
||||
client = _client(path)
|
||||
out = {}
|
||||
for role, name in names.items():
|
||||
try:
|
||||
coll = client.get_collection(name)
|
||||
except Exception:
|
||||
continue
|
||||
got = coll.get(include=["documents", "metadatas", "embeddings"])
|
||||
out[role] = sorted(
|
||||
(rid, doc, json.dumps(meta, sort_keys=True), [round(float(x), 6) for x in emb])
|
||||
for rid, doc, meta, emb in zip(got["ids"], got["documents"], got["metadatas"], got["embeddings"])
|
||||
)
|
||||
return hashlib.sha256(json.dumps(out, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def _records(path, name):
|
||||
got = _client(path).get_collection(name).get(include=["documents", "metadatas", "embeddings"])
|
||||
return {
|
||||
rid: (doc, meta, list(emb))
|
||||
for rid, doc, meta, emb in zip(got["ids"], got["documents"], got["metadatas"], got["embeddings"])
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# naming
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
class TestVectorSpaceNames:
|
||||
@pytest.mark.parametrize("dims", [512, None])
|
||||
async def test_names_equal_what_the_stores_open(self, tmp_path, dims):
|
||||
from hy_memory.coding.store import CodingMemoryStore
|
||||
from hy_memory.data.vector_store_chroma import ChromaVectorStore
|
||||
|
||||
cfg = MemoryConfig()
|
||||
cfg.vector_store.provider = "chroma"
|
||||
cfg.vector_store.persist_directory = str(tmp_path / "v")
|
||||
cfg.vector_store.collection_name = "memhub_abc123"
|
||||
cfg.vector_store.embedding_dims = dims
|
||||
cfg.history.db_path = str(tmp_path / "history.sqlite3")
|
||||
store = ChromaVectorStore(cfg)
|
||||
await store.initialize()
|
||||
await store._get_entity_collection()
|
||||
coding = []
|
||||
for variant in ("working", "coding"):
|
||||
cs = CodingMemoryStore(cfg, FakeEmbedder("m", dims or 8), variant=variant)
|
||||
await cs.initialize()
|
||||
coding.append(cs)
|
||||
opened = {c.name for c in _client(tmp_path / "v").list_collections()}
|
||||
names = vector_space_names(cfg)
|
||||
assert set(names) == set(VECTOR_ROLES)
|
||||
assert set(names.values()) == opened
|
||||
suffix = f"_{dims}" if dims else ""
|
||||
assert names["memory"] == f"memhub_abc123{suffix}"
|
||||
assert names["coding_keys_cv"] == f"memhub_abc123_coding_keys_cv{suffix}"
|
||||
await store.close()
|
||||
for cs in coding:
|
||||
await cs.close()
|
||||
|
||||
def test_explicit_base_name_and_space(self, tmp_path):
|
||||
cfg = MemoryConfig()
|
||||
cfg.vector_store.provider = "chroma"
|
||||
cfg.vector_store.embedding_dims = 1024
|
||||
names = vector_space_names(cfg, base_name="staging_x")
|
||||
assert names["memory"] == "staging_x_1024"
|
||||
assert names["entities"] == "staging_x_1024_entities"
|
||||
assert names["working_facts"] == "staging_x_working_facts_1024"
|
||||
assert vector_space_names(_space(tmp_path, "b", 512))["memory"] == "b_512"
|
||||
|
||||
def test_parse_collection_name_round_trips(self):
|
||||
for dims in (512, None):
|
||||
for role, name in vector_space_names(VectorSpace("memhub_9f", dims)).items():
|
||||
assert parse_collection_name(name) == ("memhub_9f", dims, role)
|
||||
assert parse_collection_name("memhub_9f_512_tag_index") is None
|
||||
|
||||
def test_space_identity_is_model_and_width_only(self):
|
||||
a = space_identity("BAAI/bge-small-zh-v1.5", 512)
|
||||
b = space_identity(" BAAI/bge-small-zh-v1.5 ", 512)
|
||||
assert a == b and a.key == "BAAI/bge-small-zh-v1.5#512"
|
||||
assert a.digest != space_identity("BAAI/bge-small-zh-v1.5", 1024).digest
|
||||
with pytest.raises(ValueError):
|
||||
space_identity("", 512)
|
||||
|
||||
def test_non_chroma_provider_is_not_supported(self):
|
||||
cfg = MemoryConfig()
|
||||
cfg.vector_store.provider = "qdrant"
|
||||
with pytest.raises(NotImplementedError):
|
||||
vector_space_names(cfg)
|
||||
|
||||
|
||||
class TestReembedText:
|
||||
def test_canonical_state_uses_state_retrieval_text(self):
|
||||
meta = {"content": "住在柏林", "custom": json.dumps(_custom_state())}
|
||||
assert reembed_text("memory", "住在柏林", meta) == "user lives in Berlin (home city)"
|
||||
|
||||
def test_retrieval_text_when_present(self):
|
||||
meta = {"custom": json.dumps({"retrieval_text": " pattern: plans ahead "})}
|
||||
assert reembed_text("memory", "画像", meta) == "pattern: plans ahead"
|
||||
|
||||
def test_document_otherwise_with_content_fallback(self):
|
||||
assert reembed_text("memory", "plain fact", {"custom": "{}"}) == "plain fact"
|
||||
assert reembed_text("memory", None, {"content": "from metadata"}) == "from metadata"
|
||||
assert reembed_text("memory", "doc", {"custom": "not json"}) == "doc"
|
||||
|
||||
def test_coding_key_and_working_fact_use_the_document(self):
|
||||
coding = {"custom": json.dumps({"coding_key_text": "fix flaky test",
|
||||
"retrieval_text": "ignored outside memory"})}
|
||||
assert reembed_text("coding_keys", "fix flaky test", coding) == "fix flaky test"
|
||||
assert reembed_text("working_facts", "repo: uses uv", {}) == "repo: uses uv"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# re-embed
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src_dims,tgt_dims,src_model,tgt_model",
|
||||
[(512, 512, "old-model", "new-model"), (512, 1024, "old", "new"), (1024, 512, "old", "new")],
|
||||
)
|
||||
async def test_reembed_copies_everything_and_never_touches_source(
|
||||
tmp_path, src_dims, tgt_dims, src_model, tgt_model,
|
||||
):
|
||||
path = tmp_path / "v"
|
||||
names = await _seed(path, "memhub_src", src_dims, model=src_model)
|
||||
before = _snapshot(path, names)
|
||||
source = _space(path, "memhub_src", src_dims)
|
||||
target = _space(path, "memhub_dst", tgt_dims, model=tgt_model)
|
||||
new = FakeEmbedder(tgt_model, tgt_dims)
|
||||
|
||||
plan = await plan_reembed(source, target)
|
||||
assert plan.per_role["memory"] == 14 and plan.per_role["entities"] == 2
|
||||
assert plan.total_vectors == 18 == plan.estimated_vectors
|
||||
assert plan.source_dims == src_dims and plan.target_dims == tgt_dims
|
||||
|
||||
seen = []
|
||||
report = await reembed_vector_space(
|
||||
source, target, embedding=new, batch_size=4, on_progress=seen.append,
|
||||
)
|
||||
assert report.embedded == 18 and report.reused == 0
|
||||
assert seen and seen[-1].done == seen[-1].total
|
||||
|
||||
assert _snapshot(path, names) == before # source unchanged, byte for byte
|
||||
target_names = vector_space_names(target)
|
||||
for role in ("memory", "entities", "coding_keys", "working_facts_cv"):
|
||||
src = _records(path, names[role])
|
||||
dst = _records(path, target_names[role])
|
||||
assert set(src) == set(dst)
|
||||
for rid, (doc, meta, _) in src.items():
|
||||
assert dst[rid][0] == doc
|
||||
assert dst[rid][1] == meta # status, is_latest, superseded_by, timestamps, custom
|
||||
assert len(dst[rid][2]) == tgt_dims
|
||||
assert dst[rid][2] == pytest.approx(new.vector(reembed_text(role, doc, meta)), abs=1e-5)
|
||||
memory = _records(path, target_names["memory"])
|
||||
layers = {meta["layer"] for _, meta, _ in memory.values()}
|
||||
users = {meta["user_id"] for _, meta, _ in memory.values()}
|
||||
assert "l1_raw" in layers and users == {"alice", "bob"}
|
||||
statuses = {meta["status"] for _, meta, _ in memory.values()}
|
||||
assert statuses == {"active", "superseded"}
|
||||
state = [r for r in memory.values() if "canonical_state" in r[1]["custom"]]
|
||||
assert state and state[0][2] == pytest.approx(
|
||||
new.vector("user lives in Berlin (home city)"), abs=1e-5,
|
||||
)
|
||||
meta = _client(path).get_collection(target_names["memory"]).metadata
|
||||
assert meta["hnsw:space"] == "cosine" and meta["hy_embedding_model"] == tgt_model
|
||||
|
||||
verified = await verify_reembed(source, target, embedding=new)
|
||||
assert verified.ok, verified.problems
|
||||
assert verified.per_user == {"alice": 10, "bob": 8}
|
||||
assert verified.self_recall_samples == 18 and verified.self_recall_hits == 18
|
||||
|
||||
|
||||
async def test_resume_after_kill_only_embeds_the_rest(tmp_path):
|
||||
path = tmp_path / "v"
|
||||
await _seed(path, "src", 512, per_user=20)
|
||||
source, target = _space(path, "src", 512), _space(path, "dst", 1024)
|
||||
|
||||
class Killed(Exception):
|
||||
pass
|
||||
|
||||
first = FakeEmbedder("new", 1024)
|
||||
batches = {"n": 0}
|
||||
|
||||
def _kill():
|
||||
batches["n"] += 1
|
||||
if batches["n"] > 3:
|
||||
raise Killed()
|
||||
return False
|
||||
|
||||
with pytest.raises(Killed):
|
||||
await reembed_vector_space(
|
||||
source, target, embedding=first, batch_size=5, max_inflight=1,
|
||||
roles=["memory"], cancel=_kill,
|
||||
)
|
||||
assert first.texts == 15
|
||||
second = FakeEmbedder("new", 1024)
|
||||
report = await reembed_vector_space(
|
||||
source, target, embedding=second, batch_size=5, max_inflight=1, roles=["memory"],
|
||||
)
|
||||
assert second.texts == 40 - 15
|
||||
assert report.per_role["memory"].reused == 15
|
||||
# The end state equals an uninterrupted run: every source record, verbatim,
|
||||
# with the vector of its own text.
|
||||
resumed = _records(path, "dst_1024")
|
||||
src = _records(path, "src_512")
|
||||
assert set(resumed) == set(src)
|
||||
for rid, (doc, meta, vector) in resumed.items():
|
||||
assert (doc, meta) == src[rid][:2]
|
||||
assert vector == pytest.approx(second.vector(reembed_text("memory", doc, meta)), abs=1e-5)
|
||||
assert (await verify_reembed(source, target, roles=["memory"], samples=0)).ok
|
||||
|
||||
|
||||
async def test_cancel_at_batch_boundary_leaves_no_partial_batch(tmp_path):
|
||||
path = tmp_path / "v"
|
||||
await _seed(path, "src", 512)
|
||||
calls = {"n": 0}
|
||||
|
||||
def _cancel():
|
||||
calls["n"] += 1
|
||||
return calls["n"] > 2
|
||||
|
||||
with pytest.raises(ReembedCancelled) as info:
|
||||
await reembed_vector_space(
|
||||
_space(path, "src", 512), _space(path, "dst", 512),
|
||||
embedding=FakeEmbedder("new", 512), batch_size=3, max_inflight=1,
|
||||
roles=["memory"], cancel=_cancel,
|
||||
)
|
||||
assert info.value.report.per_role["memory"].embedded == 6
|
||||
assert _client(path).get_collection("dst_512").count() == 6
|
||||
|
||||
|
||||
async def test_wrong_width_raises_before_any_upsert(tmp_path):
|
||||
path = tmp_path / "v"
|
||||
await _seed(path, "src", 512)
|
||||
bad = FakeEmbedder("new", 1024, wrong_width_after=2)
|
||||
with pytest.raises(EmbeddingDimensionMismatchError):
|
||||
await reembed_vector_space(
|
||||
_space(path, "src", 512), _space(path, "dst", 1024),
|
||||
embedding=bad, batch_size=4, roles=["memory"],
|
||||
)
|
||||
assert _client(path).get_collection("dst_1024").count() == 0
|
||||
declared_wrong = FakeEmbedder("new", 768)
|
||||
with pytest.raises(EmbeddingDimensionMismatchError):
|
||||
await reembed_vector_space(
|
||||
_space(path, "src", 512), _space(path, "dst", 1024), embedding=declared_wrong,
|
||||
)
|
||||
|
||||
|
||||
async def test_refuses_to_write_into_the_source_collection(tmp_path):
|
||||
path = tmp_path / "v"
|
||||
await _seed(path, "src", 512)
|
||||
with pytest.raises(ValueError):
|
||||
await reembed_vector_space(
|
||||
_space(path, "src", 512), _space(path, "src", 512),
|
||||
embedding=FakeEmbedder("new", 512),
|
||||
)
|
||||
|
||||
|
||||
async def test_verify_detects_missing_extra_drift_and_width(tmp_path):
|
||||
path = tmp_path / "v"
|
||||
await _seed(path, "src", 512)
|
||||
source, target = _space(path, "src", 512), _space(path, "dst", 512)
|
||||
new = FakeEmbedder("new", 512)
|
||||
await reembed_vector_space(source, target, embedding=new, roles=["memory"])
|
||||
coll = _client(path).get_collection("dst_512")
|
||||
got = coll.get(include=["documents", "metadatas"])
|
||||
victim, drifted = got["ids"][0], got["ids"][1]
|
||||
coll.delete(ids=[victim])
|
||||
meta = dict(got["metadatas"][1])
|
||||
meta["status"] = "archived"
|
||||
coll.update(ids=[drifted], metadatas=[meta])
|
||||
coll.add(ids=["stranger"], embeddings=[new.vector("x")], documents=["x"], metadatas=[{"user_id": "eve"}])
|
||||
result = await verify_reembed(source, target, embedding=new, roles=["memory"], samples=5)
|
||||
assert not result.ok
|
||||
counts = result.per_role["memory"]
|
||||
assert (counts["missing"], counts["extra"], counts["mismatched"]) == (1, 1, 1)
|
||||
assert any("eve" in p for p in result.problems)
|
||||
|
||||
wide = _space(path, "dst", 1024)
|
||||
wide_coll = _client(path).get_or_create_collection("dst_1024", metadata={"hnsw:space": "cosine"})
|
||||
src = _records(path, "src_512")
|
||||
wide_coll.add(
|
||||
ids=list(src),
|
||||
embeddings=[[0.0] * 512 for _ in src],
|
||||
documents=[v[0] for v in src.values()],
|
||||
metadatas=[v[1] for v in src.values()],
|
||||
)
|
||||
result = await verify_reembed(source, wide, roles=["memory"], samples=0)
|
||||
assert result.per_role["memory"]["bad_dims"] == len(src) and not result.ok
|
||||
|
||||
|
||||
async def test_drop_vector_space_refuses_live_collections(tmp_path):
|
||||
from hy_memory.data.vector_store_chroma import ChromaVectorStore
|
||||
|
||||
path = tmp_path / "v"
|
||||
await _seed(path, "src", 512)
|
||||
cfg = MemoryConfig()
|
||||
cfg.vector_store.provider = "chroma"
|
||||
cfg.vector_store.persist_directory = str(path)
|
||||
cfg.vector_store.collection_name = "src"
|
||||
cfg.vector_store.embedding_dims = 512
|
||||
live = ChromaVectorStore(cfg)
|
||||
await live.initialize()
|
||||
with pytest.raises(RuntimeError):
|
||||
await drop_vector_space(_space(path, "src", 512))
|
||||
await live.close()
|
||||
dropped = await drop_vector_space(_space(path, "src", 512))
|
||||
assert set(dropped) == {"src_512", "src_512_entities", "src_coding_keys_512", "src_working_facts_cv_512"}
|
||||
assert not {c.name for c in _client(path).list_collections()}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# scan, diff, merge
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
async def _two_spaces(path):
|
||||
"""A live 1024 space and a stray 512 space sharing some ids."""
|
||||
from hy_memory.data.vector_store_chroma import ChromaVectorStore
|
||||
|
||||
async def _store(base, dims, model, contents):
|
||||
cfg = MemoryConfig()
|
||||
cfg.vector_store.provider = "chroma"
|
||||
cfg.vector_store.persist_directory = str(path)
|
||||
cfg.vector_store.collection_name = base
|
||||
cfg.vector_store.embedding_dims = dims
|
||||
store = ChromaVectorStore(cfg)
|
||||
await store.initialize()
|
||||
emb = FakeEmbedder(model, dims)
|
||||
nodes = []
|
||||
for node_id, user, content, status in contents:
|
||||
node = MemoryNode(
|
||||
node_id=node_id, user_id=user, agent_id="a", session_id="s",
|
||||
layer=MemoryLayer.L2_FACT, content=content, status=status,
|
||||
)
|
||||
node.embedding = emb.vector(content)
|
||||
nodes.append(node)
|
||||
await store.upsert_batch(nodes)
|
||||
await store.close()
|
||||
|
||||
ids = [f"00000000-0000-0000-0000-00000000000{i}" for i in range(6)]
|
||||
await _store("memhub_old", 512, "bge", [
|
||||
(ids[0], "alice", "shared one", MemoryStatus.ACTIVE),
|
||||
(ids[1], "alice", "shared two (stray wording)", MemoryStatus.ACTIVE),
|
||||
(ids[2], "bob", "only in stray", MemoryStatus.ACTIVE),
|
||||
(ids[3], "bob", "only in stray, deleted", MemoryStatus.SHADOW),
|
||||
])
|
||||
await _store("memhub_new", 1024, "e5", [
|
||||
(ids[0], "alice", "shared one", MemoryStatus.ACTIVE),
|
||||
(ids[1], "alice", "shared two (live wording)", MemoryStatus.ARCHIVED),
|
||||
(ids[4], "carol", "only in live", MemoryStatus.ACTIVE),
|
||||
])
|
||||
return ids
|
||||
|
||||
|
||||
async def test_scan_diff_and_merge(tmp_path):
|
||||
path = tmp_path / "v"
|
||||
ids = await _two_spaces(path)
|
||||
spaces = await scan_vector_spaces(path)
|
||||
by_base = {(s.base_name, s.dims): s for s in spaces}
|
||||
assert set(by_base) == {("memhub_old", 512), ("memhub_new", 1024)}
|
||||
old, live = by_base[("memhub_old", 512)], by_base[("memhub_new", 1024)]
|
||||
assert old.roles["memory"].count == 4 and old.width == 512
|
||||
assert live.roles["memory"].count == 3 and live.width == 1024
|
||||
assert old.per_user == {"alice": 2, "bob": 2}
|
||||
assert live.per_user == {"alice": 2, "carol": 1}
|
||||
|
||||
stray, target = old.space(), _space(path, "memhub_new", 1024, model="e5")
|
||||
diff = await diff_spaces(stray, target)
|
||||
assert set(diff.missing["memory"]) == {ids[2], ids[3]} # status ignored
|
||||
assert diff.total_missing == 2
|
||||
|
||||
live_before = _records(path, "memhub_new_1024")
|
||||
stray_before = _snapshot(path, stray.names())
|
||||
merger = FakeEmbedder("e5", 1024)
|
||||
report = await reembed_vector_space(stray, target, embedding=merger)
|
||||
assert merger.texts == 2 and report.embedded == 2
|
||||
assert report.reused == 1 and report.kept == 1
|
||||
after = _records(path, "memhub_new_1024")
|
||||
assert set(after) == set(live_before) | {ids[2], ids[3]}
|
||||
for rid, record in live_before.items(): # the target wins
|
||||
assert after[rid][:2] == record[:2]
|
||||
assert after[rid][2] == pytest.approx(record[2], abs=1e-6)
|
||||
assert _snapshot(path, stray.names()) == stray_before
|
||||
assert (await diff_spaces(stray, target)).empty
|
||||
|
||||
again = FakeEmbedder("e5", 1024)
|
||||
second = await reembed_vector_space(stray, target, embedding=again)
|
||||
assert again.texts == 0 and second.embedded == 0
|
||||
assert _records(path, "memhub_new_1024").keys() == after.keys()
|
||||
rescanned = {(s.base_name, s.dims): s for s in await scan_vector_spaces(path)}
|
||||
assert rescanned[("memhub_new", 1024)].roles["memory"].count == 5
|
||||
|
||||
|
||||
async def test_scan_of_missing_directory_is_empty(tmp_path):
|
||||
assert await scan_vector_spaces(tmp_path / "nope") == []
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# EmbedService fixes
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def _fake_raw(n, dims):
|
||||
items = []
|
||||
for i in range(n):
|
||||
item = MagicMock()
|
||||
item.embedding = [0.1] * dims
|
||||
item.index = i
|
||||
items.append(item)
|
||||
resp = MagicMock()
|
||||
resp.model = "m"
|
||||
resp.usage = None
|
||||
resp.data = items
|
||||
raw = MagicMock()
|
||||
raw.status_code = 200
|
||||
raw.headers = {}
|
||||
raw.parse.return_value = resp
|
||||
return raw
|
||||
|
||||
|
||||
def _service(dims=8, **embedder):
|
||||
cfg = MemoryConfig(embedder=EmbedderConfig(
|
||||
provider="openai", model="m", api_key="sk", base_url="https://e.invalid/v1",
|
||||
embedding_dims=dims, max_retries=2, retry_delay=0.0, **embedder,
|
||||
))
|
||||
return EmbedService(cfg)
|
||||
|
||||
|
||||
async def test_embed_batch_chunks_to_batch_size():
|
||||
svc = _service()
|
||||
sizes = []
|
||||
|
||||
async def _create(**kwargs):
|
||||
sizes.append(len(kwargs["input"]))
|
||||
return _fake_raw(len(kwargs["input"]), 8)
|
||||
|
||||
client = MagicMock()
|
||||
client.embeddings.with_raw_response.create = AsyncMock(side_effect=_create)
|
||||
with patch("openai.AsyncOpenAI", return_value=client):
|
||||
vectors = await svc.embed_batch([f"t{i}" for i in range(1000)], use_cache=False)
|
||||
assert len(vectors) == 1000
|
||||
assert len(sizes) == math.ceil(1000 / 64) and max(sizes) == 64
|
||||
|
||||
small = _service(batch_size=10)
|
||||
sizes.clear()
|
||||
with patch("openai.AsyncOpenAI", return_value=client):
|
||||
_, metas = await small.embed_batch_with_meta([f"u{i}" for i in range(25)], use_cache=False)
|
||||
assert sizes == [10, 10, 5]
|
||||
assert [m["batch_index"] for m in metas] == list(range(25))
|
||||
|
||||
|
||||
async def test_429_honours_retry_after():
|
||||
svc = _service()
|
||||
|
||||
class RateLimited(Exception):
|
||||
status_code = 429
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("rate limited")
|
||||
self.response = MagicMock()
|
||||
self.response.headers = {"retry-after": "2"}
|
||||
|
||||
client = MagicMock()
|
||||
client.embeddings.with_raw_response.create = AsyncMock(
|
||||
side_effect=[RateLimited(), _fake_raw(1, 8)],
|
||||
)
|
||||
sleeps = []
|
||||
|
||||
async def _sleep(seconds):
|
||||
sleeps.append(seconds)
|
||||
|
||||
with patch("openai.AsyncOpenAI", return_value=client), \
|
||||
patch("hy_memory.core.embed_service.asyncio.sleep", _sleep):
|
||||
vectors = await svc.embed_batch(["x"], use_cache=False)
|
||||
assert len(vectors) == 1
|
||||
assert len(sleeps) == 1 and 2.0 <= sleeps[0] <= 2.5
|
||||
|
||||
|
||||
async def test_per_call_max_retries():
|
||||
svc = _service()
|
||||
|
||||
class Down(Exception):
|
||||
status_code = 503
|
||||
|
||||
client = MagicMock()
|
||||
client.embeddings.with_raw_response.create = AsyncMock(side_effect=Down("down"))
|
||||
with patch("openai.AsyncOpenAI", return_value=client), pytest.raises(Down):
|
||||
await svc.embed_batch(["x"], use_cache=False, max_retries=5)
|
||||
assert client.embeddings.with_raw_response.create.await_count == 6
|
||||
|
||||
|
||||
async def test_returned_width_is_validated_and_not_retried():
|
||||
svc = _service(dims=8)
|
||||
client = MagicMock()
|
||||
client.embeddings.with_raw_response.create = AsyncMock(return_value=_fake_raw(2, 7))
|
||||
with patch("openai.AsyncOpenAI", return_value=client), \
|
||||
pytest.raises(EmbeddingDimensionMismatchError):
|
||||
await svc.embed_batch(["a", "b"], use_cache=False)
|
||||
assert client.embeddings.with_raw_response.create.await_count == 1
|
||||
@@ -1,252 +0,0 @@
|
||||
"""Re-embed the Kuzu graph into a new embedding space.
|
||||
|
||||
The source graph is opened read-only and never modified. A new graph is built
|
||||
at the target path with the target width: every node and edge is copied, the
|
||||
embedding columns are recomputed from their text, the HNSW index is rebuilt.
|
||||
Runs in CI: kuzu is a core dependency.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from hy_memory import (
|
||||
EmbeddingDimensionMismatchError,
|
||||
MemoryConfig,
|
||||
ReembedCancelled,
|
||||
drop_graph,
|
||||
reembed_graph,
|
||||
verify_graph_reembed,
|
||||
)
|
||||
from hy_memory.models.memory import MemoryLayer, MemoryNode
|
||||
from hy_memory.reembed import hierarchy_topic_label, plan_graph_reembed
|
||||
|
||||
from .test_reembed import FakeEmbedder
|
||||
|
||||
|
||||
def _cfg(path, dims):
|
||||
cfg = MemoryConfig()
|
||||
cfg.graph_store.provider = "kuzu"
|
||||
cfg.graph_store.db_path = str(path)
|
||||
cfg.vector_store.embedding_dims = dims
|
||||
return cfg
|
||||
|
||||
|
||||
async def _open(path, dims):
|
||||
from hy_memory.data.graph_store_kuzu import KuzuGraphStore
|
||||
|
||||
store = KuzuGraphStore(_cfg(path, dims))
|
||||
await store.initialize()
|
||||
assert store._available
|
||||
return store
|
||||
|
||||
|
||||
def _node(content, *, layer=MemoryLayer.L2_FACT, custom=None, user="alice"):
|
||||
return MemoryNode(
|
||||
user_id=user, agent_id="a", session_id="s1",
|
||||
layer=layer, content=content, custom=custom or {},
|
||||
)
|
||||
|
||||
|
||||
async def _seed_graph(path, dims, model="old"):
|
||||
"""A graph with every node and edge type the SDK writes."""
|
||||
old = FakeEmbedder(model, dims)
|
||||
gs = await _open(path, dims)
|
||||
schema = _node(
|
||||
"画像:喜欢提前规划", layer=MemoryLayer.L6_SCHEMA,
|
||||
custom={"schema_type": "basic", "retrieval_text": "plans trips weeks ahead",
|
||||
"_has_beh": True, "beh_abstraction": "prepares early under uncertainty"},
|
||||
)
|
||||
schema._graph_embedding = old.vector("plans trips weeks ahead")
|
||||
core = _node("core: anticipates", layer=MemoryLayer.L6_SCHEMA, custom={"schema_type": "abstract"})
|
||||
core._graph_embedding = old.vector("core: anticipates")
|
||||
stale_beh = _node("beh without text", layer=MemoryLayer.L6_SCHEMA, custom={})
|
||||
stale_beh._graph_embedding = old.vector("beh without text")
|
||||
facts = [_node(f"fact {i}", user="bob" if i % 2 else "alice") for i in range(5)]
|
||||
for node in [schema, core, stale_beh, *facts]:
|
||||
await gs.upsert_memory_node(node)
|
||||
await gs.update_embedding(schema.node_id, beh_embedding=old.vector("prepares early under uncertainty"))
|
||||
await gs.update_embedding(stale_beh.node_id, beh_embedding=old.vector("lost text"))
|
||||
await gs.add_cross_abstracts_to(schema.node_id, core.node_id)
|
||||
ik = facts[0].get_isolation_key()
|
||||
await gs.add_edge(facts[0].node_id, facts[1].node_id, "RELATED_TO", {"relation_type": "co_occurs", "weight": 0.7})
|
||||
await gs.add_temporal_edge(facts[0].node_id, facts[2].node_id, topic_id="t")
|
||||
for fact in facts[:3]:
|
||||
await gs.add_topic_tag(fact.node_id, "travel", ik, embed_service=old)
|
||||
await gs.add_topic_tag(facts[3].node_id, "work", ik) # no embed service: NULL embedding
|
||||
await gs.add_topic_path(
|
||||
facts[0].node_id, ["Travel", "Japan trip"], ik, "alice",
|
||||
node_types=["topic", "series"], embed_service=old,
|
||||
)
|
||||
await gs.add_topic_path(facts[1].node_id, ["Work", "Standups"], ik, "alice", embed_service=old)
|
||||
await gs.add_entity_mention(facts[0].node_id, "Kyoto", "GPE", ik)
|
||||
await gs.ensure_vdbref("raw-1", "l1_raw")
|
||||
await gs.add_derived_from(schema.node_id, "raw-1")
|
||||
await gs.close()
|
||||
return old, {"schema": schema, "core": core, "stale_beh": stale_beh, "facts": facts}
|
||||
|
||||
|
||||
def _digest(path):
|
||||
"""Hash of the Kuzu DB file and any sidecar files next to it."""
|
||||
h = hashlib.sha256()
|
||||
for name in sorted(os.listdir(path.parent)):
|
||||
if name == path.name or name.startswith(path.name + "."):
|
||||
h.update(name.encode())
|
||||
with open(path.parent / name, "rb") as handle:
|
||||
h.update(handle.read())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _read(path, query):
|
||||
import kuzu
|
||||
|
||||
db = kuzu.Database(str(path), read_only=True)
|
||||
conn = kuzu.Connection(db)
|
||||
try:
|
||||
result = conn.execute(query)
|
||||
rows = []
|
||||
while result.has_next():
|
||||
rows.append(result.get_next())
|
||||
return rows
|
||||
finally:
|
||||
conn.close()
|
||||
db.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src_dims,tgt_dims,src_model,tgt_model",
|
||||
[(512, 512, "old", "new"), (512, 1024, "old", "new"), (1024, 512, "old", "new")],
|
||||
)
|
||||
async def test_reembed_graph_copies_everything_at_the_new_width(
|
||||
tmp_path, src_dims, tgt_dims, src_model, tgt_model,
|
||||
):
|
||||
source, target = tmp_path / "graph", tmp_path / "graph-new"
|
||||
_, nodes = await _seed_graph(source, src_dims, model=src_model)
|
||||
before = _digest(source)
|
||||
plan = await plan_graph_reembed(source, target_dims=tgt_dims)
|
||||
assert plan.source_width == src_dims and plan.target_width == tgt_dims
|
||||
assert plan.memory_nodes == 8 and plan.topics == 2 and plan.hierarchy_topics == 4
|
||||
assert plan.memory_embeddings == 3 and plan.beh_embeddings == 2
|
||||
assert plan.topic_embeddings == 1 and plan.hierarchy_topic_embeddings == 4
|
||||
|
||||
new = FakeEmbedder(tgt_model, tgt_dims)
|
||||
seen = []
|
||||
report = await reembed_graph(
|
||||
source, target, embedding=new, target_dims=tgt_dims, batch_size=3, on_progress=seen.append,
|
||||
)
|
||||
assert _digest(source) == before # the source graph is untouched
|
||||
assert not os.path.exists(str(target) + ".reembed-partial")
|
||||
assert report.source_width == src_dims and report.target_width == tgt_dims
|
||||
assert report.node_counts == plan.node_counts
|
||||
assert report.rel_counts == plan.rel_counts
|
||||
assert report.embedded == {
|
||||
"Memory.embedding": 3, "Memory.beh_embedding": 1,
|
||||
"Topic.embedding": 1, "HierarchyTopic.embedding": 4,
|
||||
}
|
||||
assert report.cleared == {"Memory.beh_embedding": 1} # beh vector with no beh_abstraction
|
||||
assert report.hierarchy_labels_exact == 4 and report.hierarchy_labels_inexact == ()
|
||||
assert report.vector_indexes == ("memory_content_idx",)
|
||||
assert seen
|
||||
|
||||
check = await verify_graph_reembed(source, target, target_dims=tgt_dims)
|
||||
assert check.ok, check.problems
|
||||
assert check.vector_index and check.target_width == tgt_dims
|
||||
assert all(src == dst for src, dst in check.node_counts.values())
|
||||
assert all(src == dst for src, dst in check.rel_counts.values())
|
||||
|
||||
schema = nodes["schema"]
|
||||
rows = _read(target, f"MATCH (m:Memory {{node_id: '{schema.node_id}'}}) RETURN m.embedding, m.beh_embedding, m.custom_json, m.content;")
|
||||
emb, beh, custom, content = rows[0]
|
||||
assert len(emb) == tgt_dims
|
||||
assert emb == pytest.approx(new.vector("plans trips weeks ahead"), abs=1e-5)
|
||||
assert beh == pytest.approx(new.vector("prepares early under uncertainty"), abs=1e-5)
|
||||
assert json.loads(custom)["retrieval_text"] == "plans trips weeks ahead" and content == schema.content
|
||||
rows = _read(target, f"MATCH (m:Memory {{node_id: '{nodes['core'].node_id}'}}) RETURN m.embedding;")
|
||||
assert rows[0][0] == pytest.approx(new.vector("core: anticipates"), abs=1e-5)
|
||||
rows = _read(target, f"MATCH (m:Memory {{node_id: '{nodes['facts'][0].node_id}'}}) RETURN m.embedding IS NULL;")
|
||||
assert rows[0][0] is True # a NULL source vector stays NULL
|
||||
topics = dict(_read(target, "MATCH (t:Topic) RETURN t.name, t.embedding;"))
|
||||
assert topics["travel"] == pytest.approx(new.vector("travel"), abs=1e-5)
|
||||
assert topics["work"] is None
|
||||
for subject, path_json, embedding in _read(target, "MATCH (t:HierarchyTopic) RETURN t.subject, t.path_json, t.embedding;"):
|
||||
label, exact = hierarchy_topic_label(subject, path_json)
|
||||
assert exact and embedding == pytest.approx(new.vector(label), abs=1e-5)
|
||||
|
||||
# The rebuilt graph serves the SDK: vector search on the new index works.
|
||||
gs = await _open(target, tgt_dims)
|
||||
try:
|
||||
hits = await gs.vector_search(
|
||||
query_embedding=new.vector("plans trips weeks ahead"),
|
||||
isolation_key=schema.get_isolation_key(), limit=1,
|
||||
)
|
||||
assert hits and hits[0]["node_id"] == schema.node_id
|
||||
width = await gs._run("CALL table_info('Memory') RETURN *;")
|
||||
assert dict((r[1], r[2]) for r in width)["embedding"] == f"FLOAT[{tgt_dims}]"
|
||||
finally:
|
||||
await gs.close()
|
||||
|
||||
|
||||
async def test_source_labels_rebuild_exactly(tmp_path):
|
||||
"""The stored HierarchyTopic vectors were computed from the rebuilt label."""
|
||||
source = tmp_path / "graph"
|
||||
old, _ = await _seed_graph(source, 64)
|
||||
rows = _read(source, "MATCH (t:HierarchyTopic) RETURN t.subject, t.path_json, t.embedding;")
|
||||
assert len(rows) == 4
|
||||
for subject, path_json, embedding in rows:
|
||||
label, exact = hierarchy_topic_label(subject, path_json)
|
||||
assert exact
|
||||
assert embedding == pytest.approx(old.vector(label), abs=1e-6)
|
||||
|
||||
|
||||
def test_legacy_path_json_is_flagged_inexact():
|
||||
label, exact = hierarchy_topic_label("alice", json.dumps(["Travel", "Japan"]), "Japan")
|
||||
assert label == "alice: topic:Travel > facet:Japan" and not exact
|
||||
label, exact = hierarchy_topic_label("alice", "", "Japan")
|
||||
assert label == "alice: Japan" and not exact
|
||||
|
||||
|
||||
async def test_cancel_and_failure_leave_no_target(tmp_path):
|
||||
source, target = tmp_path / "graph", tmp_path / "graph-new"
|
||||
await _seed_graph(source, 64)
|
||||
with pytest.raises(ReembedCancelled):
|
||||
await reembed_graph(
|
||||
source, target, embedding=FakeEmbedder("new", 32), target_dims=32,
|
||||
cancel=lambda: True,
|
||||
)
|
||||
with pytest.raises(EmbeddingDimensionMismatchError):
|
||||
await reembed_graph(
|
||||
source, target, embedding=FakeEmbedder("new", 32, wrong_width_after=1), target_dims=32,
|
||||
)
|
||||
assert not os.path.exists(target)
|
||||
assert not os.path.exists(str(target) + ".reembed-partial")
|
||||
# A restart starts over and succeeds.
|
||||
report = await reembed_graph(source, target, embedding=FakeEmbedder("new", 32), target_dims=32)
|
||||
assert report.node_counts["Memory"] == 8
|
||||
with pytest.raises(FileExistsError):
|
||||
await reembed_graph(source, target, embedding=FakeEmbedder("new", 32), target_dims=32)
|
||||
again = await reembed_graph(
|
||||
source, target, embedding=FakeEmbedder("new", 32), target_dims=32, overwrite=True,
|
||||
)
|
||||
assert again.rel_counts == report.rel_counts
|
||||
|
||||
|
||||
async def test_refuses_open_source_and_drops_only_closed_graphs(tmp_path):
|
||||
source, target = tmp_path / "graph", tmp_path / "graph-new"
|
||||
await _seed_graph(source, 64)
|
||||
live = await _open(source, 64)
|
||||
try:
|
||||
with pytest.raises(RuntimeError):
|
||||
await reembed_graph(source, target, embedding=FakeEmbedder("new", 64), target_dims=64)
|
||||
with pytest.raises(RuntimeError):
|
||||
await drop_graph(source)
|
||||
finally:
|
||||
await live.close()
|
||||
with pytest.raises(ValueError):
|
||||
await reembed_graph(source, source, embedding=FakeEmbedder("new", 64), target_dims=64)
|
||||
await reembed_graph(source, target, embedding=FakeEmbedder("new", 64), target_dims=64)
|
||||
assert await drop_graph(target) is True
|
||||
assert not os.path.exists(target)
|
||||
assert await drop_graph(target) is False
|
||||
assert os.path.exists(source)
|
||||
Reference in new issue
Block a user