diff --git a/hy_memory/client.py b/hy_memory/client.py index e9319e9..51249b0 100644 --- a/hy_memory/client.py +++ b/hy_memory/client.py @@ -1864,6 +1864,7 @@ class HyMemoryClient: branch: Optional[str] = None, request_id: Optional[str] = None, layers: Optional[List[Any]] = None, + statuses: Optional[List[Any]] = None, embed_override: Optional[Dict[str, Any]] = None, query_embedding: Optional[List[float]] = None, rerank: bool = True, @@ -1956,6 +1957,8 @@ class HyMemoryClient: layers: 可选。pro/ultra 下只召回指定层级（字符串或 MemoryLayer），在召回阶段生效。lite 下忽略该参数并 强制只召回 L1_RAW。 + statuses: 仅用于 L1_RAW 定向搜索，可显式包含 active / shadow。 + 非 raw 搜索或 scene="all" 传入时拒绝，避免静默忽略。 rerank: 默认 True（原行为）。False 时本次调用跳过全部重排阶段： chat 路的本地 cross-encoder 重排（recall.local_rerank_enabled） 与记忆关系图重排（graph_rerank / episode 扩展），working/ @@ -1999,7 +2002,7 @@ class HyMemoryClient: intention_limit=intention_limit, created_after=created_after, reader=reader, workspace_id=workspace_id, branch=branch, - request_id=request_id, layers=layers, + request_id=request_id, layers=layers, statuses=statuses, embed_override=embed_override, query_embedding=query_embedding, rerank=rerank, @@ -3387,12 +3390,14 @@ class HyMemoryClient: request_id: str, t0: float, query_embedding: Optional[List[float]], + statuses: Optional[List[Any]], ) -> Dict[str, Any]: """Implement `search(..., layers=['l1_raw'])` without a second API.""" from .models.memory import MemoryLayer, MemoryStatus embedding = query_embedding or await self._embed_service.embed(query) fetch = min(max(limit * (10 if session_ids else 2), limit), 10_000) + status_filter = self._normalize_statuses(statuses) or [MemoryStatus.ACTIVE] hits = await self._vector_store.search( query_embedding=embedding, user_ids=user_ids, @@ -3402,7 +3407,7 @@ class HyMemoryClient: layers=[MemoryLayer.L1_RAW], limit=fetch, score_threshold=min_score, - status_filter=[MemoryStatus.ACTIVE], + status_filter=status_filter, only_latest=False, created_after=created_after, ) @@ -3420,7 +3425,7 @@ class HyMemoryClient: "memory_id": row.pop("raw_id"), **row, "layer": MemoryLayer.L1_RAW.value, - "status": MemoryStatus.ACTIVE.value, + "status": node.status.value, "score": round(float(hit.get("score", 0.0)), 4), } ) @@ -3864,6 +3869,7 @@ class HyMemoryClient: branch: Optional[str] = None, request_id: Optional[str] = None, layers: Optional[List[Any]] = None, + statuses: Optional[List[Any]] = None, embed_override: Optional[Dict[str, Any]] = None, query_embedding: Optional[List[float]] = None, rerank: bool = True, @@ -3893,7 +3899,7 @@ class HyMemoryClient: schema_min_score=schema_min_score, schema_limit=schema_limit, intention_limit=intention_limit, created_after=created_after, reader=reader, workspace_id=workspace_id, branch=branch, - request_id=request_id, layers=layers, + request_id=request_id, layers=layers, statuses=statuses, query_embedding=query_embedding, rerank=rerank, ) @@ -3908,7 +3914,7 @@ class HyMemoryClient: schema_min_score=schema_min_score, schema_limit=schema_limit, intention_limit=intention_limit, created_after=created_after, reader=reader, workspace_id=workspace_id, branch=branch, - request_id=request_id, layers=layers, + request_id=request_id, layers=layers, statuses=statuses, query_embedding=query_embedding, rerank=rerank, ) @@ -3938,6 +3944,7 @@ class HyMemoryClient: branch: Optional[str] = None, request_id: Optional[str] = None, layers: Optional[List[Any]] = None, + statuses: Optional[List[Any]] = None, query_embedding: Optional[List[float]] = None, rerank: bool = True, ) -> Dict[str, Any]: @@ -4023,6 +4030,17 @@ class HyMemoryClient: "Use single agent_id with multiple session_ids, or multiple agent_ids without session_ids." ) + if statuses is not None: + from .models.memory import MemoryLayer + + requested_layers = self._normalize_layers(layers) + if scene_all or ( + not lite_mode and requested_layers != [MemoryLayer.L1_RAW] + ): + raise ValueError( + "statuses is supported only for explicit L1_RAW search" + ) + if scene_all: return await self._search_all_scenes( query, queries=queries, search_mode=search_mode, @@ -4358,6 +4376,7 @@ class HyMemoryClient: request_id=request_id, t0=t0, query_embedding=query_embedding, + statuses=statuses, ) read_req = ReadRequest( diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py index c2fe6fe..c6d7f21 100644 --- a/memhub/apps/core/src/one_memory/api.py +++ b/memhub/apps/core/src/one_memory/api.py @@ -270,6 +270,7 @@ from one_memory.onboarding import ( OnboardingState, OnboardingStore, ) +from one_memory.raw_turns import RawDisplayTurn, raw_display_turns from one_memory.recall import ( GovernedRecallTarget, RecallReceipt, @@ -604,24 +605,37 @@ class SdkRawItem(HySdkRawRecord): role: str | None = None content_hash: str created: datetime | None = None + turn_id: str | None = Field(default=None, min_length=1, max_length=96) + turn_index: int | None = Field(default=None, ge=0) + turn_count: int | None = Field(default=None, ge=1) @classmethod - def of(cls, record: HySdkRawRecord) -> "SdkRawItem": + def of( + cls, record: HySdkRawRecord, turn: RawDisplayTurn | None = None + ) -> "SdkRawItem": + content = turn.content if turn is not None else record.content + raw_messages = ( + turn.messages if turn is not None else raw_turn_messages(record.content) + ) messages = tuple( SdkRawMessage(role=role, content=text) - for role, text in raw_turn_messages(record.content) + for role, text in raw_messages ) roles = {message.role for message in messages} return cls( - **record.model_dump(), + **record.model_dump(exclude={"content"}), + content=content, messages=messages, role=next(iter(roles)) if len(roles) == 1 and "" not in roles else None, - content_hash="sha256:" + hashlib.sha256(record.content.encode("utf-8")).hexdigest(), + content_hash="sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest(), created=( datetime.fromtimestamp(record.gmt_created, tz=UTC) if record.gmt_created is not None else None ), + turn_id=turn.turn_id if turn is not None else None, + turn_index=turn.turn_index if turn is not None else None, + turn_count=turn.turn_count if turn is not None else None, ) @@ -634,9 +648,9 @@ class SdkRawPage(BaseModel): total: int = Field(ge=0) offset: int = Field(ge=0) limit: int = Field(ge=1, le=500) - #: Set when a time filter was applied to this page in Core (the SDK list - #: has none): `items` is the page's rows inside the range, `total` stays - #: the SDK's count before that filter. + unit: Literal["raw", "turn"] = "raw" + #: Set when a time filter was applied in Core. For storage `raw`, `total` + #: remains the SDK count; for product `turn`, it is the filtered turn count. time_filtered: bool = False @@ -4441,6 +4455,16 @@ def create_app( raise _sdk_http_error(error, "hy_memory_delete_failed") from None return Response(status_code=204) + raw_turn_cache: dict[ + tuple[object, ...], + tuple[ + int, + tuple[tuple[str, str | None], ...], + tuple[tuple[HySdkRawRecord, RawDisplayTurn], ...], + ], + ] = {} + raw_turn_cache_locks: dict[tuple[object, ...], asyncio.Lock] = {} + @app.get( "/api/v2/raw", response_model=SdkRawPage, @@ -4450,6 +4474,7 @@ def create_app( source: list[str] | None = Query(default=None), # noqa: B008 offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=1, le=500), + unit: Literal["raw", "turn"] = Query(default="raw"), user_id: str = Query(default=DEFAULT_USER_ID, pattern=USER_ID_PATTERN), exclude_sources: list[str] | None = Query(default=None), # noqa: B008 session_id: list[str] | None = Query(default=None), # noqa: B008 @@ -4463,7 +4488,10 @@ def create_app( The SDK lists raw only when asked for `layers=["l1_raw"]`, and only active rows unless told otherwise. Pro shadows a raw once extraction succeeds, so a raw page reads active and shadow (`statuses` narrows - it). `source` and `session_id` (a conversation) may repeat or be comma + it). `unit=turn` projects storage batches into stable product turns; + the first read builds an append-only in-memory index and later polls + only probe the latest SDK page unless it changed. `source` and + `session_id` (a conversation) may repeat or be comma separated. `after`/`before` (unix seconds, against the conversation time, else the stored time) have no SDK list filter: they cut this page in Core and mark it `time_filtered`. @@ -4477,24 +4505,145 @@ def create_app( sessions = split_csv(session_id) try: client = await _sdk_client_handle(app).client() - value = await asyncio.to_thread( - client.list_memories, - user_id=user_id, + query = { + "user_id": user_id, **raw_list_filters(chosen_statuses), - include_sources=sources or None, - exclude_sources=excluded or None, - session_ids=sessions or None, - offset=offset, - limit=limit, - order=order, - ) - page = HySdkRawPage.from_memory_list(value, limit=limit, offset=offset) + "include_sources": sources or None, + "exclude_sources": excluded or None, + "session_ids": sessions or None, + "order": order if unit == "raw" else "desc", + } + if unit == "raw": + value = await asyncio.to_thread( + client.list_memories, + **query, + offset=offset, + limit=limit, + ) + page = HySdkRawPage.from_memory_list(value, limit=limit, offset=offset) + records = page.raw + else: + cache_key = ( + user_id, + tuple(sorted(chosen_statuses)), + tuple(sorted(sources)), + tuple(sorted(excluded)), + tuple(sorted(sessions)), + ) + if len(raw_turn_cache_locks) > 64: + for stale_key, stale_lock in tuple(raw_turn_cache_locks.items()): + if ( + stale_key not in raw_turn_cache + and not stale_lock.locked() + ): + raw_turn_cache_locks.pop(stale_key, None) + lock = raw_turn_cache_locks.setdefault(cache_key, asyncio.Lock()) + async with lock: + for _attempt in range(3): + value = await asyncio.to_thread( + client.list_memories, + **query, + offset=0, + limit=500, + ) + first_page = HySdkRawPage.from_memory_list( + value, limit=500, offset=0 + ) + signature = tuple( + (record.raw_id, record.status) for record in first_page.raw + ) + cached = raw_turn_cache.get(cache_key) + if ( + cached is not None + and cached[0] == first_page.total + and cached[1] == signature + ): + projected_turns = cached[2] + break + + records_list = list(first_page.raw) + raw_offset = len(first_page.raw) + raw_total = first_page.total + while first_page.raw and raw_offset < raw_total: + value = await asyncio.to_thread( + client.list_memories, + **query, + offset=raw_offset, + limit=500, + ) + raw_page = HySdkRawPage.from_memory_list( + value, limit=500, offset=raw_offset + ) + records_list.extend(raw_page.raw) + raw_offset += len(raw_page.raw) + if not raw_page.raw: + break + + verify_value = await asyncio.to_thread( + client.list_memories, + **query, + offset=0, + limit=500, + ) + verify = HySdkRawPage.from_memory_list( + verify_value, limit=500, offset=0 + ) + verify_signature = tuple( + (record.raw_id, record.status) for record in verify.raw + ) + if ( + verify.total != first_page.total + or verify_signature != signature + ): + continue + projected_turns = tuple( + (record, turn) + for record in records_list + for turn in reversed(raw_display_turns(record)) + ) + raw_turn_cache[cache_key] = ( + first_page.total, + signature, + projected_turns, + ) + while len(raw_turn_cache) > 32: + oldest = next(iter(raw_turn_cache)) + raw_turn_cache.pop(oldest) + old_lock = raw_turn_cache_locks.get(oldest) + if old_lock is not None and not old_lock.locked(): + raw_turn_cache_locks.pop(oldest, None) + break + else: + raise RuntimeError("hy_memory_raw_snapshot_busy") + if cache_key not in raw_turn_cache and not lock.locked(): + raw_turn_cache_locks.pop(cache_key, None) except HTTPException: raise except Exception as error: raise _sdk_http_error(error, "hy_memory_raw_list_failed") from None - records = page.raw time_filtered = after is not None or before is not None + if unit == "turn": + turns = projected_turns + if time_filtered: + turns = tuple( + (record, turn) + for record, turn in turns + if (at := record.memory_at or record.gmt_created) is not None + and (after is None or at >= after) + and (before is None or at < before) + ) + if order == "asc": + turns = tuple(reversed(turns)) + selected = turns[offset : offset + limit] + return SdkRawPage( + user_id=user_id, + items=tuple(SdkRawItem.of(record, turn) for record, turn in selected), + total=len(turns), + offset=offset, + limit=limit, + unit="turn", + time_filtered=time_filtered, + ) if time_filtered: records = tuple( record @@ -4509,6 +4658,7 @@ def create_app( total=page.total, offset=offset, limit=limit, + unit="raw", time_filtered=time_filtered, ) diff --git a/memhub/apps/core/src/one_memory/context_models.py b/memhub/apps/core/src/one_memory/context_models.py index afef2bc..3d0b5a4 100644 --- a/memhub/apps/core/src/one_memory/context_models.py +++ b/memhub/apps/core/src/one_memory/context_models.py @@ -66,6 +66,8 @@ class SdkContextSearchRequest(ContextSearchRequest, SdkSearchOptions): limit: int = Field(default=10, ge=1, le=20, strict=True) schema_limit: int = Field(default=3, ge=0, le=20, strict=True) intention_limit: int = Field(default=1, ge=0, le=20, strict=True) + #: Product callers can request one item per turn instead of SDK raw batches. + raw_unit: Literal["raw", "turn"] = "raw" #: Whose memories to search. The App never sends it and reads `local`. user_id: UserId = DEFAULT_USER_ID @@ -123,6 +125,9 @@ class SdkRawContextItem(BaseModel): memory_at: datetime | None = None score: float = Field(ge=0, le=1) truncated: bool = False + turn_id: str | None = Field(default=None, min_length=1, max_length=96) + turn_index: int | None = Field(default=None, ge=0) + turn_count: int | None = Field(default=None, ge=1) class SdkMemoryContextItem(BaseModel): diff --git a/memhub/apps/core/src/one_memory/context_routes.py b/memhub/apps/core/src/one_memory/context_routes.py index da27e6d..8518858 100644 --- a/memhub/apps/core/src/one_memory/context_routes.py +++ b/memhub/apps/core/src/one_memory/context_routes.py @@ -30,6 +30,7 @@ from one_memory.hy_sdk_types import ( parse_sdk_search, ) from one_memory.imports.storage import EvidenceStore +from one_memory.raw_turns import raw_display_turns from one_memory.sdk_search_options import SdkSearchFilters, SearchLane from one_memory.semantic_deadline import SemanticMemoryHit from one_memory.user_scope import owns @@ -120,6 +121,9 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter: sources = raw_lane_sources(authority.raw_sources, value.include_sources) if sources is None: return () + candidate_limit = ( + min(value.limit * 10, 200) if value.raw_unit == "turn" else value.limit + ) raw = await asyncio.to_thread( client.search, value.query, @@ -127,7 +131,7 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter: "user_ids": [value.user_id], "agent_ids": [], "include_sources": sources, - "limit": value.limit, + "limit": candidate_limit, "min_score": 0.0, "schema_min_score": 0.0, "schema_limit": 0, @@ -135,27 +139,59 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter: "scene": "chat", "reader": HY_MEMORY_READER, "layers": ["l1_raw"], + "statuses": ["active", "shadow"], **raw_lane_overrides(value), }, ) hits = tuple( hit for hit in parse_sdk_raw_search(raw) if value.passes_floor("raw", hit.score) ) - return tuple( - SdkRawContextItem( - raw_id=hit.raw_id, - content=hit.content, - source=hit.source, - session_id=hit.session_id, - memory_at=( - datetime.fromtimestamp(hit.memory_at, tz=UTC) - if hit.memory_at is not None - else None - ), - score=hit.score, - ) - for hit in hits - ) + items: list[SdkRawContextItem] = [] + for hit in hits: + turns = raw_display_turns(hit) if value.raw_unit == "turn" else () + if turns: + query = value.query.casefold() + lexical = tuple(turn for turn in turns if query in turn.content.casefold()) + # A batch-level embedding cannot identify which contained + # turn matched semantically. Return only turns grounded by + # the query text instead of assigning every sibling the + # batch score and showing unrelated conversations. + for turn in lexical: + items.append( + SdkRawContextItem( + raw_id=hit.raw_id, + turn_id=turn.turn_id, + turn_index=turn.turn_index, + turn_count=turn.turn_count, + content=turn.content, + source=hit.source, + session_id=hit.session_id, + memory_at=( + datetime.fromtimestamp(hit.memory_at, tz=UTC) + if hit.memory_at is not None + else None + ), + score=hit.score, + ) + ) + else: + items.append( + SdkRawContextItem( + raw_id=hit.raw_id, + content=hit.content, + source=hit.source, + session_id=hit.session_id, + memory_at=( + datetime.fromtimestamp(hit.memory_at, tz=UTC) + if hit.memory_at is not None + else None + ), + score=hit.score, + ) + ) + if len(items) >= value.limit: + break + return tuple(items[: value.limit]) async def memory_lane() -> tuple[SdkMemoryContextItem, ...]: # lite extracts nothing: its search answers raw turns, which the diff --git a/memhub/apps/core/src/one_memory/raw_turns.py b/memhub/apps/core/src/one_memory/raw_turns.py new file mode 100644 index 0000000..990b1d9 --- /dev/null +++ b/memhub/apps/core/src/one_memory/raw_turns.py @@ -0,0 +1,62 @@ +"""Turn-level read projection for SDK raw batches. + +HY keeps several complete conversation turns in one L1 raw node so model +writes stay batched. Product readers must not expose that storage batch as +one conversation item: this module gives every contained turn a stable +identity while retaining the parent raw id for traceability. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +from one_memory.hy_sdk_types import HySdkRawRecord, raw_turn_messages + + +@dataclass(frozen=True) +class RawDisplayTurn: + turn_id: str + turn_index: int + turn_count: int + content: str + messages: tuple[tuple[str, str], ...] + + +def raw_display_turns(record: HySdkRawRecord) -> tuple[RawDisplayTurn, ...]: + """Split one stored raw batch into user-opened turns. + + A user message opens a new turn; assistant/tool/system messages stay with + that user message until the next user message. Legacy content with no user + marker remains one turn so it never disappears from the product. + """ + messages = raw_turn_messages(record.content) + grouped: list[list[tuple[str, str]]] = [] + for message in messages: + role, _ = message + has_user = bool(grouped) and any(item[0] == "user" for item in grouped[-1]) + if not grouped or (role == "user" and has_user): + grouped.append([message]) + else: + grouped[-1].append(message) + if not grouped: + grouped = [[("", record.content)]] + + digest = hashlib.sha256(record.raw_id.encode("utf-8")).hexdigest() + count = len(grouped) + return tuple( + RawDisplayTurn( + turn_id=f"raw-turn:{digest}:{index}", + turn_index=index, + turn_count=count, + content=_turn_content(turn), + messages=tuple(turn), + ) + for index, turn in enumerate(grouped) + ) + + +def _turn_content(messages: list[tuple[str, str]]) -> str: + return "\n".join( + f"[{role}]: {content}" if role else content for role, content in messages + ) diff --git a/memhub/apps/core/tests/test_sdk_api_surface.py b/memhub/apps/core/tests/test_sdk_api_surface.py index a907328..4b14e07 100644 --- a/memhub/apps/core/tests/test_sdk_api_surface.py +++ b/memhub/apps/core/tests/test_sdk_api_surface.py @@ -42,7 +42,10 @@ RAW_ROWS = { "local": [ { "memory_id": "raw-1", - "content": "[user]: 我叫费天翔\n[assistant]: 好的，记住了。", + "content": ( + "[user]: 我叫费天翔\n[assistant]: 好的，记住了。" + "\n[user]: 我喜欢绿茶\n[assistant]: 也记住了。" + ), "status": "shadow", "session_id": "s-1", "source": "hermes", @@ -347,6 +350,33 @@ def test_every_context_search_option_reaches_the_sdk_and_absent_ones_keep_the_ol run_with_sdk(tmp_path, monkeypatch, scenario) +@pytest.mark.fresh_sdk +def test_raw_context_search_can_return_turns_instead_of_storage_batches( + tmp_path, monkeypatch +) -> None: + async def scenario(app, sdk, client): + response = await client.post( + "/api/v2/context/search", + json={ + "query": "绿茶", + "kinds": ["raw"], + "raw_unit": "turn", + "limit": 20, + }, + ) + assert response.status_code == 200, response.text + raw_one = [ + item for item in response.json()["items"] if item["raw_id"] == "raw-1" + ] + assert len(raw_one) == 1 + assert raw_one[0]["turn_index"] == 1 + assert "我喜欢绿茶" in raw_one[0]["content"] + assert "我叫费天翔" not in raw_one[0]["content"] + assert sdk.last("search")["statuses"] == ["active", "shadow"] + + run_with_sdk(tmp_path, monkeypatch, scenario) + + @pytest.mark.fresh_sdk def test_every_recall_and_ask_option_reaches_the_sdk(tmp_path, monkeypatch) -> None: async def scenario(app, sdk, client): @@ -574,7 +604,12 @@ def test_raw_lists_active_and_shadow_turns_with_roles_and_hashes(tmp_path, monke assert sent["layers"] == ["l1_raw"] and sent["statuses"] == ["active", "shadow"] first = next(item for item in body["items"] if item["raw_id"] == "raw-1") assert first["status"] == "shadow" - assert [message["role"] for message in first["messages"]] == ["user", "assistant"] + assert [message["role"] for message in first["messages"]] == [ + "user", + "assistant", + "user", + "assistant", + ] assert first["messages"][0]["content"] == "我叫费天翔" assert first["role"] is None expected = hashlib.sha256(first["content"].encode()).hexdigest() @@ -584,6 +619,27 @@ def test_raw_lists_active_and_shadow_turns_with_roles_and_hashes(tmp_path, monke second = next(item for item in body["items"] if item["raw_id"] == "raw-2") assert second["role"] == "user" + turns = (await client.get("/api/v2/raw", params={"unit": "turn"})).json() + assert turns["unit"] == "turn" and turns["total"] == 3 + first_turns = [item for item in turns["items"] if item["raw_id"] == "raw-1"] + assert [item["turn_index"] for item in first_turns] == [1, 0] + assert len({item["turn_id"] for item in first_turns}) == 2 + assert [[message["content"] for message in item["messages"]] for item in first_turns] == [ + ["我喜欢绿茶", "也记住了。"], + ["我叫费天翔", "好的，记住了。"], + ] + calls_before_refresh = len(sdk.all("list_memories")) + await client.get("/api/v2/raw", params={"unit": "turn"}) + assert len(sdk.all("list_memories")) == calls_before_refresh + 1 + + ascending = ( + await client.get("/api/v2/raw", params={"unit": "turn", "order": "asc"}) + ).json() + ascending_first = [ + item for item in ascending["items"] if item["raw_id"] == "raw-1" + ] + assert [item["turn_index"] for item in ascending_first] == [0, 1] + await client.get( "/api/v2/raw", params=[("source", "hermes,codex"), ("session_id", "s-1"), ("order", "asc")], diff --git a/memhub/apps/web/src/MemorySpace.r6.test.tsx b/memhub/apps/web/src/MemorySpace.r6.test.tsx index 15a14fc..19b4131 100644 --- a/memhub/apps/web/src/MemorySpace.r6.test.tsx +++ b/memhub/apps/web/src/MemorySpace.r6.test.tsx @@ -387,6 +387,67 @@ it("pages and filters the raw conversation data plane in lite", async () => { ); }); +it("selects and renders one turn when two turns share a raw storage batch", async () => { + setup(); + reads.processing.mockResolvedValue(modeProcessing("lite")); + reads.raw.mockResolvedValue({ + items: [ + { + raw_id: "raw-batch", + turn_id: "raw-turn:batch:0", + turn_index: 0, + turn_count: 2, + content: "[user]: 第一问\n[assistant]: 第一答", + user_id: "local", + agent_id: "all-apps", + source: "cursor", + session_id: "session-batch", + memory_at: null, + gmt_created: 1_761_000_000, + custom: {}, + messages: [ + { role: "user", content: "第一问" }, + { role: "assistant", content: "第一答" }, + ], + }, + { + raw_id: "raw-batch", + turn_id: "raw-turn:batch:1", + turn_index: 1, + turn_count: 2, + content: "[user]: 第二问\n[assistant]: 第二答", + user_id: "local", + agent_id: "all-apps", + source: "cursor", + session_id: "session-batch", + memory_at: null, + gmt_created: 1_761_000_000, + custom: {}, + messages: [ + { role: "user", content: "第二问" }, + { role: "assistant", content: "第二答" }, + ], + }, + ], + total: 2, + limit: 20, + offset: 0, + unit: "turn", + }); + const user = userEvent.setup(); + render(); + + await user.click(await screen.findByRole("button", { name: /第二问/ })); + + await waitFor(() => { + const detail = document.querySelector(".map-dialogue-reading"); + expect(detail?.textContent).toContain("第二问"); + expect(detail?.textContent).toContain("第二答"); + expect(detail?.textContent).not.toContain("第一问"); + expect(detail?.textContent).not.toContain("第一答"); + }); +}); + it("switches data planes without keeping rows or details from the prior mode", async () => { setup(); vi.useFakeTimers(); @@ -482,6 +543,27 @@ it("opens a search hit with the complete role transcript", async () => { items: [ { raw_id: "raw-search", + turn_id: "raw-turn:search:0", + turn_index: 0, + turn_count: 2, + content: "[user]: 第一轮问题\n[assistant]: 第一轮回答", + user_id: "local", + agent_id: "all-apps", + source: "cursor", + session_id: "session-search", + memory_at: null, + gmt_created: 1_761_000_000, + custom: {}, + messages: [ + { role: "user", content: "第一轮问题" }, + { role: "assistant", content: "第一轮回答" }, + ], + }, + { + raw_id: "raw-search", + turn_id: "raw-turn:search:1", + turn_index: 1, + turn_count: 2, content: "[user]: 完整问题\n[assistant]: 完整回答", user_id: "local", agent_id: "all-apps", @@ -496,7 +578,7 @@ it("opens a search hit with the complete role transcript", async () => { ], }, ], - total: 1, + total: 2, limit: 500, offset: 0, } @@ -506,6 +588,9 @@ it("opens a search hit with the complete role transcript", async () => { { kind: "raw", raw_id: "raw-search", + turn_id: "raw-turn:search:1", + turn_index: 1, + turn_count: 2, content: "被截断的搜索摘要", source: "cursor", session_id: "session-search", @@ -525,6 +610,9 @@ it("opens a search hit with the complete role transcript", async () => { expect(await screen.findByText("完整问题")).toBeTruthy(); expect(screen.getByText("完整回答")).toBeTruthy(); + expect( + document.querySelector(".map-dialogue-reading")?.textContent, + ).not.toContain("第一轮问题"); expect(reads.raw).toHaveBeenCalledWith( expect.objectContaining({ source: "cursor", diff --git a/memhub/apps/web/src/MemorySpace.tsx b/memhub/apps/web/src/MemorySpace.tsx index f0a7595..4be8d00 100644 --- a/memhub/apps/web/src/MemorySpace.tsx +++ b/memhub/apps/web/src/MemorySpace.tsx @@ -61,6 +61,7 @@ import { type RawMapRecord, rawDetail, rawLibraryPage, + rawRecordId, rawSearchPage, } from "./sdk-raw-map"; import { sourceNameOr, UNATTRIBUTED_SOURCE_ID } from "./source-label"; @@ -277,8 +278,11 @@ export function MemorySpace({ const rememberRaw = (items: RawMapRecord[]) => { const next = new Map(); if (selectedRawRef.current) - next.set(selectedRawRef.current.raw_id, selectedRawRef.current); - for (const item of items) next.set(item.raw_id, item); + next.set( + rawRecordId(selectedRawRef.current), + selectedRawRef.current, + ); + for (const item of items) next.set(rawRecordId(item), item); rawRecords.current = next; }; const [page, graph] = lite @@ -717,7 +721,9 @@ export function MemorySpace({ offset, limit: 500, }); - const full = page.items.find((item) => item.raw_id === memoryId); + const full = page.items.find( + (item) => rawRecordId(item) === memoryId, + ); if (full) { record = full; rawRecords.current.set(memoryId, full); diff --git a/memhub/apps/web/src/SdkRawHistory.tsx b/memhub/apps/web/src/SdkRawHistory.tsx index c59af4b..b0aac72 100644 --- a/memhub/apps/web/src/SdkRawHistory.tsx +++ b/memhub/apps/web/src/SdkRawHistory.tsx @@ -3,6 +3,7 @@ import { type FormEvent, useEffect, useRef, useState } from "react"; import { getSdkRaw, searchSdkRaw } from "./api"; import { useGuiLanguage } from "./gui-language"; import { subscribePoll } from "./poll-scheduler"; +import { rawRecordId } from "./sdk-raw-map"; export function SdkRawHistory({ active = true }: { active?: boolean }) { const { locale, t, tn } = useGuiLanguage(); @@ -121,7 +122,7 @@ export function SdkRawHistory({ active = true }: { active?: boolean }) { ) : null}
    {items.map((item) => ( -
  1. +

  2. {item.content}

    {item.source} · {item.session_id} @@ -140,6 +141,9 @@ function asContextItem(item: SdkRawRecord): SdkRawContextItem { return { kind: "raw", raw_id: item.raw_id, + turn_id: item.turn_id, + turn_index: item.turn_index, + turn_count: item.turn_count, content: item.content, source: item.source, session_id: item.session_id, diff --git a/memhub/apps/web/src/SelectedRawDetail.tsx b/memhub/apps/web/src/SelectedRawDetail.tsx index b24d357..fc55a37 100644 --- a/memhub/apps/web/src/SelectedRawDetail.tsx +++ b/memhub/apps/web/src/SelectedRawDetail.tsx @@ -5,7 +5,7 @@ import type { } from "@one-memory/contracts"; import { clockStamp } from "./clock-stamp"; import { useGuiLanguage } from "./gui-language"; -import { rawRecordTime } from "./sdk-raw-map"; +import { rawRecordId, rawRecordTime } from "./sdk-raw-map"; import { sourceNameOr, UNATTRIBUTED_SOURCE_ID } from "./source-label"; type RawRecord = SdkRawRecord | SdkRawContextItem; @@ -50,7 +50,7 @@ export function SelectedRawDetail({

    {t("memory:原始对话")}

      {messages.map((message) => ( -
    • +
    • {authorLabel(message.role, t)}

      {message.content}

      • diff --git a/memhub/apps/web/src/api.raw.test.ts b/memhub/apps/web/src/api.raw.test.ts index f6d9257..0bd9db9 100644 --- a/memhub/apps/web/src/api.raw.test.ts +++ b/memhub/apps/web/src/api.raw.test.ts @@ -32,6 +32,7 @@ it("lists raw conversations for the selected application", async () => { session_id: undefined, offset: 20, limit: 20, + unit: "turn", }, signal, }); @@ -49,6 +50,7 @@ it("keeps the application filter when searching raw conversations", async () => query: "release plan", kinds: ["raw"], include_sources: ["claude-code"], + raw_unit: "turn", limit: 20, character_budget: 12_000, }, diff --git a/memhub/apps/web/src/api.ts b/memhub/apps/web/src/api.ts index d866095..74d2538 100644 --- a/memhub/apps/web/src/api.ts +++ b/memhub/apps/web/src/api.ts @@ -986,16 +986,18 @@ export async function getSdkRaw({ sessionId, offset = 0, limit = 100, + unit = "turn", signal, }: { source?: string; sessionId?: string; offset?: number; limit?: number; + unit?: "raw" | "turn"; signal?: AbortSignal; } = {}): Promise { const response = await client.get("/api/v2/raw", { - params: { source, session_id: sessionId, offset, limit }, + params: { source, session_id: sessionId, offset, limit, unit }, signal, }); return response.data; @@ -1011,6 +1013,7 @@ export async function searchSdkRaw( query, kinds: ["raw"], ...(options.source ? { include_sources: [options.source] } : {}), + raw_unit: "turn", limit: 20, character_budget: 12_000, }, diff --git a/memhub/apps/web/src/sdk-raw-map.test.ts b/memhub/apps/web/src/sdk-raw-map.test.ts index a912b27..b9860df 100644 --- a/memhub/apps/web/src/sdk-raw-map.test.ts +++ b/memhub/apps/web/src/sdk-raw-map.test.ts @@ -76,4 +76,28 @@ describe("raw map projection", () => { history: [], }); }); + + it("uses a turn identity while retaining the parent raw id", () => { + const turn = { + ...record, + raw_id: "raw-batch", + turn_id: "raw-turn:batch:1", + turn_index: 1, + turn_count: 2, + content: "[user]: second question\n[assistant]: second answer", + messages: [ + { role: "user", content: "second question" }, + { role: "assistant", content: "second answer" }, + ], + }; + + expect( + rawLibraryPage({ items: [turn], total: 2, offset: 0, limit: 20 }), + ).toMatchObject({ + items: [{ memory_id: "raw-turn:batch:1", content: turn.content }], + source_ids_by_memory: { "raw-turn:batch:1": ["cursor"] }, + }); + expect(rawDetail(turn).memory.memory_id).toBe("raw-turn:batch:1"); + expect(turn.raw_id).toBe("raw-batch"); + }); }); diff --git a/memhub/apps/web/src/sdk-raw-map.ts b/memhub/apps/web/src/sdk-raw-map.ts index 4cdc81f..bb70418 100644 --- a/memhub/apps/web/src/sdk-raw-map.ts +++ b/memhub/apps/web/src/sdk-raw-map.ts @@ -9,6 +9,10 @@ import type { export type RawMapRecord = SdkRawRecord | SdkRawContextItem; +export function rawRecordId(record: RawMapRecord): string { + return record.turn_id || record.raw_id; +} + const EARLIEST_PLAUSIBLE = Date.UTC(2000, 0, 1); const LATEST_PLAUSIBLE = () => Date.now() + 86_400_000; @@ -43,7 +47,7 @@ function sourceOf(record: RawMapRecord): string { export function rawMapMemory(record: RawMapRecord): GovernedMemory { const created = rawRecordTime(record) ?? ""; return { - memory_id: record.raw_id, + memory_id: rawRecordId(record), user_id: "user_id" in record ? record.user_id : "local", kind: "chat", status: "confirmed", @@ -75,17 +79,18 @@ function pageFor( { at: string; basis: "conversation" | "import" } > = {}; for (const record of records) { - sourceIds[record.raw_id] = [sourceOf(record)]; + const recordId = rawRecordId(record); + sourceIds[recordId] = [sourceOf(record)]; const conversationAt = isoTime(record.memory_at); const importedAt = "gmt_created" in record ? isoTime(record.gmt_created) : ""; if (conversationAt) - sourceTime[record.raw_id] = { + sourceTime[recordId] = { at: conversationAt, basis: "conversation", }; else if (importedAt) - sourceTime[record.raw_id] = { at: importedAt, basis: "import" }; + sourceTime[recordId] = { at: importedAt, basis: "import" }; } return { items: records.map(rawMapMemory), diff --git a/memhub/docs/contracts/memory-mode-evolve-v1.md b/memhub/docs/contracts/memory-mode-evolve-v1.md index ec2a698..138d30b 100644 --- a/memhub/docs/contracts/memory-mode-evolve-v1.md +++ b/memhub/docs/contracts/memory-mode-evolve-v1.md @@ -12,7 +12,7 @@ memory route. `user_id` is optional (default `local`) and must match | Mode | SDK client | What happens to a conversation | What search reads | |---|---|---|---| -| `lite` | `mode="lite"` | Stored as one raw turn with one embedding. No model call; no generation model needs to be configured. | Raw turns only (the SDK forces `L1_RAW`). | +| `lite` | `mode="lite"` | Stored in embedded L1 raw batches. No model call; no generation model needs to be configured. The product page projects each batch back into individual conversation turns. | Raw turns only (the SDK forces `L1_RAW`). | | `pro` (default) | `mode="pro"` | Stored as raw, then a model extracts memories (System 1). The raw becomes SHADOW once extraction succeeds. | Extracted memories (facts, profile, intentions). | | `ultra` | `mode="ultra"` | Same as pro, and new facts are also projected into the knowledge graph. Every night, evolve runs System 2 for every user. | Same as pro, plus the graph-backed lanes below. | diff --git a/memhub/packages/contracts/src/context.ts b/memhub/packages/contracts/src/context.ts index 32abbea..4386893 100644 --- a/memhub/packages/contracts/src/context.ts +++ b/memhub/packages/contracts/src/context.ts @@ -87,7 +87,12 @@ export interface ContextSearchResponse { export interface SdkRawContextItem { kind: "raw"; + /** Parent SDK L1 raw storage batch. */ raw_id: string; + /** Stable product identity of one turn inside `raw_id`, when requested. */ + turn_id?: string | null; + turn_index?: number | null; + turn_count?: number | null; content: string; source: string; session_id: string; diff --git a/memhub/packages/contracts/src/index.ts b/memhub/packages/contracts/src/index.ts index 4011d7e..dd86494 100644 --- a/memhub/packages/contracts/src/index.ts +++ b/memhub/packages/contracts/src/index.ts @@ -675,7 +675,12 @@ export interface SdkUserProfileResponse { } export interface SdkRawRecord { + /** Parent SDK L1 raw storage batch. */ raw_id: string; + /** Stable product identity of one turn inside `raw_id`, when unit=turn. */ + turn_id?: string | null; + turn_index?: number | null; + turn_count?: number | null; content: string; user_id: string; agent_id: string; @@ -703,6 +708,7 @@ export interface SdkRawPage { total: number; limit: number; offset: number; + unit?: "raw" | "turn"; /** A time range was applied to this page in Core (the SDK list has none). */ time_filtered?: boolean; } diff --git a/tests/test_raw_public_api.py b/tests/test_raw_public_api.py index 3619bfb..4fd3db9 100644 --- a/tests/test_raw_public_api.py +++ b/tests/test_raw_public_api.py @@ -27,6 +27,7 @@ class RawStore: exclude_sources, layers, limit, + status_filter, **_kwargs, ): rows = [] @@ -41,11 +42,18 @@ class RawStore: continue if node.layer not in layers: continue + if node.status not in status_filter: + continue rows.append({"node": node, "score": 0.9}) return rows[:limit] -def raw_node(raw_id: str, session_id: str, source: str) -> MemoryNode: +def raw_node( + raw_id: str, + session_id: str, + source: str, + status: MemoryStatus = MemoryStatus.ACTIVE, +) -> MemoryNode: return MemoryNode( node_id=raw_id, user_id="u1", @@ -55,7 +63,7 @@ def raw_node(raw_id: str, session_id: str, source: str) -> MemoryNode: layer=MemoryLayer.L1_RAW, content=f"content {raw_id}", source_type=SourceType.EXPLICIT, - status=MemoryStatus.ACTIVE, + status=status, is_latest=True, embedding=[0.1, 0.2], ) @@ -102,6 +110,7 @@ async def test_existing_search_contract_can_explicitly_search_raw(): request_id="search-1", t0=0.0, query_embedding=[0.1, 0.2], + statuses=None, ) assert [item["memory_id"] for item in result["memories"]["normal"]] == [ @@ -109,3 +118,33 @@ async def test_existing_search_contract_can_explicitly_search_raw(): ] assert result["memories"]["normal"][0]["layer"] == "l1_raw" assert result["memories"]["normal"][0]["source"] == "claude" + + +@pytest.mark.asyncio +async def test_explicit_raw_search_can_include_shadow_rows(): + client = HyMemoryClient.__new__(HyMemoryClient) + client._vector_store = RawStore( + [raw_node("raw-shadow", "s1", "cursor", MemoryStatus.SHADOW)] + ) + client._embed_service = AsyncMock() + client._embed_service.embed.return_value = [0.1, 0.2] + + result = await HyMemoryClient._search_explicit_raw_layer.__wrapped__( + client, + "content", + user_ids=["u1"], + agent_ids=["a1"], + session_ids=[], + include_sources=[], + exclude_sources=[], + limit=5, + min_score=0.0, + created_after=None, + request_id="search-shadow", + t0=0.0, + query_embedding=[0.1, 0.2], + statuses=["active", "shadow"], + ) + + assert result["memories"]["normal"][0]["memory_id"] == "raw-shadow" + assert result["memories"]["normal"][0]["status"] == "shadow"