fix(memhub): Lite search is plain RAG over raw
Lite context search, recall and the Host Bridge return every raw vector hit above the floor, ranked by score, over active and shadow raws. The turn view shows the turns containing the query, or else the hit's closest turn, instead of dropping semantic hits. Pro and ultra raw search stays active-only. A new test runs the raw routes in a child interpreter against the installed hy-memory wheel, and the route-test fake SDK now rejects search kwargs the pinned release does not accept. The lite raw-turn page projects only raws newly put on top instead of re-listing the whole raw layer on every poll. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
58f2fa884a
commit
5fdaf6c280
9 files changed
+525
-82
No files matched your search
+12
-1
@@ -11,6 +11,17 @@ Hy Memory and MemHub are versioned and released independently.
|
||||
schedule and manual evolve. The memory page shows original conversations in
|
||||
lite and advanced memories in pro/ultra. Existing installs remain pro unless
|
||||
the user changes them, and embedding maintenance blocks mode/evolve changes.
|
||||
- MemHub pins `hy-memory==2.0.14`. Lite search is plain RAG over raw: the
|
||||
App's lite search, CLI context search, `/api/v1/recall` and the Host Bridge
|
||||
(`MemHub_recall`, context search) return every vector hit above the floor,
|
||||
ranked by score, over active and shadow raws, so history pro already
|
||||
processed is still found after switching to lite. The turn view no longer
|
||||
drops a hit whose turns do not literally contain the query; it shows the
|
||||
hit's closest turn instead. Pro and ultra raw search stays active-only.
|
||||
Fixes every raw search failing in packaged builds since #316, which passed
|
||||
`statuses` to an SDK (2.0.13) that did not accept it. The lite raw-turn
|
||||
page also stops re-listing the whole raw layer on each poll when a live
|
||||
session only adds new raws.
|
||||
|
||||
## Hy Memory 2.0.13 — 2026-11-02
|
||||
|
||||
@@ -83,7 +94,7 @@ Hy Memory and MemHub are versioned and released independently.
|
||||
|
||||
## MemHub 1.0.6 — 2026-11-02
|
||||
|
||||
Build 40, on `hy-memory==2.0.13`. Linux x64/arm64 Runtime and macOS arm64
|
||||
Build 40, on `hy-memory==2.0.14`. Linux x64/arm64 Runtime and macOS arm64
|
||||
App and Runtime.
|
||||
|
||||
- Interfaces in 11 languages: English, Simplified and Traditional Chinese,
|
||||
|
||||
@@ -270,7 +270,7 @@ from one_memory.onboarding import (
|
||||
OnboardingState,
|
||||
OnboardingStore,
|
||||
)
|
||||
from one_memory.raw_turns import RawDisplayTurn, raw_display_turns
|
||||
from one_memory.raw_turns import RawDisplayTurn, raw_display_turns, raw_prefix_added
|
||||
from one_memory.recall import (
|
||||
GovernedRecallTarget,
|
||||
RecallReceipt,
|
||||
@@ -610,17 +610,10 @@ class SdkRawItem(HySdkRawRecord):
|
||||
turn_count: int | None = Field(default=None, ge=1)
|
||||
|
||||
@classmethod
|
||||
def of(
|
||||
cls, record: HySdkRawRecord, turn: RawDisplayTurn | None = None
|
||||
) -> "SdkRawItem":
|
||||
def of(cls, record: HySdkRawRecord, turn: RawDisplayTurn | None = None) -> "SdkRawItem":
|
||||
content = turn.content if turn is not None else record.content
|
||||
raw_messages = (
|
||||
turn.messages if turn is not None else raw_turn_messages(record.content)
|
||||
)
|
||||
messages = tuple(
|
||||
SdkRawMessage(role=role, content=text)
|
||||
for role, text in raw_messages
|
||||
)
|
||||
raw_messages = turn.messages if turn is not None else raw_turn_messages(record.content)
|
||||
messages = tuple(SdkRawMessage(role=role, content=text) for role, text in raw_messages)
|
||||
roles = {message.role for message in messages}
|
||||
return cls(
|
||||
**record.model_dump(exclude={"content"}),
|
||||
@@ -4532,10 +4525,7 @@ def create_app(
|
||||
)
|
||||
if len(raw_turn_cache_locks) > 64:
|
||||
for stale_key, stale_lock in tuple(raw_turn_cache_locks.items()):
|
||||
if (
|
||||
stale_key not in raw_turn_cache
|
||||
and not stale_lock.locked()
|
||||
):
|
||||
if stale_key not in raw_turn_cache and not stale_lock.locked():
|
||||
raw_turn_cache_locks.pop(stale_key, None)
|
||||
lock = raw_turn_cache_locks.setdefault(cache_key, asyncio.Lock())
|
||||
async with lock:
|
||||
@@ -4546,9 +4536,7 @@ def create_app(
|
||||
offset=0,
|
||||
limit=500,
|
||||
)
|
||||
first_page = HySdkRawPage.from_memory_list(
|
||||
value, limit=500, offset=0
|
||||
)
|
||||
first_page = HySdkRawPage.from_memory_list(value, limit=500, offset=0)
|
||||
signature = tuple(
|
||||
(record.raw_id, record.status) for record in first_page.raw
|
||||
)
|
||||
@@ -4560,6 +4548,29 @@ def create_app(
|
||||
):
|
||||
projected_turns = cached[2]
|
||||
break
|
||||
added = (
|
||||
None
|
||||
if cached is None
|
||||
else raw_prefix_added(cached[1], signature, cached[0], first_page.total)
|
||||
)
|
||||
if cached is not None and added is not None:
|
||||
# A live session only puts new raws on top: project
|
||||
# those and keep the rest, instead of re-listing the
|
||||
# whole raw layer on every poll.
|
||||
projected_turns = (
|
||||
tuple(
|
||||
(record, turn)
|
||||
for record in first_page.raw[:added]
|
||||
for turn in reversed(raw_display_turns(record))
|
||||
)
|
||||
+ cached[2]
|
||||
)
|
||||
raw_turn_cache[cache_key] = (
|
||||
first_page.total,
|
||||
signature,
|
||||
projected_turns,
|
||||
)
|
||||
break
|
||||
|
||||
records_list = list(first_page.raw)
|
||||
raw_offset = len(first_page.raw)
|
||||
@@ -4585,16 +4596,11 @@ def create_app(
|
||||
offset=0,
|
||||
limit=500,
|
||||
)
|
||||
verify = HySdkRawPage.from_memory_list(
|
||||
verify_value, limit=500, offset=0
|
||||
)
|
||||
verify = HySdkRawPage.from_memory_list(verify_value, limit=500, offset=0)
|
||||
verify_signature = tuple(
|
||||
(record.raw_id, record.status) for record in verify.raw
|
||||
)
|
||||
if (
|
||||
verify.total != first_page.total
|
||||
or verify_signature != signature
|
||||
):
|
||||
if verify.total != first_page.total or verify_signature != signature:
|
||||
continue
|
||||
projected_turns = tuple(
|
||||
(record, turn)
|
||||
@@ -6372,7 +6378,7 @@ async def _search_sdk_raw(
|
||||
`filters` are a caller's pass-through options: its floor and row filters
|
||||
apply, and its `include_sources` only narrow `sources`, never widen them.
|
||||
"""
|
||||
from one_memory.context_routes import raw_lane_overrides
|
||||
from one_memory.context_routes import raw_lane_overrides, raw_lane_statuses
|
||||
|
||||
include = sorted(sources) if sources else None
|
||||
if filters is not None and filters.include_sources is not None:
|
||||
@@ -6399,6 +6405,8 @@ async def _search_sdk_raw(
|
||||
"scene": "chat",
|
||||
"reader": HY_MEMORY_READER,
|
||||
"layers": ["l1_raw"],
|
||||
# Lite recall is plain RAG over every raw, shadow included.
|
||||
**raw_lane_statuses(lite=app.state.memory_mode() == "lite"),
|
||||
**(raw_lane_overrides(filters) if filters is not None else {}),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -30,7 +30,7 @@ from one_memory.hy_sdk_types import (
|
||||
parse_sdk_search,
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.raw_turns import raw_display_turns
|
||||
from one_memory.raw_turns import raw_display_turns, turns_for_hit
|
||||
from one_memory.sdk_search_options import SdkSearchFilters, SearchLane
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
from one_memory.user_scope import owns
|
||||
@@ -71,6 +71,18 @@ def raw_lane_overrides(value: SdkSearchFilters) -> dict[str, object]:
|
||||
return found
|
||||
|
||||
|
||||
def raw_lane_statuses(*, lite: bool) -> dict[str, list[str]]:
|
||||
"""Which raw statuses the raw lane searches.
|
||||
|
||||
Lite extracts nothing, so its search is plain RAG over every raw,
|
||||
including the shadow raws pro already processed before the switch to
|
||||
lite. Pro and ultra keep the SDK default (active only): a shadow raw is
|
||||
already represented by the memories extracted from it, and returning it
|
||||
again would repeat that content in the caller's context budget.
|
||||
"""
|
||||
return {"statuses": ["active", "shadow"]} if lite else {}
|
||||
|
||||
|
||||
def hit_namespace(hit: SemanticMemoryHit) -> dict[str, str]:
|
||||
"""The `get` namespace of a hit: nothing for chat, kind and variant otherwise."""
|
||||
if hit.kind in {"experience", "fact"} and hit.variant is not None:
|
||||
@@ -121,9 +133,9 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
sources = raw_lane_sources(authority.raw_sources, value.include_sources)
|
||||
if sources is None:
|
||||
return ()
|
||||
candidate_limit = (
|
||||
min(value.limit * 10, 200) if value.raw_unit == "turn" else value.limit
|
||||
)
|
||||
# Plain RAG: every vector hit over the floor is an answer, ranked
|
||||
# by its score. Each hit yields at least one item, so the SDK
|
||||
# needs to return no more hits than the caller's limit.
|
||||
raw = await asyncio.to_thread(
|
||||
client.search,
|
||||
value.query,
|
||||
@@ -131,7 +143,7 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
"user_ids": [value.user_id],
|
||||
"agent_ids": [],
|
||||
"include_sources": sources,
|
||||
"limit": candidate_limit,
|
||||
"limit": value.limit,
|
||||
"min_score": 0.0,
|
||||
"schema_min_score": 0.0,
|
||||
"schema_limit": 0,
|
||||
@@ -139,59 +151,60 @@ def sdk_context_router(dependencies: Sequence[Depends]) -> APIRouter:
|
||||
"scene": "chat",
|
||||
"reader": HY_MEMORY_READER,
|
||||
"layers": ["l1_raw"],
|
||||
"statuses": ["active", "shadow"],
|
||||
**raw_lane_statuses(lite=lite),
|
||||
**raw_lane_overrides(value),
|
||||
},
|
||||
)
|
||||
hits = tuple(
|
||||
hit for hit in parse_sdk_raw_search(raw) if value.passes_floor("raw", hit.score)
|
||||
)
|
||||
items: list[SdkRawContextItem] = []
|
||||
# One item per hit first, so a hit is never crowded out by another
|
||||
# hit's sibling turns; the siblings fill whatever room is left.
|
||||
primary: list[SdkRawContextItem] = []
|
||||
siblings: list[SdkRawContextItem] = []
|
||||
for hit in hits:
|
||||
memory_at = (
|
||||
datetime.fromtimestamp(hit.memory_at, tz=UTC)
|
||||
if hit.memory_at is not None
|
||||
else None
|
||||
)
|
||||
turns = raw_display_turns(hit) if value.raw_unit == "turn" else ()
|
||||
if turns:
|
||||
query = value.query.casefold()
|
||||
lexical = tuple(turn for turn in turns if query in turn.content.casefold())
|
||||
# A batch-level embedding cannot identify which contained
|
||||
# turn matched semantically. Return only turns grounded by
|
||||
# the query text instead of assigning every sibling the
|
||||
# batch score and showing unrelated conversations.
|
||||
for turn in lexical:
|
||||
items.append(
|
||||
SdkRawContextItem(
|
||||
raw_id=hit.raw_id,
|
||||
turn_id=turn.turn_id,
|
||||
turn_index=turn.turn_index,
|
||||
turn_count=turn.turn_count,
|
||||
content=turn.content,
|
||||
source=hit.source,
|
||||
session_id=hit.session_id,
|
||||
memory_at=(
|
||||
datetime.fromtimestamp(hit.memory_at, tz=UTC)
|
||||
if hit.memory_at is not None
|
||||
else None
|
||||
),
|
||||
score=hit.score,
|
||||
)
|
||||
)
|
||||
else:
|
||||
items.append(
|
||||
if not turns:
|
||||
primary.append(
|
||||
SdkRawContextItem(
|
||||
raw_id=hit.raw_id,
|
||||
content=hit.content,
|
||||
source=hit.source,
|
||||
session_id=hit.session_id,
|
||||
memory_at=(
|
||||
datetime.fromtimestamp(hit.memory_at, tz=UTC)
|
||||
if hit.memory_at is not None
|
||||
else None
|
||||
),
|
||||
memory_at=memory_at,
|
||||
score=hit.score,
|
||||
)
|
||||
)
|
||||
if len(items) >= value.limit:
|
||||
break
|
||||
return tuple(items[: value.limit])
|
||||
continue
|
||||
# The embedding is the batch's: the hit is shown by the turns
|
||||
# that contain the query, or else by its closest turn, each
|
||||
# with the hit's score.
|
||||
shown = [
|
||||
SdkRawContextItem(
|
||||
raw_id=hit.raw_id,
|
||||
turn_id=turn.turn_id,
|
||||
turn_index=turn.turn_index,
|
||||
turn_count=turn.turn_count,
|
||||
content=turn.content,
|
||||
source=hit.source,
|
||||
session_id=hit.session_id,
|
||||
memory_at=memory_at,
|
||||
score=hit.score,
|
||||
)
|
||||
for turn in turns_for_hit(turns, value.query)
|
||||
]
|
||||
primary.append(shown[0])
|
||||
siblings.extend(shown[1:])
|
||||
chosen = primary[: value.limit]
|
||||
chosen.extend(siblings[: value.limit - len(chosen)])
|
||||
# Stable: a hit's turns stay together, in hit-score order.
|
||||
chosen.sort(key=lambda item: item.score, reverse=True)
|
||||
return tuple(chosen)
|
||||
|
||||
async def memory_lane() -> tuple[SdkMemoryContextItem, ...]:
|
||||
# lite extracts nothing: its search answers raw turns, which the
|
||||
|
||||
@@ -9,6 +9,7 @@ identity while retaining the parent raw id for traceability.
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from one_memory.hy_sdk_types import HySdkRawRecord, raw_turn_messages
|
||||
@@ -57,6 +58,66 @@ def raw_display_turns(record: HySdkRawRecord) -> tuple[RawDisplayTurn, ...]:
|
||||
|
||||
|
||||
def _turn_content(messages: list[tuple[str, str]]) -> str:
|
||||
return "\n".join(
|
||||
f"[{role}]: {content}" if role else content for role, content in messages
|
||||
)
|
||||
return "\n".join(f"[{role}]: {content}" if role else content for role, content in messages)
|
||||
|
||||
|
||||
def raw_prefix_added(
|
||||
cached: tuple[tuple[str, str | None], ...],
|
||||
current: tuple[tuple[str, str | None], ...],
|
||||
cached_total: int,
|
||||
current_total: int,
|
||||
) -> int | None:
|
||||
"""How many raws were only put on top of a cached newest-first listing.
|
||||
|
||||
`cached` and `current` are the (raw id, status) of the first page of each
|
||||
listing. The answer is a count when the current page is the cached one
|
||||
with new raws before it and nothing else changed; None when anything else
|
||||
happened (a delete, a status change, a raw dated into the middle), which
|
||||
needs a full re-listing.
|
||||
"""
|
||||
added = current_total - cached_total
|
||||
if added <= 0 or added > len(current):
|
||||
return None
|
||||
kept = current[added:]
|
||||
if not kept and cached_total:
|
||||
return None
|
||||
if kept != cached[: len(kept)]:
|
||||
return None
|
||||
return added
|
||||
|
||||
|
||||
_WORD = re.compile(r"\w+")
|
||||
|
||||
|
||||
def _query_terms(query: str) -> frozenset[str]:
|
||||
"""Words of the query, plus character bigrams of unspaced (CJK) words."""
|
||||
terms: set[str] = set()
|
||||
for word in _WORD.findall(query.casefold()):
|
||||
terms.add(word)
|
||||
if not word.isascii() and len(word) > 1:
|
||||
terms.update(word[index : index + 2] for index in range(len(word) - 1))
|
||||
return frozenset(terms)
|
||||
|
||||
|
||||
def turns_for_hit(turns: tuple[RawDisplayTurn, ...], query: str) -> tuple[RawDisplayTurn, ...]:
|
||||
"""The turns one semantic raw hit shows, never none while it has turns.
|
||||
|
||||
The SDK embeds a whole raw batch, so a vector hit cannot say which of its
|
||||
turns matched. Every turn containing the query text is shown; when none
|
||||
does, the turn sharing the most query terms (the earliest on a tie) stands
|
||||
for the hit. A semantic hit is never dropped for lacking the literal query.
|
||||
"""
|
||||
if not turns:
|
||||
return ()
|
||||
needle = query.casefold().strip()
|
||||
if needle:
|
||||
lexical = tuple(turn for turn in turns if needle in turn.content.casefold())
|
||||
if lexical:
|
||||
return lexical
|
||||
terms = _query_terms(query)
|
||||
|
||||
def overlap(turn: RawDisplayTurn) -> tuple[int, int]:
|
||||
content = turn.content.casefold()
|
||||
return (sum(1 for term in terms if term in content), -turn.turn_index)
|
||||
|
||||
return (max(turns, key=overlap),)
|
||||
@@ -213,6 +213,7 @@
|
||||
| `test_onboarding_profile.py` | `integration` | R-09:首次安装引导把画像录下来,不问用户一句话。 |
|
||||
| `test_oversized_file_guard.py` | `contract` | One pass over a source file is bounded by a byte budget, not skipped. |
|
||||
| `test_parent_watchdog.py` | `integration` | A Core that outlives its launcher makes the next launch impossible. |
|
||||
| `test_pinned_sdk_raw_search.py` | `integration` | MemHub's raw search against the hy-memory wheel it pins, not the repo source. |
|
||||
| `test_policy_api_wiring.py` | `integration` | 覆盖 global model permission off does not backfill、backfill puts history on write jobs without scheduling memory tasks、source pause skips history until the source is reenabled 场景。 |
|
||||
| `test_policy_production_entry.py` | `contract` | 覆盖 production serve passes only supported app options、maintenance serve does not own the runtime 场景。 |
|
||||
| `test_precipitation_concurrency_setting.py` | `contract` | Rule 2/9: precipitation concurrency is configurable, not hardcoded to 1. |
|
||||
@@ -321,4 +322,4 @@
|
||||
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
|
||||
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
|
||||
|
||||
共 316 个测试文件。
|
||||
共 317 个测试文件。
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
|
||||
EXPECTED_TEST_FILE_COUNT = 316
|
||||
EXPECTED_TEST_FILES_SHA256 = "011482974a26e9d43ba03bb88cbd168e645175beacd96c02ff642bc17dac49da"
|
||||
EXPECTED_TEST_FILE_COUNT = 317
|
||||
EXPECTED_TEST_FILES_SHA256 = "b1370c1ea537ecb3f9837d0a6354e27c8ebe7c588a5b2927044e6ac27532fd26"
|
||||
|
||||
# Pure transformations, validation models, parity checks and state machines.
|
||||
# This list is intentionally small: uncertain files default to contract.
|
||||
@@ -93,6 +93,7 @@ INTEGRATION_FILES = frozenset(
|
||||
"test_model_profile_context_reset.py",
|
||||
"test_onboarding_profile.py",
|
||||
"test_parent_watchdog.py",
|
||||
"test_pinned_sdk_raw_search.py",
|
||||
"test_policy_api_wiring.py",
|
||||
"test_recall_scope_control.py",
|
||||
"test_retrieval_lifespan.py",
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""MemHub's raw search against the hy-memory wheel it pins, not the repo source.
|
||||
|
||||
The suite's pythonpath puts the repository root first, so `import hy_memory`
|
||||
in a test reads the repo's `hy_memory/`, which can be ahead of the pinned
|
||||
release. Search kwargs the release does not accept then pass every test and
|
||||
fail every raw search in a packaged build (a `statuses` argument did exactly
|
||||
that against 2.0.13). These tests run in a child interpreter started with
|
||||
`-I`, so `hy_memory` resolves to the wheel installed in the Core venv, and
|
||||
drive the real routes over a real lite SDK client with a deterministic local
|
||||
embedder (no model, no network, no ONNX assets).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.test_sdk_api_surface import SDK_SEARCH_PARAMETERS
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
_CHILD = textwrap.dedent(
|
||||
r'''
|
||||
import asyncio
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
|
||||
import hy_memory
|
||||
from hy_memory import HyMemoryClient
|
||||
|
||||
from one_memory.hy_runtime_config import HY_MEMORY_VERSION
|
||||
|
||||
home = Path(sys.argv[1])
|
||||
wheel = Path(hy_memory.__file__).resolve()
|
||||
assert "site-packages" in wheel.parts, f"not the installed wheel: {wheel}"
|
||||
assert version("hy-memory") == HY_MEMORY_VERSION, version("hy-memory")
|
||||
parameters = sorted(
|
||||
name
|
||||
for name, parameter in inspect.signature(HyMemoryClient.search).parameters.items()
|
||||
if name not in {"self", "query"} and parameter.kind is not inspect.Parameter.VAR_KEYWORD
|
||||
)
|
||||
var_keyword = any(
|
||||
parameter.kind is inspect.Parameter.VAR_KEYWORD
|
||||
for parameter in inspect.signature(HyMemoryClient.search).parameters.values()
|
||||
)
|
||||
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from pydantic import SecretStr
|
||||
|
||||
import one_memory.api as api
|
||||
from one_memory.api import create_app
|
||||
from one_memory.config import Settings
|
||||
from one_memory.embedding_assets import BUILTIN_EMBEDDING_DIMENSIONS
|
||||
from one_memory.hy_sdk_runtime import (
|
||||
LocalOnnxEmbeddingService,
|
||||
SdkClientHandle,
|
||||
builtin_retrieval_snapshot,
|
||||
)
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
WORD = re.compile(r"\w+")
|
||||
|
||||
class HashedEmbedder:
|
||||
"""Bag of words and CJK bigrams, hashed: texts sharing words are near."""
|
||||
|
||||
dimensions = BUILTIN_EMBEDDING_DIMENSIONS
|
||||
|
||||
def encode(self, texts):
|
||||
vectors = []
|
||||
for text in texts:
|
||||
vector = [0.0] * self.dimensions
|
||||
for word in WORD.findall(text.casefold()):
|
||||
grams = [word]
|
||||
if not word.isascii():
|
||||
grams += [word[i : i + 2] for i in range(len(word) - 1)]
|
||||
for gram in grams:
|
||||
digest = hashlib.sha256(gram.encode()).digest()
|
||||
vector[int.from_bytes(digest[:4], "big") % self.dimensions] += 1.0
|
||||
norm = math.sqrt(sum(value * value for value in vector)) or 1.0
|
||||
vectors.append([value / norm for value in vector])
|
||||
return vectors
|
||||
|
||||
LocalOnnxEmbeddingService._embedder = HashedEmbedder()
|
||||
|
||||
async def lite_has_no_model(_app):
|
||||
return False
|
||||
|
||||
api._recall_model_ready = lite_has_no_model
|
||||
|
||||
async def main():
|
||||
settings = Settings(
|
||||
data_dir=home / "data",
|
||||
source_home=home,
|
||||
api_token=SecretStr("pinned-sdk"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
)
|
||||
app = create_app(settings, secret_store=InMemorySecretStore())
|
||||
app.state.memory_mode = lambda: "lite"
|
||||
handle = SdkClientHandle(
|
||||
builtin_retrieval_snapshot(),
|
||||
api_key=None,
|
||||
data_dir=home / "data",
|
||||
mode="lite",
|
||||
)
|
||||
result = {"parameters": parameters, "var_keyword": var_keyword}
|
||||
async with app.router.lifespan_context(app):
|
||||
app.state.sdk_client_handle = handle
|
||||
client = await handle.client()
|
||||
for session, messages in (
|
||||
(
|
||||
"s-hike",
|
||||
[
|
||||
("user", "我叫小林"),
|
||||
("assistant", "你好小林"),
|
||||
("user", "I love hiking mountain trails at dawn"),
|
||||
("assistant", "Noted, early hikes."),
|
||||
],
|
||||
),
|
||||
(
|
||||
"s-tea",
|
||||
[
|
||||
("user", "my favourite drink is green tea"),
|
||||
("assistant", "Green tea, noted."),
|
||||
],
|
||||
),
|
||||
):
|
||||
written = client.add(
|
||||
[{"role": role, "content": content} for role, content in messages],
|
||||
user_id="local",
|
||||
source="hermes",
|
||||
session_id=session,
|
||||
)
|
||||
assert written.get("success", True), written
|
||||
listed = client.list_memories(user_id="local", layers=["l1_raw"], limit=10)
|
||||
rows = listed.get("vdb", listed)["memories"]
|
||||
tea = next(row for row in rows if "green tea" in row["content"])
|
||||
# What pro leaves behind: the raw it already processed is shadow.
|
||||
client._loop_thread.run(
|
||||
client._vector_store.update_payload(tea["memory_id"], {"status": "shadow"})
|
||||
)
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=app),
|
||||
base_url="http://test",
|
||||
headers={"Authorization": "Bearer pinned-sdk"},
|
||||
) as http:
|
||||
for name, body in (
|
||||
("hike_turn", {"query": "hiking trails mountain", "raw_unit": "turn"}),
|
||||
("tea_turn", {"query": "favourite green tea drink", "raw_unit": "turn"}),
|
||||
("tea_raw", {"query": "favourite green tea drink"}),
|
||||
):
|
||||
response = await http.post(
|
||||
"/api/v2/context/search", json={"kinds": ["raw"], "limit": 5, **body}
|
||||
)
|
||||
result[name] = {"status": response.status_code, "body": response.json()}
|
||||
response = await http.post(
|
||||
"/api/v1/recall", json={"query": "favourite green tea drink"}
|
||||
)
|
||||
result["recall"] = {"status": response.status_code, "body": response.json()}
|
||||
await handle.close()
|
||||
print("RESULT " + json.dumps(result, ensure_ascii=False))
|
||||
|
||||
asyncio.run(main())
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pinned(tmp_path_factory) -> dict:
|
||||
root = tmp_path_factory.mktemp("pinned-sdk")
|
||||
script = root / "scripts" / "child.py"
|
||||
script.parent.mkdir()
|
||||
script.write_text(_CHILD, encoding="utf-8")
|
||||
home = root / "home"
|
||||
home.mkdir()
|
||||
environment = {
|
||||
**os.environ,
|
||||
"ANONYMIZED_TELEMETRY": "False",
|
||||
"ONE_MEMORY_DATA_DIR": str(home / "data"),
|
||||
}
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-I", str(script), str(home)],
|
||||
cwd=home,
|
||||
env=environment,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=240,
|
||||
check=False,
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr[-4000:]
|
||||
[line] = [line for line in completed.stdout.splitlines() if line.startswith("RESULT ")]
|
||||
return json.loads(line.removeprefix("RESULT "))
|
||||
|
||||
|
||||
def test_the_fake_sdk_accepts_exactly_the_pinned_search_parameters(pinned) -> None:
|
||||
# The route tests' stand-in rejects what the pinned SDK rejects.
|
||||
assert pinned["var_keyword"] is False
|
||||
assert sorted(SDK_SEARCH_PARAMETERS) == pinned["parameters"]
|
||||
|
||||
|
||||
def test_lite_turn_search_returns_semantic_hits_without_the_literal_query(pinned) -> None:
|
||||
hike = pinned["hike_turn"]
|
||||
assert hike["status"] == 200, hike
|
||||
items = hike["body"]["items"]
|
||||
assert items, hike["body"]
|
||||
top = items[0]
|
||||
# No turn contains "hiking trails mountain"; the hit still comes back,
|
||||
# shown by its closest turn and carrying the hit's score.
|
||||
assert "hiking mountain trails" in top["content"]
|
||||
assert "小林" not in top["content"]
|
||||
assert top["turn_index"] == 1 and top["turn_count"] == 2
|
||||
assert [item["score"] for item in items] == sorted(
|
||||
(item["score"] for item in items), reverse=True
|
||||
)
|
||||
|
||||
|
||||
def test_lite_search_and_recall_include_shadow_raws(pinned) -> None:
|
||||
for name in ("tea_turn", "tea_raw"):
|
||||
result = pinned[name]
|
||||
assert result["status"] == 200, result
|
||||
assert any("green tea" in item["content"] for item in result["body"]["items"]), result
|
||||
recall = pinned["recall"]
|
||||
assert recall["status"] == 200, recall
|
||||
assert recall["body"]["mode"] == "raw"
|
||||
assert any("green tea" in hit["content"] for hit in recall["body"]["hits"]), recall
|
||||
@@ -91,6 +91,40 @@ def _chat(user_id: str, memory_id: str, content: str, layer: str, source: str) -
|
||||
}
|
||||
|
||||
|
||||
#: The keyword parameters of the pinned `HyMemoryClient.search`, which takes
|
||||
#: no **kwargs. `test_pinned_sdk_raw_search` checks this set against the
|
||||
#: installed wheel, so the stand-in below rejects what the release rejects.
|
||||
SDK_SEARCH_PARAMETERS = frozenset(
|
||||
{
|
||||
"queries",
|
||||
"scene",
|
||||
"search_mode",
|
||||
"user_ids",
|
||||
"agent_ids",
|
||||
"session_ids",
|
||||
"include_sources",
|
||||
"exclude_sources",
|
||||
"limit",
|
||||
"min_score",
|
||||
"working_fact_min_score",
|
||||
"working_experience_min_score",
|
||||
"schema_min_score",
|
||||
"schema_limit",
|
||||
"intention_limit",
|
||||
"created_after",
|
||||
"reader",
|
||||
"workspace_id",
|
||||
"branch",
|
||||
"request_id",
|
||||
"layers",
|
||||
"statuses",
|
||||
"embed_override",
|
||||
"query_embedding",
|
||||
"rerank",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class RecordingSdk:
|
||||
"""The public client surface, recording every call's keyword arguments."""
|
||||
|
||||
@@ -109,6 +143,10 @@ class RecordingSdk:
|
||||
# The SDK answers rows below a caller's floor in some lanes (a schema
|
||||
# floor, a fused rank score), so Core must filter what it returns.
|
||||
def search(self, query, **kwargs):
|
||||
unknown = set(kwargs) - SDK_SEARCH_PARAMETERS
|
||||
if unknown:
|
||||
# The pinned `HyMemoryClient.search` takes no **kwargs.
|
||||
raise TypeError(f"search() got unexpected keyword arguments {sorted(unknown)}")
|
||||
self._record("search", {"query": query, **kwargs})
|
||||
[user_id] = kwargs["user_ids"]
|
||||
if kwargs.get("layers") == ["l1_raw"]:
|
||||
@@ -274,6 +312,10 @@ async def _model_ready(_app) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
async def _model_not_ready(_app) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def run_with_sdk(tmp_path, monkeypatch, scenario) -> None:
|
||||
async def main() -> None:
|
||||
app = application(tmp_path)
|
||||
@@ -365,14 +407,56 @@ def test_raw_context_search_can_return_turns_instead_of_storage_batches(
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
raw_one = [
|
||||
item for item in response.json()["items"] if item["raw_id"] == "raw-1"
|
||||
]
|
||||
raw_one = [item for item in response.json()["items"] if item["raw_id"] == "raw-1"]
|
||||
assert len(raw_one) == 1
|
||||
assert raw_one[0]["turn_index"] == 1
|
||||
assert "我喜欢绿茶" in raw_one[0]["content"]
|
||||
assert "我叫费天翔" not in raw_one[0]["content"]
|
||||
# pro/ultra: a shadow raw is already in its extracted memories.
|
||||
assert "statuses" not in sdk.last("search")
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
|
||||
@pytest.mark.fresh_sdk
|
||||
def test_lite_raw_search_is_plain_rag_over_active_and_shadow_raws(tmp_path, monkeypatch) -> None:
|
||||
async def scenario(app, sdk, client):
|
||||
app.state.memory_mode = lambda: "lite"
|
||||
response = await client.post(
|
||||
"/api/v2/context/search",
|
||||
json={"query": "喜欢喝绿茶的人", "kinds": ["raw"], "raw_unit": "turn", "limit": 5},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
sent = sdk.last("search")
|
||||
assert sent["statuses"] == ["active", "shadow"] and sent["limit"] == 5
|
||||
items = response.json()["items"]
|
||||
# No turn contains the query, yet every vector hit is returned, in
|
||||
# score order: raw-1 by its closest turn, raw-2 by its only one.
|
||||
assert [(item["raw_id"], item["turn_index"]) for item in items] == [
|
||||
("raw-1", 1),
|
||||
("raw-2", 0),
|
||||
]
|
||||
assert [item["score"] for item in items] == [0.97, 0.3]
|
||||
assert "我喜欢绿茶" in items[0]["content"]
|
||||
|
||||
floored = await client.post(
|
||||
"/api/v2/context/search",
|
||||
json={
|
||||
"query": "喜欢喝绿茶的人",
|
||||
"kinds": ["raw"],
|
||||
"raw_unit": "turn",
|
||||
"min_score": 0.5,
|
||||
},
|
||||
)
|
||||
assert [item["raw_id"] for item in floored.json()["items"]] == ["raw-1"]
|
||||
|
||||
# Lite has no model, so recall answers from the raw layer.
|
||||
monkeypatch.setattr("one_memory.api._recall_model_ready", _model_not_ready)
|
||||
recall = await client.post("/api/v1/recall", json={"query": "喜欢喝绿茶的人"})
|
||||
assert recall.status_code == 200, recall.text
|
||||
assert recall.json()["mode"] == "raw"
|
||||
assert sdk.last("search")["statuses"] == ["active", "shadow"]
|
||||
assert [hit["memory_id"] for hit in recall.json()["hits"]][:1] == ["raw-1"]
|
||||
|
||||
run_with_sdk(tmp_path, monkeypatch, scenario)
|
||||
|
||||
@@ -632,12 +716,39 @@ def test_raw_lists_active_and_shadow_turns_with_roles_and_hashes(tmp_path, monke
|
||||
await client.get("/api/v2/raw", params={"unit": "turn"})
|
||||
assert len(sdk.all("list_memories")) == calls_before_refresh + 1
|
||||
|
||||
# A live session puts a new raw on top: one listing call, not a rescan.
|
||||
newest = {
|
||||
**RAW_ROWS["local"][1],
|
||||
"memory_id": "raw-new",
|
||||
"content": "[user]: 刚刚说的",
|
||||
"status": "active",
|
||||
}
|
||||
original = RAW_ROWS["local"]
|
||||
monkeypatch.setitem(RAW_ROWS, "local", [newest, *original])
|
||||
calls_before_append = len(sdk.all("list_memories"))
|
||||
appended = (await client.get("/api/v2/raw", params={"unit": "turn"})).json()
|
||||
assert len(sdk.all("list_memories")) == calls_before_append + 1
|
||||
assert appended["total"] == 4
|
||||
assert [item["raw_id"] for item in appended["items"]][:1] == ["raw-new"]
|
||||
assert [item["turn_id"] for item in appended["items"][1:]] == [
|
||||
item["turn_id"] for item in turns["items"]
|
||||
]
|
||||
# Anything else (here a raw going away) lists the raw layer again.
|
||||
monkeypatch.setitem(RAW_ROWS, "local", [newest, original[1]])
|
||||
calls_before_delete = len(sdk.all("list_memories"))
|
||||
shrunk = (await client.get("/api/v2/raw", params={"unit": "turn"})).json()
|
||||
assert len(sdk.all("list_memories")) > calls_before_delete + 1
|
||||
assert shrunk["total"] == 2
|
||||
monkeypatch.setitem(RAW_ROWS, "local", original)
|
||||
calls_before_restore = len(sdk.all("list_memories"))
|
||||
restored = (await client.get("/api/v2/raw", params={"unit": "turn"})).json()
|
||||
assert len(sdk.all("list_memories")) > calls_before_restore + 1
|
||||
assert restored["total"] == 3
|
||||
|
||||
ascending = (
|
||||
await client.get("/api/v2/raw", params={"unit": "turn", "order": "asc"})
|
||||
).json()
|
||||
ascending_first = [
|
||||
item for item in ascending["items"] if item["raw_id"] == "raw-1"
|
||||
]
|
||||
ascending_first = [item for item in ascending["items"] if item["raw_id"] == "raw-1"]
|
||||
assert [item["turn_index"] for item in ascending_first] == [0, 1]
|
||||
|
||||
await client.get(
|
||||
|
||||
@@ -12,7 +12,7 @@ memory route. `user_id` is optional (default `local`) and must match
|
||||
|
||||
| Mode | SDK client | What happens to a conversation | What search reads |
|
||||
|---|---|---|---|
|
||||
| `lite` | `mode="lite"` | Stored in embedded L1 raw batches. No model call; no generation model needs to be configured. The product page projects each batch back into individual conversation turns. | Raw turns only (the SDK forces `L1_RAW`). |
|
||||
| `lite` | `mode="lite"` | Stored in embedded L1 raw batches. No model call; no generation model needs to be configured. The product page projects each batch back into individual conversation turns. | Plain vector RAG over raw (the SDK forces `L1_RAW`), active and shadow, so history pro already processed stays findable. Every hit above the floor is returned by score; the turn view shows the turns containing the query, or else the hit's closest turn. |
|
||||
| `pro` (default) | `mode="pro"` | Stored as raw, then a model extracts memories (System 1). The raw becomes SHADOW once extraction succeeds. | Extracted memories (facts, profile, intentions). |
|
||||
| `ultra` | `mode="ultra"` | Same as pro, and new facts are also projected into the knowledge graph. Every night, evolve runs System 2 for every user. | Same as pro, plus the graph-backed lanes below. |
|
||||
|
||||
|
||||
Reference in new issue
Block a user