# -*- coding: utf-8 -*- """ SharedRuntime 端到端测试。 覆盖: 1. Solo 模式(不传 runtime)行为完全不变 — backward compat 回归 2. Shared 模式两个 client 可以共存 3. Shared 模式两个 client 共享同一个 cache + 同一个 loop 4. Shared 模式 client.close() 不关 shared cache、不停 shared loop 5. Shared 模式 runtime.close() 关掉 cache 和 loop 6. MetricsCollector 单例在 shared 模式下只 bind 一次(关键 cross-loop bug 防护) 7. 多个 client 真的能并行 add/search 不挂 测试依赖只用 Chroma + SQLite cache(零外部依赖),避免 MySQL/网络。 """ import asyncio import uuid import pytest from hy_memory import HyMemoryClient, SharedRuntime, MemoryConfig from hy_memory.metrics import MetricsCollector def run(coro): loop = asyncio.new_event_loop() try: return loop.run_until_complete(coro) finally: loop.close() def _make_config(tmp_path, *, dims=32, collection_suffix=""): """构造一个不依赖外部服务的 config(chroma + sqlite cache)""" suffix = collection_suffix or uuid.uuid4().hex[:8] return MemoryConfig.from_dict({ "vector_store": { "provider": "chroma", "collection_name": f"test_runtime_{suffix}", "persist_directory": str(tmp_path / f"chroma_{suffix}"), "embedding_dims": dims, }, "cache": { "backend": "sqlite", "sqlite_path": str(tmp_path / "cache.db"), }, "history": {"enable": False}, }) @pytest.fixture(autouse=True) def _reset_metrics_singleton(): """每个 test reset MetricsCollector 单例,避免 cross-test 污染""" MetricsCollector.reset() yield MetricsCollector.reset() @pytest.fixture(autouse=True) def _safe_env(monkeypatch, tmp_path): """避免 test 误连真实 redis/mysql""" monkeypatch.setenv("MEMORY_CACHE_BACKEND", "sqlite") monkeypatch.delenv("REDIS_HOST", raising=False) monkeypatch.delenv("MEMORY_VECTOR_HOST", raising=False) monkeypatch.setenv("MEMORY_DATA_DIR", str(tmp_path / ".hy_memory")) monkeypatch.setenv("MEMORY_LOG_DIR", str(tmp_path / "logs")) yield # ================================================================ # Phase 1: Solo 模式向后兼容 # ================================================================ class TestSoloModeUnchanged: """Solo 模式(不传 runtime)行为必须完全跟改造前一致""" def test_solo_client_init_close(self, tmp_path): cfg = _make_config(tmp_path) client = HyMemoryClient(cfg, mode="lite") try: # solo 模式 owns_runtime=True assert client._owns_runtime is True # 自己有 cache + loop_thread assert client._cache is not None assert client._loop_thread is not None finally: client.close() def test_solo_client_metrics_bound(self, tmp_path): cfg = _make_config(tmp_path) client = HyMemoryClient(cfg, mode="lite") try: mc = MetricsCollector.get() assert mc._cache is client._cache assert mc._background_started is True finally: client.close() # ================================================================ # Phase 2: Shared 模式 — 单 client # ================================================================ class TestSharedRuntimeSingleClient: def test_runtime_create_and_close(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: assert runtime.cache is not None assert runtime.loop_thread is not None assert runtime.metrics is MetricsCollector.get() # cache 已经 bind 到 metrics assert runtime.metrics._cache is runtime.cache assert runtime.metrics._background_started is True finally: runtime.close() def test_client_with_runtime_shares_resources(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: client_cfg = _make_config(tmp_path, collection_suffix="cli1") client = HyMemoryClient(client_cfg, mode="lite", runtime=runtime) try: assert client._owns_runtime is False assert client._loop_thread is runtime.loop_thread assert client._cache is runtime.cache finally: client.close() finally: runtime.close() def test_client_close_does_not_close_shared_cache(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: client_cfg = _make_config(tmp_path, collection_suffix="cli1") client = HyMemoryClient(client_cfg, mode="lite", runtime=runtime) shared_cache = runtime.cache client.close() # client.close 后 runtime.cache 仍可用 assert runtime.cache is shared_cache # runtime loop 仍在跑 assert runtime.loop_thread is not None finally: runtime.close() # ================================================================ # Phase 3: Shared 模式 — 多 client(核心 bug 防护场景) # ================================================================ class TestSharedRuntimeMultiClient: def test_two_clients_share_one_cache(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: cfg_a = _make_config(tmp_path, collection_suffix="a") cfg_b = _make_config(tmp_path, collection_suffix="b") client_a = HyMemoryClient(cfg_a, mode="lite", runtime=runtime) client_b = HyMemoryClient(cfg_b, mode="lite", runtime=runtime) try: assert client_a._cache is client_b._cache assert client_a._loop_thread is client_b._loop_thread assert client_a._cache is runtime.cache finally: client_a.close() client_b.close() finally: runtime.close() def test_metrics_singleton_only_bound_once(self, tmp_path): """关键测试:原 bug 是 bind_cache 被第二个 client 覆盖; shared 模式下应该只 runtime 调一次 bind_cache,client 都不调""" cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) bound_cache_after_runtime = runtime.metrics._cache bg_started_after_runtime = runtime.metrics._background_started try: cfg_a = _make_config(tmp_path, collection_suffix="a") cfg_b = _make_config(tmp_path, collection_suffix="b") client_a = HyMemoryClient(cfg_a, mode="lite", runtime=runtime) client_b = HyMemoryClient(cfg_b, mode="lite", runtime=runtime) try: # 创建俩 client 之后,metrics._cache 仍然是 runtime.cache, # 没被 client 的 bind_cache 覆盖 assert MetricsCollector.get()._cache is bound_cache_after_runtime assert MetricsCollector.get()._cache is runtime.cache # background 状态保持 assert MetricsCollector.get()._background_started is bg_started_after_runtime finally: client_a.close() client_b.close() finally: runtime.close() def test_clients_use_same_loop_thread(self, tmp_path): """关键 cross-loop 防护:所有 client 用同一个 loop""" cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: cfg_a = _make_config(tmp_path, collection_suffix="a") cfg_b = _make_config(tmp_path, collection_suffix="b") client_a = HyMemoryClient(cfg_a, mode="lite", runtime=runtime) client_b = HyMemoryClient(cfg_b, mode="lite", runtime=runtime) try: # 同一个 _LoopThread 实例 assert client_a._loop_thread is client_b._loop_thread # 同一个底层 event loop assert client_a._loop_thread._loop is client_b._loop_thread._loop finally: client_a.close() client_b.close() finally: runtime.close() # ================================================================ # Phase 4: 真正跑 add / 验证 functionally works # ================================================================ class TestSharedRuntimeFunctional: """真的发请求过 add/search 看流程能跑通,不挂在 cross-loop""" def test_two_clients_add_concurrently_no_crossloop(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: cfg_a = _make_config(tmp_path, collection_suffix="a") cfg_b = _make_config(tmp_path, collection_suffix="b") client_a = HyMemoryClient(cfg_a, mode="lite", runtime=runtime) client_b = HyMemoryClient(cfg_b, mode="lite", runtime=runtime) try: # 直接走 runtime 的 cache 上的 store/get_pipeline_log(base 协议层落库) # 关键验证:两个 client 不在自己的 _LoopThread 上读写 cache # 都委托给 runtime 的 loop(避免跨 loop pool 错误) shared = runtime.cache # 通过 client 的 loop_thread.run 写 cache(应该能跑通) async def _w(rid): from hy_memory.utils.pipeline_observability import ( drain_pipeline_log_tasks, ) await shared.store_pipeline_log( request_id=rid, user_id="u", agent_id="a", step="TEST", prompt="p", response="r", ) await drain_pipeline_log_tasks() async def _r(rid): return await shared.get_pipeline_logs(request_id=rid) client_a._loop_thread.run(_w("rid_a")) client_b._loop_thread.run(_w("rid_b")) # 跨 client 读对方写的,证明同一 pool assert len(client_a._loop_thread.run(_r("rid_b"))) == 1 assert len(client_b._loop_thread.run(_r("rid_a"))) == 1 finally: client_a.close() client_b.close() finally: runtime.close() # ================================================================ # Phase 5: aclose / close 语义 # ================================================================ class TestSharedRuntimeClose: def test_runtime_close_idempotent(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) runtime.close() # 再关一次不挂 runtime.close() def test_aclose_async(self, tmp_path): cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) # 用 runtime.loop_thread.run 调 aclose(因为 cache 在那个 loop 上) runtime.close() # ================================================================ # Phase 6: search 专用 loop 与 write loop 隔离 # search 使用独立 loop,避免被 write 队列阻塞。 # ================================================================ class TestSearchLoopIsolation: """search 走独立 _LoopThread,不被 write 队列挤占调度。""" def test_solo_search_loop_is_distinct_thread(self, tmp_path): """solo 模式:search loop 与 write loop 是两个独立的 event loop/线程。""" cfg = _make_config(tmp_path) client = HyMemoryClient(cfg, mode="lite") try: assert client._search_loop_thread is not None assert client._search_loop_thread is not client._loop_thread assert client._search_loop_thread._loop is not client._loop_thread._loop assert client._search_loop_thread._thread is not client._loop_thread._thread assert client._search_loop_thread._thread.is_alive() finally: client.close() def test_shared_search_loop_from_runtime(self, tmp_path): """shared 模式:client 的 search loop 来自 runtime.search_loop_thread, 且与 write loop 隔离;多个 client 共用同一个 search loop。""" cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: assert runtime.search_loop_thread is not None assert runtime.search_loop_thread is not runtime.loop_thread assert runtime.search_loop_thread._loop is not runtime.loop_thread._loop cfg_a = _make_config(tmp_path, collection_suffix="a") cfg_b = _make_config(tmp_path, collection_suffix="b") client_a = HyMemoryClient(cfg_a, mode="lite", runtime=runtime) client_b = HyMemoryClient(cfg_b, mode="lite", runtime=runtime) try: # search loop 来自 runtime,与 write loop 隔离 assert client_a._search_loop_thread is runtime.search_loop_thread assert client_a._search_loop_thread is not client_a._loop_thread # 多 client 共用同一个 search loop assert client_a._search_loop_thread is client_b._search_loop_thread finally: client_a.close() client_b.close() finally: runtime.close() def test_backward_compat_old_runtime_without_search_loop(self, tmp_path): """兼容旧 runtime(无 search_loop_thread 属性):回落到 write loop,不报错。""" cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: # 模拟旧 runtime:抹掉 search 专用 loop old_search = runtime._search_loop_thread runtime._search_loop_thread = None client_cfg = _make_config(tmp_path, collection_suffix="compat") client = HyMemoryClient(client_cfg, mode="lite", runtime=runtime) try: # 回落到 write loop(行为等价旧路径,不隔离但不崩) assert client._search_loop_thread is runtime.loop_thread finally: client.close() runtime._search_loop_thread = old_search finally: runtime.close() def test_search_loop_schedules_while_write_loop_blocked(self, tmp_path): """核心回归:write loop 被一个长时间同步阻塞的协程占满时,search loop 仍能及时调度协程——证明二者互不挤占。 直接在 loop 调度层验证(不拉入 chroma/embed),精确对应本次修复: 改动前 search 和 write 共用一个 loop,提交到该 loop 的 search 协程会被 卡到 write 的阻塞结束;改动后 search 有独立 loop,调度不受影响。 """ import time import asyncio as _aio cfg = _make_config(tmp_path) runtime = run(SharedRuntime.create(cfg)) try: write_loop = runtime.loop_thread._loop search_loop = runtime.search_loop_thread._loop assert write_loop is not search_loop # 前提:确实是两个 loop BLOCK_SEC = 2.0 async def _hog(): # 同步 sleep 占满 write loop(协作式调度下彻底卡住该 loop), # 模拟 EXTRACT/RECONCILE 中夹杂的非 await 同步代码段。 time.sleep(BLOCK_SEC) async def _probe(): # 极轻量探针:代表一次 search 协程被提交后能否被及时调度执行。 return "scheduled" # 1. 占满 write loop hog_fut = _aio.run_coroutine_threadsafe(_hog(), write_loop) time.sleep(0.1) # 确保 hog 真正开始执行、占住 write loop # 2. 此刻 write loop 卡死 → 提交到 search loop 的探针应立刻被调度 t0 = time.perf_counter() probe_fut = _aio.run_coroutine_threadsafe(_probe(), search_loop) assert probe_fut.result(timeout=1.0) == "scheduled" search_latency = time.perf_counter() - t0 # search loop 调度延迟应是「毫秒级」——独立 loop 完全不受 write 阻塞影响。 # 阈值取远小于 BLOCK_SEC 的绝对值:改动前 search/write 共用一个 loop 时, # 探针会被 hog 卡到接近 BLOCK_SEC 才调度,必然超过此阈值 → 回归被捕获。 _SCHED_BUDGET = 0.3 assert search_latency < _SCHED_BUDGET, ( f"search loop 调度耗时 {search_latency:.3f}s(应 < {_SCHED_BUDGET}s)," f"说明未与 write loop(阻塞 {BLOCK_SEC}s)隔离" ) # 3. 对照:同一探针提交到 write loop 会被 hog 卡住,直到阻塞结束 t1 = time.perf_counter() write_probe_fut = _aio.run_coroutine_threadsafe(_probe(), write_loop) assert write_probe_fut.result(timeout=BLOCK_SEC + 1.0) == "scheduled" write_latency = time.perf_counter() - t1 # write loop 上的探针确实被 hog 挤压(证明 hog 生效、对照有意义) assert write_latency > search_latency, ( "write loop 探针应比 search loop 探针慢(被 hog 阻塞)," "否则说明 hog 没有真正占用 write loop,对照无效" ) hog_fut.result(timeout=BLOCK_SEC + 1) # 收尾,避免 teardown 残留 finally: runtime.close()