From e540386db1bec5eecf515e7bd2df6106f0e01eb4 Mon Sep 17 00:00:00 2001 From: alvinfei Date: Tue, 27 Oct 2026 15:34:41 +0800 Subject: [PATCH] fix(sdk): gate near-duplicate demotion with the MMR top-k threshold near_dup_demote compares each candidate with every earlier non-duplicate by Jaccard, quadratic when candidates differ. Its pool grows with limit (about max(3*limit, 30) plus keyword hits), so a limit-200 page could cost ~180k comparisons. It now runs only when the caller's limit is below MMR_MAX_TOP_K, like MMR; large pages want every relevant item, similar ones included. The read reports near_dup_applied next to mmr_applied. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 ++ hy_memory/pipelines/_retrieval/diversity.py | 11 +++++ hy_memory/pipelines/reader_base.py | 6 ++- tests/test_search_rerank_mmr_gates.py | 53 ++++++++++++++++++++- 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0240c6a..a3c4081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ Hy Memory and MemHub are versioned and released independently. below 20 (`MMR_MAX_TOP_K`) and is skipped for larger limits, where its quadratic token-overlap pass is slow and gains little. Previously it ran for every limit. `MEMORY_HYBRID_V2_MMR_LAMBDA` still sets the weight. + Near-duplicate demotion follows the same gate: it compares each candidate + with every earlier non-duplicate, which is quadratic in a pool that grows + with `limit`, and large result pages should show similar items anyway. - SDK: `search(scene="all")` searches chat, working and coding at the same time and merges them with Reciprocal Rank Fusion (k=60) over each scene's own ranking, so no scene is crowded out by the other scenes' score scales. Each diff --git a/hy_memory/pipelines/_retrieval/diversity.py b/hy_memory/pipelines/_retrieval/diversity.py index b81ede6..e759b95 100644 --- a/hy_memory/pipelines/_retrieval/diversity.py +++ b/hy_memory/pipelines/_retrieval/diversity.py @@ -171,6 +171,17 @@ def mmr_applies(top_k: int, lambda_: float) -> bool: return 0.0 < float(lambda_) < 0.999 and int(top_k) < MMR_MAX_TOP_K +def near_dup_applies(top_k: int) -> bool: + """本次检索是否走近重复降权：与 MMR 同一个 top-k 门槛。 + + near_dup_demote 让每个候选和此前所有"非重复"候选两两算 Jaccard，候选 + 互不相似时是 O(pool^2)；池子随 limit 放大（约 max(3*limit, 30) + 关键词 + 命中），limit 达到 MMR_MAX_TOP_K 时代价同样陡增。大 k 的调用方（如记忆 + 面板翻页）要的是全部相关项，相似项也应照常出现，降权没有收益。 + """ + return int(top_k) < MMR_MAX_TOP_K + + def mmr_select( hits: Sequence[Dict[str, Any]], limit: int, diff --git a/hy_memory/pipelines/reader_base.py b/hy_memory/pipelines/reader_base.py index afeab3c..bc288d4 100644 --- a/hy_memory/pipelines/reader_base.py +++ b/hy_memory/pipelines/reader_base.py @@ -57,6 +57,7 @@ from ._retrieval.ner import ( memory_matches_time_hints, ) from ._retrieval.diversity import ( + near_dup_applies, near_dup_demote, mmr_applies, mmr_select, @@ -1438,6 +1439,8 @@ class BaseReadPipeline(ReadPipeline): # MMR 只在调用方的 top-k（final_limit，即 search 的 limit）< MMR_MAX_TOP_K # 时启用；池子（LOCAL_RERANK 下是 final_limit 的若干倍）不参与判定。 MMR_ON = mmr_applies(final_limit, MMR_LAMBDA) + # 近重复降权同样是候选两两比较，用同一个 top-k 门槛。 + NEAR_DUP_ON = near_dup_applies(final_limit) vdb_sem_limit = max(final_limit * 3, 30) vdb_kw_limit = max(final_limit * 5, 80) graph_limit = max( @@ -2532,7 +2535,7 @@ class BaseReadPipeline(ReadPipeline): df_thr=POOL_DF_THR, penalty=POOL_DF_PENALTY, ) - if NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0: + if NEAR_DUP_ON and NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0: normal_results = near_dup_demote( normal_results, jaccard_thr=NEAR_DUP_THR, @@ -3553,6 +3556,7 @@ class BaseReadPipeline(ReadPipeline): "local_rerank_enabled": LOCAL_RERANK_ON, "rerank": RERANK_ON, "mmr_applied": MMR_ON, + "near_dup_applied": NEAR_DUP_ON, "local_rerank_model": getattr( self.config.recall, "local_rerank_model", diff --git a/tests/test_search_rerank_mmr_gates.py b/tests/test_search_rerank_mmr_gates.py index aca395e..dbdad01 100644 --- a/tests/test_search_rerank_mmr_gates.py +++ b/tests/test_search_rerank_mmr_gates.py @@ -15,7 +15,11 @@ import pytest from hy_memory.config import MemoryConfig from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus from hy_memory.pipelines import reader_base -from hy_memory.pipelines._retrieval.diversity import MMR_MAX_TOP_K, mmr_applies +from hy_memory.pipelines._retrieval.diversity import ( + MMR_MAX_TOP_K, + mmr_applies, + near_dup_applies, +) from hy_memory.pipelines.base import ReadRequest DIMS = 32 @@ -126,6 +130,53 @@ def test_mmr_skipped_when_limit_reaches_threshold(config, mmr_spy): assert mmr_spy == [] +@pytest.fixture +def near_dup_spy(monkeypatch): + calls = [] + real = reader_base.near_dup_demote + + def _spy(hits, **kw): + calls.append(len(hits)) + return real(hits, **kw) + + monkeypatch.setattr(reader_base, "near_dup_demote", _spy) + return calls + + +def test_near_dup_shares_the_mmr_threshold(): + # Pairwise Jaccard against every kept candidate: quadratic in the pool, + # which grows with limit. Same gate as MMR. + assert near_dup_applies(MMR_MAX_TOP_K - 1) + assert not near_dup_applies(MMR_MAX_TOP_K) + + +def test_near_dup_runs_when_limit_below_threshold(config, near_dup_spy): + async def _run(): + reader, store = await _reader_with(config, 12) + try: + resp = await reader.read(_request(5)) + assert resp.extra["config"]["near_dup_applied"] is True + finally: + await store.close() + + asyncio.run(_run()) + assert len(near_dup_spy) == 1 + + +def test_near_dup_skipped_when_limit_reaches_threshold(config, near_dup_spy): + async def _run(): + reader, store = await _reader_with(config, MMR_MAX_TOP_K + 6) + try: + resp = await reader.read(_request(MMR_MAX_TOP_K)) + assert resp.success + assert resp.extra["config"]["near_dup_applied"] is False + finally: + await store.close() + + asyncio.run(_run()) + assert near_dup_spy == [] + + class _FakeReranker: model_name = "fake-cross-encoder" device = "cpu"