fix(sdk): gate near-duplicate demotion with the MMR top-k threshold

near_dup_demote compares each candidate with every earlier
non-duplicate by Jaccard, quadratic when candidates differ. Its pool
grows with limit (about max(3*limit, 30) plus keyword hits), so a
limit-200 page could cost ~180k comparisons. It now runs only when the
caller's limit is below MMR_MAX_TOP_K, like MMR; large pages want every
relevant item, similar ones included. The read reports
near_dup_applied next to mmr_applied.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -14,6 +14,9 @@ Hy Memory and MemHub are versioned and released independently.
   below 20 (`MMR_MAX_TOP_K`) and is skipped for larger limits, where its
   quadratic token-overlap pass is slow and gains little. Previously it ran for
   every limit. `MEMORY_HYBRID_V2_MMR_LAMBDA` still sets the weight.
+  Near-duplicate demotion follows the same gate: it compares each candidate
+  with every earlier non-duplicate, which is quadratic in a pool that grows
+  with `limit`, and large result pages should show similar items anyway.
 - SDK: `search(scene="all")` searches chat, working and coding at the same time
   and merges them with Reciprocal Rank Fusion (k=60) over each scene's own
   ranking, so no scene is crowded out by the other scenes' score scales. Each

hy_memory/pipelines/_retrieval/diversity.py

@@ -171,6 +171,17 @@ def mmr_applies(top_k: int, lambda_: float) -> bool:
     return 0.0 < float(lambda_) < 0.999 and int(top_k) < MMR_MAX_TOP_K
 
 
+def near_dup_applies(top_k: int) -> bool:
+    """本次检索是否走近重复降权：与 MMR 同一个 top-k 门槛。
+
+    near_dup_demote 让每个候选和此前所有"非重复"候选两两算 Jaccard，候选
+    互不相似时是 O(pool^2)；池子随 limit 放大（约 max(3*limit, 30) + 关键词
+    命中），limit 达到 MMR_MAX_TOP_K 时代价同样陡增。大 k 的调用方（如记忆
+    面板翻页）要的是全部相关项，相似项也应照常出现，降权没有收益。
+    """
+    return int(top_k) < MMR_MAX_TOP_K
+
+
 def mmr_select(
     hits: Sequence[Dict[str, Any]],
     limit: int,

hy_memory/pipelines/reader_base.py

@@ -57,6 +57,7 @@ from ._retrieval.ner import (
     memory_matches_time_hints,
 )
 from ._retrieval.diversity import (
+    near_dup_applies,
     near_dup_demote,
     mmr_applies,
     mmr_select,
@@ -1438,6 +1439,8 @@ class BaseReadPipeline(ReadPipeline):
             # MMR 只在调用方的 top-k（final_limit，即 search 的 limit）< MMR_MAX_TOP_K
             # 时启用；池子（LOCAL_RERANK 下是 final_limit 的若干倍）不参与判定。
             MMR_ON = mmr_applies(final_limit, MMR_LAMBDA)
+            # 近重复降权同样是候选两两比较，用同一个 top-k 门槛。
+            NEAR_DUP_ON = near_dup_applies(final_limit)
             vdb_sem_limit = max(final_limit * 3, 30)
             vdb_kw_limit = max(final_limit * 5, 80)
             graph_limit = max(
@@ -2532,7 +2535,7 @@ class BaseReadPipeline(ReadPipeline):
                         df_thr=POOL_DF_THR,
                         penalty=POOL_DF_PENALTY,
                     )
-                if NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0:
+                if NEAR_DUP_ON and NEAR_DUP_THR > 0 and NEAR_DUP_PENALTY > 0:
                     normal_results = near_dup_demote(
                         normal_results,
                         jaccard_thr=NEAR_DUP_THR,
@@ -3553,6 +3556,7 @@ class BaseReadPipeline(ReadPipeline):
                 "local_rerank_enabled": LOCAL_RERANK_ON,
                 "rerank": RERANK_ON,
                 "mmr_applied": MMR_ON,
+                "near_dup_applied": NEAR_DUP_ON,
                 "local_rerank_model": getattr(
                     self.config.recall,
                     "local_rerank_model",

tests/test_search_rerank_mmr_gates.py

@@ -15,7 +15,11 @@ import pytest
 from hy_memory.config import MemoryConfig
 from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus
 from hy_memory.pipelines import reader_base
-from hy_memory.pipelines._retrieval.diversity import MMR_MAX_TOP_K, mmr_applies
+from hy_memory.pipelines._retrieval.diversity import (
+    MMR_MAX_TOP_K,
+    mmr_applies,
+    near_dup_applies,
+)
 from hy_memory.pipelines.base import ReadRequest
 
 DIMS = 32
@@ -126,6 +130,53 @@ def test_mmr_skipped_when_limit_reaches_threshold(config, mmr_spy):
     assert mmr_spy == []
 
 
+@pytest.fixture
+def near_dup_spy(monkeypatch):
+    calls = []
+    real = reader_base.near_dup_demote
+
+    def _spy(hits, **kw):
+        calls.append(len(hits))
+        return real(hits, **kw)
+
+    monkeypatch.setattr(reader_base, "near_dup_demote", _spy)
+    return calls
+
+
+def test_near_dup_shares_the_mmr_threshold():
+    # Pairwise Jaccard against every kept candidate: quadratic in the pool,
+    # which grows with limit. Same gate as MMR.
+    assert near_dup_applies(MMR_MAX_TOP_K - 1)
+    assert not near_dup_applies(MMR_MAX_TOP_K)
+
+
+def test_near_dup_runs_when_limit_below_threshold(config, near_dup_spy):
+    async def _run():
+        reader, store = await _reader_with(config, 12)
+        try:
+            resp = await reader.read(_request(5))
+            assert resp.extra["config"]["near_dup_applied"] is True
+        finally:
+            await store.close()
+
+    asyncio.run(_run())
+    assert len(near_dup_spy) == 1
+
+
+def test_near_dup_skipped_when_limit_reaches_threshold(config, near_dup_spy):
+    async def _run():
+        reader, store = await _reader_with(config, MMR_MAX_TOP_K + 6)
+        try:
+            resp = await reader.read(_request(MMR_MAX_TOP_K))
+            assert resp.success
+            assert resp.extra["config"]["near_dup_applied"] is False
+        finally:
+            await store.close()
+
+    asyncio.run(_run())
+    assert near_dup_spy == []
+
+
 class _FakeReranker:
     model_name = "fake-cross-encoder"
     device = "cpu"