diff --git a/memhub/apps/core/src/one_memory/call_event_references.py b/memhub/apps/core/src/one_memory/call_event_references.py index b48cbde..d3f6979 100644 --- a/memhub/apps/core/src/one_memory/call_event_references.py +++ b/memhub/apps/core/src/one_memory/call_event_references.py @@ -76,8 +76,7 @@ def _memory(db: sqlite3.Connection, reference: CallReference) -> SourceRecordRef if row["status"] == "deleted" or row["deleted_at"] is not None: return _hidden(reference) - # Same local revocation/ownership predicate as SourceReadAccess.memories, - # evaluated BEFORE freshness (which can read support text), in this snapshot. + # Same local revocation/ownership predicate as SourceReadAccess.memories. # Its public methods open their own connections, so cannot be used here. access = db.execute( "SELECT EXISTS (SELECT 1 FROM governed_memory_evidence " @@ -121,8 +120,8 @@ def resolve_call_references(database_path: Path, event: CallEvent) -> list[Sourc Local-owner only: event host attribution is NOT authorization. Explicit local source revocations deny access; absent grants retain legacy local semantics. - A memory needs at least one accessible active citation. No proof preserves - shared freshness compatibility, not a claim of verified historical freshness. + A memory needs at least one accessible active citation. Evidence freshness + proofs are retired and do not affect availability. Any read/parse/budget failure raises a fixed whole-request error, never a partial result or per-item deletion. The route owner must map this separately diff --git a/memhub/apps/core/src/one_memory/memory_evidence_freshness.py b/memhub/apps/core/src/one_memory/memory_evidence_freshness.py index 464853f..875f53e 100644 --- a/memhub/apps/core/src/one_memory/memory_evidence_freshness.py +++ b/memhub/apps/core/src/one_memory/memory_evidence_freshness.py @@ -1,36 +1,25 @@ -"""Read-only, exact-revision support checks inside the caller's SQLite snapshot. +"""Memory evidence freshness checks are retired. -No proof is legacy/unknown, not stale. One complete fresh proof suffices; all -complete proofs stale means false. Malformed records raise. An exceeded budget -means the memory is too large to verify, not that it is stale: it reads as -current, the same as having no proof, and is reported once in a while as a -content-free diagnostic. One such memory used to fail every COUNT over the -library, so the homepage returned 503 for as long as it existed (D76). -Set ONE_MEMORY_FRESHNESS_BUDGET_RAISES=1 to restore the old raising behavior. -The caller owns BEGIN/deadline/progress-handler and error-to-unavailable mapping. +Read paths used to re-validate ``memory_task_outputs.support_refs`` against +live ``evidence`` rows via ``is_memory_evidence_current``. That design hid +memories when evidence changed, but at high cost (library 503s, per-row SQL, +segment-key breakage). Call sites still import this module; registration is +now a no-op that always admits the row. + +Proof rows may still be written by the ingest path; they are ignored on read. +See ``新鲜度_todo.md`` for the remaining teardown steps. """ -import json -import logging -import os +from __future__ import annotations + import sqlite3 -import threading -import time from collections.abc import Iterator, Sequence from contextlib import contextmanager from one_memory.database_access import register_function -from one_memory.diagnostic_log import diagnostic_event -from one_memory.imports.models import EvidenceRecord -from one_memory.memory_tasks import MemoryTaskOutput, matches_evidence_ref -_PROOFS = 64 -_REFERENCES = 128 -_BODY_BYTES = 65_536 -_EVIDENCE_BYTES = 8 * 1024 * 1024 -_SCOPE_STEPS = 64 - -#: Budget refusals: the check could not finish, the data is not malformed. +#: Historical budget error codes. Retained so diagnostics/env docs do not break +#: mid-teardown; nothing raises them on the read path anymore. BUDGET_ERRORS = frozenset( { "memory_freshness_proof_budget", @@ -39,80 +28,23 @@ BUDGET_ERRORS = frozenset( "memory_freshness_scope_budget", } ) -_REPORT_INTERVAL_SECONDS = 60.0 -_reported: dict[str, tuple[float, int]] = {} -_reported_lock = threading.Lock() def budget_errors_raise() -> bool: - """Opt back into the pre-D76 behavior, where one large memory fails the query.""" - return os.environ.get("ONE_MEMORY_FRESHNESS_BUDGET_RAISES", "").strip() == "1" + """Former opt-in to raise on budget overrun. Always false after retirement.""" + return False -def _report_budget(code: str) -> None: - """Count every budget skip; write at most one line per reason per interval.""" - now = time.monotonic() - with _reported_lock: - last, pending = _reported.get(code, (float("-inf"), 0)) - pending += 1 - if now - last < _REPORT_INTERVAL_SECONDS: - _reported[code] = (last, pending) - return - _reported[code] = (now, 0) - diagnostic_event( - level="warn", - component="memory.freshness", - event="memory_freshness.budget_exceeded", - outcome="skipped", - details={"reason": code, "count": pending}, +def has_proof_table(db: sqlite3.Connection) -> bool: + """Whether ``memory_task_outputs`` exists. Unused by the retired check.""" + return ( + db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_task_outputs'" + ).fetchone() + is not None ) -def _object(pairs: list[tuple[str, object]]) -> dict[str, object]: - result: dict[str, object] = {} - for key, value in pairs: - if key in result: - raise ValueError("memory_freshness_invalid_proof") - result[key] = value - return result - - -class _Prefetched: - """Prefetched rows for the current batch, private to thread and connection. - - Scoped, never global: a cache that outlived the transaction would answer - from rows a later write had already replaced. - """ - - def __init__(self) -> None: - self._local = threading.local() - - def get(self) -> dict[str, list[sqlite3.Row]] | None: - return getattr(self._local, "rows", None) - - def evidence(self) -> dict[str, sqlite3.Row] | None: - return getattr(self._local, "evidence", None) - - @contextmanager - def using( - self, - rows: dict[str, list[sqlite3.Row]], - evidence: dict[str, sqlite3.Row] | None = None, - ) -> Iterator[None]: - previous = getattr(self._local, "rows", None) - previous_evidence = getattr(self._local, "evidence", None) - self._local.rows = rows - self._local.evidence = evidence - try: - yield - finally: - self._local.rows = previous - self._local.evidence = previous_evidence - - -_prefetched_proofs = _Prefetched() - - @contextmanager def freshness_proofs_prefetched( db: sqlite3.Connection, @@ -120,328 +52,16 @@ def freshness_proofs_prefetched( *, evidence_ids: Sequence[str] = (), ) -> Iterator[None]: - """Serve the freshness check's proof reads from one query. - - The decisions are unchanged — the rows come from the same transaction — - but the per-memory reads collapse into the one in `proof_rows_for`. - """ - with _prefetched_proofs.using( - proof_rows_for(db, memory_ids), - evidence_rows_for(db, evidence_ids) if evidence_ids else None, - ): - yield - - -def has_proof_table(db: sqlite3.Connection) -> bool: - """Whether this database keeps proofs at all. - - Without the table the registered UDF answers 1 unconditionally, so there - is nothing to prefetch and nothing to read. - """ - return ( - db.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_task_outputs'" - ).fetchone() - is not None - ) - - -_EVIDENCE_COLUMNS = ( - "evidence_id,user_id,source_id,workspace,project,conversation_id," - "message_id,correlation_id,source_ordinal,content_kind,role,written_by," - "created_at,verbatim,content_hash,revision,imported_at,updated_at" -) - -#: The budget measure the single read applies, kept as one string so the -#: batched read cannot compute it differently. -_EVIDENCE_SIZE_SQL = ( - "length(CAST(verbatim AS BLOB)) + " - "length(CAST(json_array(evidence_id,user_id,source_id,workspace,project," - "conversation_id,message_id,correlation_id,source_ordinal,content_kind," - "role,written_by,created_at,content_hash,revision,imported_at,updated_at) " - "AS BLOB))" -) - - -def evidence_rows_for( - db: sqlite3.Connection, - evidence_ids: Sequence[str], -) -> dict[str, sqlite3.Row]: - """Read the cited evidence rows for a page in one query. - - The check reads every citation twice — once for its size, against the - evidence budget, and once for the row it validates against the proof's - reference. Both are per citation, so both grow with the page. - - The size comes from the same expression the single read uses, so the - budget arithmetic cannot drift between the two paths. - """ - if not evidence_ids: - return {} - cursor = db.cursor() - cursor.row_factory = sqlite3.Row - by_id: dict[str, sqlite3.Row] = {} - for start in range(0, len(evidence_ids), 900): - batch = list(dict.fromkeys(evidence_ids[start : start + 900])) - placeholders = ",".join("?" for _ in batch) - for row in cursor.execute( - f"SELECT {_EVIDENCE_COLUMNS},{_EVIDENCE_SIZE_SQL} AS measure " - f"FROM evidence WHERE evidence_id IN ({placeholders})", - batch, - ): - by_id[str(row["evidence_id"])] = row - return by_id - - -def proof_rows_for( - db: sqlite3.Connection, - memory_ids: Sequence[str], -) -> dict[str, list[sqlite3.Row]]: - """Read the proof rows for a whole page in one query. - - The freshness check reads these per memory, and that is the read path's - whole cost: measured on the old-user library, 160 memories cost 0.69 s in - the UDF against 0.08 s for the citation subquery, and grew further while - the library was also being written to — enough to cross the five second - interrupt on `/api/v1/memories` and on the map. - - Returns only ids that have rows; a caller falling back to a single read - for anything absent keeps the same answer. - """ - if not memory_ids: - return {} - if not has_proof_table(db): - return {} - cursor = db.cursor() - cursor.row_factory = sqlite3.Row - by_id: dict[str, list[sqlite3.Row]] = {} - for start in range(0, len(memory_ids), 400): - batch = list(memory_ids[start : start + 400]) - placeholders = ",".join("?" for _ in batch) - for row in cursor.execute( - "SELECT task_id,memory_id,length(CAST(body AS BLOB)) AS size," - "substr(CAST(body AS BLOB),1,?) AS body FROM memory_task_outputs " - f"WHERE memory_id IN ({placeholders})", - (_BODY_BYTES + 1, *batch), - ): - by_id.setdefault(str(row["memory_id"]), []).append(row) - return by_id - - -def _proof_revision( - cursor: sqlite3.Cursor, - memory_id: str, - revision: int, - user_id: str, - available: set[int], -) -> int: - if revision in available or not available: - return revision - if ( - cursor.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='governed_memory_versions'" - ).fetchone() - is None - ): - return revision - original = revision - for step in range(_SCOPE_STEPS + 1): - if revision in available: - return revision - row = cursor.execute( - "SELECT v.action, typeof(v.content)='text' AND typeof(p.content)='text' " - "AND CAST(v.content AS BLOB)=CAST(p.content AS BLOB) AS same_content, " - "v.status=p.status AS same_status, " - "CAST(v.content AS BLOB)=CAST(m.content AS BLOB) AS current_content " - "FROM governed_memory_versions v " - "JOIN governed_memory_versions p ON p.memory_id=v.memory_id " - "AND p.revision=v.revision-1 " - "JOIN governed_memories m ON m.memory_id=v.memory_id " - "WHERE v.memory_id=? AND v.revision=? AND m.user_id=? AND m.revision=?", - (memory_id, revision, user_id, original), - ).fetchone() - if row is None or row[0] != "set_scopes" or not all(row[index] == 1 for index in (1, 2, 3)): - return original - if step == _SCOPE_STEPS: - raise ValueError("memory_freshness_scope_budget") - revision -= 1 - raise ValueError("memory_freshness_scope_budget") + """No-op. Kept so batched read callers need not change in this PR.""" + del db, memory_ids, evidence_ids + yield def register_memory_freshness(db: sqlite3.Connection) -> str: - """Register UDF; return a predicate for alias m. Does not change row_factory. + """Register a always-true UDF; return SQL predicate ``1``. - Production initialization supplies memory_task_outputs(memory_id) index. - Old no-index databases depend on the caller's SQL deadline. Per evaluation: - 65 rows / 64KiB each, 129 active IDs, 8MiB total Evidence material. No library - bodies are loaded; Evidence objects live only for this evaluation. No cache - persists across transactions (safe when a caller reuses its connection). + Callers that embed the return value in ``WHERE`` keep working. Callers that + invoke ``is_memory_evidence_current`` by name also get current. """ - if not has_proof_table(db): - register_function(db, "is_memory_evidence_current", 3, lambda *_: 1) - return "1" - - raising = budget_errors_raise() - - def current(memory_id: str, revision: int, user_id: str) -> int: - try: - return checked(memory_id, revision, user_id) - except ValueError as error: - code = str(error) - if raising or code not in BUDGET_ERRORS: - raise - _report_budget(code) - return 1 - except sqlite3.Error as error: - # A freshness proof is an enhancement, never the reason a person - # cannot see their library. Under a concurrent history import the - # 5s read deadline interrupts this UDF mid-statement; the old - # behaviour let that escape, and the caller turned it into - # `memory_library_unavailable`, so the memory list stayed empty for - # as long as the import ran. Answer "current" instead and record - # why, which is the same trade the D76 budget skip already makes. - _report_budget("memory_freshness_sql_error") - logging.getLogger(__name__).warning( - "memory_freshness_udf_failed %s: %s", type(error).__name__, error - ) - return 1 - - def checked(memory_id: str, revision: int, user_id: str) -> int: - if not isinstance(memory_id, str) or type(revision) is not int or revision < 1: - raise ValueError("memory_freshness_invalid_identity") - if not isinstance(user_id, str): - raise ValueError("memory_freshness_invalid_identity") - cursor = db.cursor() - cursor.row_factory = sqlite3.Row - prefetched = _prefetched_proofs.get() - if prefetched is not None: - rows = prefetched.get(memory_id, ()) - else: - rows = cursor.execute( - "SELECT task_id,memory_id,length(CAST(body AS BLOB)) AS size," - "substr(CAST(body AS BLOB),1,?) AS body FROM memory_task_outputs " - "WHERE memory_id=? LIMIT ?", - (_BODY_BYTES + 1, memory_id, _PROOFS + 1), - ).fetchall() - rows = list(rows) - if len(rows) > _PROOFS: - raise ValueError("memory_freshness_proof_budget") - decoded = [] - for row in rows: - if type(row["size"]) is not int or row["size"] > _BODY_BYTES: - raise ValueError("memory_freshness_proof_budget") - body = json.loads(row["body"], object_pairs_hook=_object) - if not isinstance(body, dict) or type(body.get("memory_revision")) is not int: - raise ValueError("memory_freshness_invalid_proof") - decoded.append((row, body)) - revision = _proof_revision( - cursor, - memory_id, - revision, - user_id, - {body["memory_revision"] for _, body in decoded}, - ) - applicable = [] - for row, body in decoded: - if body["memory_revision"] != revision: - continue - raw_refs = body.get("support_refs", []) - if not isinstance(raw_refs, list) or any( - not isinstance(ref, dict) - or set(ref) - {"evidence_id", "revision", "content_hash", "identity_hash"} - for ref in raw_refs - ): - raise ValueError("memory_freshness_invalid_reference") - output = MemoryTaskOutput.model_validate_json(row["body"], strict=True) - if output.memory_id != memory_id or output.task_id != row["task_id"]: - raise ValueError("memory_freshness_proof_identity") - if len(output.support_refs) > _REFERENCES: - raise ValueError("memory_freshness_reference_budget") - for ref in output.support_refs: - if ( - set(ref) - {"evidence_id", "revision", "content_hash", "identity_hash"} - or not ref["evidence_id"] - or ref["revision"] < 1 - or any( - len(ref[key]) != 64 - or any(char not in "0123456789abcdef" for char in ref[key]) - for key in ("content_hash", "identity_hash") - if key in ref - ) - ): - raise ValueError("memory_freshness_invalid_reference") - proof_ids = [ref["evidence_id"] for ref in output.support_refs] - if len(proof_ids) != len(set(proof_ids)): - raise ValueError("memory_freshness_duplicate_reference") - applicable.append(output) - if not applicable: - return 1 - active = cursor.execute( - "SELECT evidence_id FROM governed_memory_evidence " - "WHERE memory_id=? AND active=1 LIMIT ?", - (memory_id, _REFERENCES + 1), - ).fetchall() - if len(active) > _REFERENCES: - raise ValueError("memory_freshness_reference_budget") - ids = {row[0] for row in active} - complete = [ - output - for output in applicable - if output.support_refs and {ref["evidence_id"] for ref in output.support_refs} == ids - ] - if not complete: - return 1 - records: dict[str, EvidenceRecord | None] = {} - used = 0 - prefetched_evidence = _prefetched_proofs.evidence() - for identifier in ids: - evidence_row = ( - prefetched_evidence.get(identifier) if prefetched_evidence is not None else None - ) - if evidence_row is not None: - size = evidence_row["measure"] - if type(size) is not int or size < 0: - raise ValueError("memory_freshness_invalid_evidence") - else: - measured = cursor.execute( - f"SELECT {_EVIDENCE_SIZE_SQL} FROM evidence WHERE evidence_id=?", - (identifier,), - ).fetchone() - if measured is None: - records[identifier] = None - continue - if type(measured[0]) is not int or measured[0] < 0: - raise ValueError("memory_freshness_invalid_evidence") - size = measured[0] - used += size - if used > _EVIDENCE_BYTES: - raise ValueError("memory_freshness_evidence_budget") - if evidence_row is None: - evidence_row = cursor.execute( - f"SELECT {_EVIDENCE_COLUMNS} FROM evidence WHERE evidence_id=?", - (identifier,), - ).fetchone() - if evidence_row is None: - records[identifier] = None - continue - # evidence_ref explicitly excludes origin_meta/imported_at/updated_at. - # `sqlite3.Row` iterates values, not column names, so pair the - # names with the values explicitly. - fields = { - name: value - for name, value in zip(evidence_row.keys(), tuple(evidence_row), strict=False) - if name != "measure" - } - records[identifier] = EvidenceRecord.model_validate({**fields, "origin_meta": {}}) - for output in complete: - if all( - (record := records[ref["evidence_id"]]) is not None - and record.user_id == user_id - and matches_evidence_ref(record, ref) - for ref in output.support_refs - ): - return 1 - return 0 - - register_function(db, "is_memory_evidence_current", 3, current) - return "is_memory_evidence_current(m.memory_id,m.revision,m.user_id)" + register_function(db, "is_memory_evidence_current", 3, lambda *_: 1) + return "1" diff --git a/memhub/apps/core/tests/TEST_INDEX.md b/memhub/apps/core/tests/TEST_INDEX.md index 6ca4079..cb443bb 100644 --- a/memhub/apps/core/tests/TEST_INDEX.md +++ b/memhub/apps/core/tests/TEST_INDEX.md @@ -203,7 +203,7 @@ | `test_memory_correction_sync.py` | `integration` | R-27: a correction that only lands locally, while HY keeps the old text. | | `test_memory_deletion_outbox.py` | `contract` | 删除发件箱：一条删除同一时刻只归一个执行者，结果只由持有领取凭证的那个写。 | | `test_memory_dispatch_lifecycle.py` | `contract` | Exercise the automatic enqueue boundary without producing substitute memories. | -| `test_memory_evidence_freshness.py` | `contract` | 覆盖 fresh then revised and identity only change、any complete fresh proof wins over old proof、other revision and incomplete proof do not hide legacy or correction 等 21 个场景。 | +| `test_memory_evidence_freshness.py` | `contract` | 新鲜度拆除后：读路径恒为 current；证据修订 / 畸形 proof / segment ref 不再藏记忆或 503。 | | `test_memory_freshness_integration_review.py` | `integration` | Independent synthetic read-path review; no lifespan, model or user database. | | `test_memory_governance.py` | `contract` | 覆盖 correction keeps verbatim and appends history、mutations are revision checked and request idempotent、delete result survives restart and request replay 等 42 个场景。 | | `test_memory_governance_api.py` | `integration` | 覆盖 memory governance api preserves evidence and reports cleanup、memory validation does not echo content 场景。 | diff --git a/memhub/apps/core/tests/test_call_event_references.py b/memhub/apps/core/tests/test_call_event_references.py index 0e5c220..2ba01ca 100644 --- a/memhub/apps/core/tests/test_call_event_references.py +++ b/memhub/apps/core/tests/test_call_event_references.py @@ -355,8 +355,9 @@ def test_actual_evidence_proof_revision_and_corrected_memory(fresh): assert all(item.available for item in resolve_call_references(path, event(raw, reference))) evidence.put(value.model_copy(update={"verbatim": "revised evidence"})) result = resolve_call_references(path, event(raw, reference)) + # Raw refs still track evidence.revision; memory refs no longer use freshness. assert_hidden(result[0], raw, "changed") - assert_hidden(result[1], reference, "changed") + assert result[1].available corrected = as_person(MemoryGovernanceStore(path)).correct( memory.memory_id, user_id="local", @@ -369,7 +370,7 @@ def test_actual_evidence_proof_revision_and_corrected_memory(fresh): assert resolve_call_references(path, event(current))[0].title == corrected.content -def test_scope_only_revision_still_checks_original_support(fresh): +def test_scope_only_revision_keeps_memory_readable_when_evidence_changes(fresh): path, evidence, value, record, memory, tasks, output = fresh updated = as_person(MemoryGovernanceStore(path)).set_scopes( memory.memory_id, @@ -381,25 +382,27 @@ def test_scope_only_revision_still_checks_original_support(fresh): reference = memory_ref(updated.memory_id, updated.revision) assert resolve_call_references(path, event(reference))[0].available evidence.put(value.model_copy(update={"verbatim": "revised evidence"})) - assert_hidden(resolve_call_references(path, event(reference))[0], reference, "changed") + assert resolve_call_references(path, event(reference))[0].available -def test_freshness_identity_change_without_revision_change(fresh): +def test_evidence_identity_change_without_revision_keeps_memory_readable(fresh): path, evidence, value, record, memory, tasks, output = fresh reference = memory_ref(memory.memory_id, memory.revision) assert resolve_call_references(path, event(reference))[0].available with closing(sqlite3.connect(path)) as db, db: db.execute("UPDATE evidence SET workspace='workspace:changed'") - assert_hidden(resolve_call_references(path, event(reference))[0], reference, "changed") + assert resolve_call_references(path, event(reference))[0].available -def test_invalid_freshness_proof_fails_whole_response_but_revoked_never_reads_it(fresh): +def test_invalid_proof_is_ignored_but_revocation_still_hides_memory(fresh): path, evidence, value, record, memory, tasks, output = fresh reference = memory_ref(memory.memory_id, memory.revision) with closing(sqlite3.connect(path)) as db, db: db.execute("UPDATE memory_task_outputs SET body='invalid secret proof'") - with pytest.raises(CallReferencesUnavailable, match="^call_references_unavailable$"): - resolve_call_references(path, event(ref(key=record.evidence_id), reference)) + assert all( + item.available + for item in resolve_call_references(path, event(ref(key=record.evidence_id), reference)) + ) with closing(sqlite3.connect(path)) as db, db: grant(db, "cursor") result = resolve_call_references(path, event(reference)) @@ -455,9 +458,13 @@ def test_authorization_revision_freshness_and_titles_share_one_snapshot(fresh, m assert [item.title for item in snapshot] == [memory.content, record.verbatim] assert changed == [True] and reads and all(reads) current = resolve_call_references(path, event(reference, raw)) - assert_hidden( - current[0], reference, "changed" if change in ("evidence", "memory") else "unavailable" - ) + # Memory refs no longer hide on evidence revision/identity via freshness. + if change == "evidence": + assert current[0].available + else: + assert_hidden( + current[0], reference, "changed" if change == "memory" else "unavailable" + ) assert current[1].available == (change in ("memory", "link")) diff --git a/memhub/apps/core/tests/test_memory_evidence_freshness.py b/memhub/apps/core/tests/test_memory_evidence_freshness.py index 1dfc508..6d80e61 100644 --- a/memhub/apps/core/tests/test_memory_evidence_freshness.py +++ b/memhub/apps/core/tests/test_memory_evidence_freshness.py @@ -1,3 +1,5 @@ +"""Freshness retirement: read path always admits, even when evidence changed.""" + import json import sqlite3 @@ -5,7 +7,11 @@ import pytest from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy from one_memory.imports.storage import EvidenceStore -from one_memory.memory_evidence_freshness import register_memory_freshness +from one_memory.memory_evidence_freshness import ( + budget_errors_raise, + freshness_proofs_prefetched, + register_memory_freshness, +) from one_memory.memory_governance import MemoryDraft, MemoryGovernanceStore, MemoryKind from one_memory.memory_tasks import MemoryTaskOutput, MemoryTaskStore, evidence_ref from tests.person_door import as_person @@ -61,46 +67,45 @@ def check(path, memory): ).fetchone()[0] -def test_fresh_then_revised_and_identity_only_change(case): +def test_register_returns_constant_true_predicate(case): + path, evidence, value, record, memory, tasks, output = case + with sqlite3.connect(path) as db: + db.execute("BEGIN") + assert register_memory_freshness(db) == "1" + assert db.execute("SELECT is_memory_evidence_current(?,?,?)", ("x", 1, "local")).fetchone() == ( + 1, + ) + + +def test_revised_evidence_no_longer_hides_memory(case): path, evidence, value, record, memory, tasks, output = case assert check(path, memory) == 1 evidence.put(value.model_copy(update={"verbatim": "new statement"})) - assert check(path, memory) == 0 - - -def test_any_complete_fresh_proof_wins_over_old_proof(case): - path, evidence, value, record, memory, tasks, output = case - revised = evidence.put(value.model_copy(update={"verbatim": "new statement"})).record - tasks.save_output( - output.model_copy(update={"task_id": "task2", "support_refs": (evidence_ref(revised),)}) - ) assert check(path, memory) == 1 -def test_other_revision_and_incomplete_proof_do_not_hide_legacy_or_correction(case): - path, evidence, value, record, memory, tasks, output = case - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - tasks.save_output(output.model_copy(update={"memory_revision": memory.revision + 1})) - assert check(path, memory) == 1 - tasks.save_output(output.model_copy(update={"support_refs": ()})) - assert check(path, memory) == 1 - - -@pytest.mark.parametrize("broken", ["json", "identity", "ref"]) -def test_bad_proof_raises_not_false_empty(case, broken): +def test_segment_support_refs_do_not_fail_reads(case): path, evidence, value, record, memory, tasks, output = case body = output.model_dump(mode="json") - if broken == "identity": - body["memory_id"] = "other" - if broken == "ref": - body["support_refs"][0]["revision"] = "1" - payload = json.dumps(body) - if broken == "json": - payload = "{" + body["support_refs"][0].update( + segment_index=0, + segment_start=0, + segment_end=len(record.verbatim), + segment_hash="0" * 64, + ) with sqlite3.connect(path) as db: - db.execute("UPDATE memory_task_outputs SET body=?", (payload,)) - with pytest.raises(sqlite3.OperationalError): - check(path, memory) + db.execute("UPDATE memory_task_outputs SET body=?", (json.dumps(body),)) + assert check(path, memory) == 1 + + +def test_malformed_proof_does_not_fail_reads(case): + path, evidence, value, record, memory, tasks, output = case + with sqlite3.connect(path) as db: + db.execute( + "UPDATE memory_task_outputs SET body=? WHERE memory_id=?", + ("{", memory.memory_id), + ) + assert check(path, memory) == 1 def test_missing_table_compatibility_is_readonly(): @@ -111,239 +116,19 @@ def test_missing_table_compatibility_is_readonly(): assert db.execute("SELECT name FROM sqlite_master").fetchall() == [] -def test_table_presence_is_checked_once_at_registration(case): +def test_prefetch_is_noop(case): path, evidence, value, record, memory, tasks, output = case with sqlite3.connect(path) as db: db.execute("BEGIN") - statements = [] - db.set_trace_callback(statements.append) - predicate = register_memory_freshness(db) - for _ in range(3): - assert db.execute(f"SELECT {predicate} FROM governed_memories m").fetchone() == (1,) - assert sum("sqlite_master" in statement for statement in statements) == 1 + with freshness_proofs_prefetched(db, [memory.memory_id], evidence_ids=[record.evidence_id]): + assert register_memory_freshness(db) == "1" + assert check(path, memory) == 1 -def test_proof_budget_reads_current_instead_of_failing(case): - path, evidence, value, record, memory, tasks, output = case - for index in range(65): - tasks.save_output(output.model_copy(update={"task_id": f"task-{index}"})) - assert check(path, memory) == 1 - - -def test_proof_budget_still_raises_when_opted_back(case, monkeypatch): +def test_budget_raise_env_is_inert(case, monkeypatch): path, evidence, value, record, memory, tasks, output = case monkeypatch.setenv("ONE_MEMORY_FRESHNESS_BUDGET_RAISES", "1") + assert budget_errors_raise() is False for index in range(65): tasks.save_output(output.model_copy(update={"task_id": f"task-{index}"})) - with pytest.raises(sqlite3.OperationalError): - check(path, memory) - - -def test_snapshot_cache_does_not_hide_changes_across_transactions(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - db.execute("BEGIN") - predicate = register_memory_freshness(db) - query = f"SELECT {predicate} FROM governed_memories m" - assert db.execute(query).fetchone() == (1,) - db.commit() - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - db.execute("BEGIN") - register_memory_freshness(db) - assert db.execute(query).fetchone() == (0,) - - -def test_identity_hash_detects_metadata_change_without_revision_or_content_change(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - db.execute("UPDATE evidence SET workspace='workspace:changed'") - assert check(path, memory) == 0 - - -def test_incomplete_fresh_proof_cannot_override_complete_stale(case): - path, evidence, value, record, memory, tasks, output = case - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - tasks.save_output(output.model_copy(update={"task_id": "partial", "support_refs": ()})) - assert check(path, memory) == 0 - - -def test_sql_query_only_and_missing_proof_compatibility(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - db.execute("DELETE FROM memory_task_outputs") - with sqlite3.connect(path) as db: - db.execute("PRAGMA query_only=ON") - db.execute("BEGIN") - before = db.total_changes - predicate = register_memory_freshness(db) - assert db.execute(f"SELECT {predicate} FROM governed_memories m").fetchone() == (1,) - assert db.total_changes == before - - -def test_oversized_evidence_reads_current_without_loading_body(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - db.execute("UPDATE evidence SET verbatim=zeroblob(?)", (8 * 1024 * 1024 + 1,)) assert check(path, memory) == 1 - - -@pytest.mark.parametrize("revise_first", [False, True]) -def test_scope_only_chain_preserves_old_support_guard(case, revise_first): - path, evidence, value, record, memory, tasks, output = case - governance = MemoryGovernanceStore(path) - if revise_first: - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - for index in range(3): - memory = as_person(governance).set_scopes( - memory.memory_id, - user_id="local", - request_id=f"scope-{index}", - expected_revision=memory.revision, - scopes=(f"workspace:{index}",), - ) - if not revise_first: - assert check(path, memory) == 1 - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - assert check(path, memory) == 0 - - -@pytest.mark.parametrize("action", ["correct", "undo", "hy_reconcile", "unknown"]) -def test_scope_chain_does_not_cross_non_scope_action(case, action): - path, evidence, value, record, memory, tasks, output = case - governance = MemoryGovernanceStore(path) - memory = as_person(governance).correct( - memory.memory_id, - user_id="local", - request_id="correct", - expected_revision=memory.revision, - content=memory.content, - ) - with sqlite3.connect(path) as db: - db.execute( - "UPDATE governed_memory_versions SET action=? WHERE revision=?", - (action, memory.revision), - ) - memory = as_person(governance).set_scopes( - memory.memory_id, - user_id="local", - request_id="scope", - expected_revision=memory.revision, - scopes=("workspace:changed",), - ) - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - assert check(path, memory) == 1 - - -def test_scope_chain_budget_reads_current(case): - path, evidence, value, record, memory, tasks, output = case - governance = MemoryGovernanceStore(path) - for index in range(65): - memory = as_person(governance).set_scopes( - memory.memory_id, - user_id="local", - request_id=f"scope-{index}", - expected_revision=memory.revision, - scopes=(f"workspace:{index}",), - ) - assert check(path, memory) == 1 - - -def test_changed_content_in_scope_history_is_not_inherited(case): - path, evidence, value, record, memory, tasks, output = case - governance = MemoryGovernanceStore(path) - memory = as_person(governance).set_scopes( - memory.memory_id, - user_id="local", - request_id="scope", - expected_revision=memory.revision, - scopes=("workspace:changed",), - ) - with sqlite3.connect(path) as db: - db.execute( - "UPDATE governed_memory_versions SET content='different' WHERE revision=?", - (output.memory_revision,), - ) - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - assert check(path, memory) == 1 - - -def test_scope_chain_keeps_large_governed_content_inside_sql(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - content = "x" * 1_000_000 - db.execute("UPDATE governed_memories SET content=?", (content,)) - db.execute("UPDATE governed_memory_versions SET content=?", (content,)) - governance = MemoryGovernanceStore(path) - memory = as_person(governance).set_scopes( - memory.memory_id, - user_id="local", - request_id="scope", - expected_revision=memory.revision, - scopes=("workspace:changed",), - ) - evidence.put(value.model_copy(update={"verbatim": "new statement"})) - - class BoundedCursor(sqlite3.Cursor): - def fetchone(self): - row = super().fetchone() - if row is not None: - assert all(not isinstance(item, str) or len(item) < 65_536 for item in row) - return row - - class BoundedConnection(sqlite3.Connection): - def cursor(self, factory=BoundedCursor): - return super().cursor(factory) - - with sqlite3.connect(path, factory=BoundedConnection) as db: - db.execute("BEGIN") - predicate = register_memory_freshness(db) - assert db.execute(f"SELECT {predicate} FROM governed_memories m").fetchone() == (0,) - - -def _confirmed_with_oversized_proof(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - db.execute( - "UPDATE memory_task_outputs SET body=? WHERE memory_id=?", - ( - json.dumps({"memory_revision": memory.revision, "pad": "x" * 70_000}), - memory.memory_id, - ), - ) - return path, memory - - -def test_one_oversized_memory_no_longer_fails_the_library_count(case): - """D76: one memory whose proof is too large used to 503 the homepage forever.""" - path, memory = _confirmed_with_oversized_proof(case) - with sqlite3.connect(path) as db: - db.execute("BEGIN") - predicate = register_memory_freshness(db) - count = db.execute( - f"SELECT COUNT(*) FROM governed_memories m WHERE {predicate}" - ).fetchone()[0] - assert count == 1 - - -def test_malformed_proof_still_raises(case): - path, evidence, value, record, memory, tasks, output = case - with sqlite3.connect(path) as db: - db.execute( - "UPDATE memory_task_outputs SET body=? WHERE memory_id=?", ("{}", memory.memory_id) - ) - with pytest.raises(sqlite3.OperationalError): - check(path, memory) - - -def test_budget_skip_is_reported_without_content(case, monkeypatch): - import one_memory.memory_evidence_freshness as freshness - - events = [] - monkeypatch.setattr(freshness, "diagnostic_event", lambda **values: events.append(values)) - monkeypatch.setattr(freshness, "_reported", {}) - path, memory = _confirmed_with_oversized_proof(case) - for _ in range(3): - assert check(path, memory) == 1 - assert len(events) == 1 - assert events[0]["details"] == {"reason": "memory_freshness_proof_budget", "count": 1} - assert "pad" not in json.dumps(events[0]) diff --git a/memhub/apps/core/tests/test_memory_freshness_integration_review.py b/memhub/apps/core/tests/test_memory_freshness_integration_review.py index 11032bc..92e04dd 100644 --- a/memhub/apps/core/tests/test_memory_freshness_integration_review.py +++ b/memhub/apps/core/tests/test_memory_freshness_integration_review.py @@ -102,17 +102,11 @@ def revise(case): ).record -def test_complete_stale_proof_hides_all_four_reads_without_mutation(case): - # D22: the app-facing totals (library total, homepage count) count with - # the cheap permission filters only, so a stale proof no longer removes a - # row from the totals; the freshness gate still hides it from every read - # that returns rows (context, source read access). +def test_revised_evidence_remains_readable_after_freshness_retirement(case): + """Evidence revision no longer hides library / context / source reads.""" assert visible(case) == (1, 1, 1, 1) revise(case) - total, home, context_items, source_items = visible(case) - # D22: home total counts cheaply and may keep the stale row; the reads - # that return rows still hide it. - assert (home, context_items, source_items) == (1, 0, 0) + assert visible(case) == (1, 1, 1, 1) assert case.governance.get(case.memory.memory_id, user_id="local") == case.memory assert case.tasks.outputs(case.output.task_id) == (case.output,) @@ -120,11 +114,10 @@ def test_complete_stale_proof_hides_all_four_reads_without_mutation(case): @pytest.mark.parametrize( "revise_first", [False, True], ids=["scope-before-revision", "scope-after-revision"] ) -def test_scope_only_revision_must_not_resurrect_stale_derived_content(case, revise_first): +def test_scope_only_revision_keeps_memory_readable_after_evidence_change(case, revise_first): if revise_first: revise(case) - _, home, context_items, source_items = visible(case) - assert (context_items, source_items) == (0, 0) + assert visible(case) == (1, 1, 1, 1) changed = as_person(case.governance).set_scopes( case.memory.memory_id, user_id="local", @@ -136,8 +129,7 @@ def test_scope_only_revision_must_not_resurrect_stale_derived_content(case, revi if not revise_first: assert visible(case, changed) == (1, 1, 1, 1) revise(case) - _, home, context_items, source_items = visible(case, changed) - assert (context_items, source_items) == (0, 0) + assert visible(case, changed) == (1, 1, 1, 1) def test_explicit_correct_remains_readable_but_never_overrides_revocation(case): @@ -152,7 +144,10 @@ def test_explicit_correct_remains_readable_but_never_overrides_revocation(case): assert visible(case, corrected) == (1, 1, 1, 1) with closing(sqlite3.connect(case.path)) as db, db: db.execute( - "INSERT INTO source_grants VALUES ('local','cursor','/fixture','default',?,?)", + """INSERT INTO source_grants + (user_id,source_id,root_path,requested_path,root_identity, + candidate_kind,granted_at,revoked_at,revision) + VALUES ('local','cursor','/fixture','/fixture','fixture','default',?,?,1)""", ("2026-09-20T00:00:00Z", "2026-09-20T00:00:01Z"), ) assert visible(case, corrected) == (0, 0, 0, 0) @@ -181,7 +176,10 @@ def test_revocation_before_freshness_snapshot_cannot_release_memory(case, monkey # Revocation commits before any SELECT pins the freshness transaction. with closing(sqlite3.connect(case.path)) as writer, writer: writer.execute( - "INSERT INTO source_grants VALUES ('local','cursor','/fixture','default',?,?)", + """INSERT INTO source_grants + (user_id,source_id,root_path,requested_path,root_identity, + candidate_kind,granted_at,revoked_at,revision) + VALUES ('local','cursor','/fixture','/fixture','fixture','default',?,?,1)""", ("2026-09-20T00:00:00Z", "2026-09-20T00:00:01Z"), ) changes.append(True) @@ -194,7 +192,7 @@ def test_revocation_before_freshness_snapshot_cannot_release_memory(case, monkey assert returned == [] -def test_udf_uses_caller_snapshot_then_observes_revision_in_next_transaction(case): +def test_udf_stays_current_across_evidence_revision(case): with closing(sqlite3.connect(case.path)) as reader: reader.execute("PRAGMA journal_mode=WAL") reader.execute("BEGIN") @@ -206,7 +204,7 @@ def test_udf_uses_caller_snapshot_then_observes_revision_in_next_transaction(cas reader.commit() reader.execute("BEGIN") register_memory_freshness(reader) - assert reader.execute(sql).fetchone() == (0,) + assert reader.execute(sql).fetchone() == (1,) def _oversize(case, bad): @@ -227,20 +225,16 @@ def test_material_too_large_to_verify_reads_as_current(case, bad): @pytest.mark.parametrize("bad", ["duplicate-key"]) -def test_bad_material_is_unavailable_not_successful_empty(case, bad): +def test_bad_proof_material_no_longer_makes_reads_unavailable(case, bad): with closing(sqlite3.connect(case.path)) as db, db: db.execute( "UPDATE memory_task_outputs SET body=?", ('{"memory_revision":2,"memory_revision":2}',), ) + assert visible(case) == (1, 1, 1, 1) result = context(case) - assert result.status == "unavailable" and result.reason == "read_unavailable" - assert result.items == () - for read in (read_memory_library, read_home_summary): - with pytest.raises(sqlite3.Error): - read(case.path) - with pytest.raises(sqlite3.Error): - SourceReadAccess(case.path).memories([case.memory]) + assert result.status != "unavailable" + assert len(result.items) == 1 def test_route_serves_memories_whose_proof_is_too_large(case, tmp_path): diff --git a/memhub/apps/core/tests/test_memory_library.py b/memhub/apps/core/tests/test_memory_library.py index b664010..02bab88 100644 --- a/memhub/apps/core/tests/test_memory_library.py +++ b/memhub/apps/core/tests/test_memory_library.py @@ -73,8 +73,9 @@ def put( def grant(db, source, *, user="local", revoked=NOW): db.execute( """INSERT OR REPLACE INTO source_grants - (user_id,source_id,root_path,candidate_kind,granted_at,revoked_at) - VALUES (?,?, '/synthetic', 'default', ?, ?)""", + (user_id,source_id,root_path,requested_path,root_identity, + candidate_kind,granted_at,revoked_at,revision) + VALUES (?,?, '/synthetic', '/synthetic', 'synthetic', 'default', ?, ?, 1)""", (user, source, NOW, revoked), ) diff --git a/memhub/scripts/install-docs.test.mjs b/memhub/scripts/install-docs.test.mjs index 0c8e348..927d934 100644 --- a/memhub/scripts/install-docs.test.mjs +++ b/memhub/scripts/install-docs.test.mjs @@ -12,8 +12,8 @@ test("installation docs disclose publication and lifecycle boundaries", async () "utf8", ); - assert.match(readme, /公共 Python SDK 为 `hy-memory==2\.0\.3`/); - assert.match(readme, /`v1\.0\.3` 的发行目标是 Linux x64\/arm64 Runtime/); + assert.match(readme, /公共 Python SDK 为 `hy-memory==2\.0\.4`/); + assert.match(readme, /`v1\.0\.4` 的发行目标是 Linux x64\/arm64 Runtime/); assert.match(readme, /Linux 制品由 GitHub Actions/); assert.match(readme, /macOS.*Developer ID 签名、公证/s); assert.match(readme, /release-manifest\.json/); diff --git a/新鲜度_todo.md b/新鲜度_todo.md new file mode 100644 index 0000000..03208ab --- /dev/null +++ b/新鲜度_todo.md @@ -0,0 +1,70 @@ +# 新鲜度（memory evidence freshness）拆除 TODO + +状态：PR1 已开（读路径短路）——分支 `chore/retire-memory-freshness`。#86 不合并。 + +## 结论 + +- 新鲜度 = 写记忆时记下 `support_refs` 指纹，读时在 MemHub SQLite 里逐条核对 evidence 是否仍匹配；对不上则滤掉该 memory。 +- 这是过度设计：读路径成本高、失败面大（曾导致记忆库整页 503），收益仅是「证据被改过就藏记忆」。 +- 业务层为喂 SDK 再做长证据分段（`segment_*`），与 SDK 自身阶段/窗口处理重复；#86 只是继续兼容这套 proof，不应合入。 + +## 范围（MemHub Core，非 SDK） + +权威库：`~/.one-memory/one-memory.sqlite3` +不涉及 SDK `hy-memory/**` 内部 store。 + +### 读路径（优先拆） + +| 位置 | 作用 | +|---|---| +| `memhub/apps/core/src/one_memory/memory_evidence_freshness.py` | UDF `is_memory_evidence_current`、proof prefetch、预算跳过 | +| `source_read_access.py` | 列表/过滤时 per-memory freshness | +| `context_memory.py` | search/context 候选行 `_memory_row_is_fresh` | +| `memory_library.py` / `home_summary.py` | 记忆库 / 首页可读谓词 | +| `memory_visibility.py` | 可见性缓存与 freshness 注册 | +| `call_event_references.py` | 引用解析时的 freshness | + +### 写路径 / 数据（后拆，可分 PR） + +| 位置 | 作用 | +|---|---| +| `memory_tasks.py` → `MemoryTaskOutput.support_refs` / `TaskEvidenceRef` | 抽取输出里的证明 | +| `memory_task_outputs` 表 | proof 持久化 | +| `memory_processing.py` 分段（`segment_evidence_record` / `_semantic_segment_spans`） | 为 HY 窗口切段；与 freshness 耦合但不全是 freshness——分段是否保留需单独拍板 | + +### 测试（随拆除改/删） + +- `tests/test_memory_evidence_freshness.py` +- `tests/test_memory_freshness_integration_review.py` +- `tests/test_read_path_freshness_batch.py` +- 以及 mock `register_memory_freshness` 的 library / context / visibility / home_summary / call_event 测试 + +## 建议落地步骤 + +1. **读路径短路（最小 PR）** — 本 PR + `register_memory_freshness` 恒返回 `"1"`（UDF 恒 `1`），`freshness_proofs_prefetched` 为空操作。行为：证据改过也不再藏 memory；畸形/segment proof 不再 503。 + 同步改「期望 stale=0」的单测为「始终可读」。 + +2. **拆调用点** + 去掉 `AND is_memory_evidence_current(...)`、`freshness_proofs_prefetched`、`_memory_row_is_fresh` 等死代码，避免假接口长期残留。 + +3. **停写 / 弃用 proof（可选第二 PR）** + 新任务不再依赖 `support_refs` 做读决策；表与历史行可先保留只读，再决定是否 migration 清理。 + +4. **分段策略另议** + 长证据切段是 ingest 窗口问题，不是新鲜度问题。拆除 freshness 后单独决定：继续由 MemHub 切 messages，还是交给 SDK。 + +5. **对外沟通** + - #86：已留言不合并、将拆除设计。 + - 相关 issue（如 #74 里 freshness 503）关闭或改指向「拆除」而非「修 segment」。 + +## 验收 + +- 记忆库 / search / context 在存在「证据已修订 + 旧 proof」时仍返回该 memory，且不再因 malformed/segment ref 503。 +- 无 `ONE_MEMORY_FRESHNESS_*` 行为依赖；文档/env 若有提及一并删。 +- Core 相关测试绿。 + +## 非目标 + +- 不改 SDK 内部抽取/存储。 +- 不在本轮做「用别的一致性模型替换 freshness」；先删，再按需重设计。