refactor(memhub): simplify durable memory scheduling

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
alvinfeiandCursor committed 2026-10-18 16:11:02 +08:00
1 parent cdd549fdd1
commit fb71eafa35
43 files changed
+1230 -598

No files matched your search

+8 -16
View File
@@ -133,7 +133,7 @@ from one_memory.imports.jobs import (
from one_memory.imports.live_sync import LiveSyncHealth, PollingLiveSyncSupervisor
from one_memory.imports.live_watch import WATCH_MODE, FileWatchLiveSync
from one_memory.imports.locations import SourceLocation, inspect_source_location
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
from one_memory.imports.runner import live_sync_arrival
from one_memory.imports.sources.common import SourceInstallState
from one_memory.imports.sources.registry import (
@@ -1159,9 +1159,7 @@ def create_app(
# write gate admits it by its grant. Every manual note's job was
# therefore skipped on every drain tick — the note stayed
# "saved, waiting" with no task behind it, forever.
source_claim_enabled=lambda source_id: _sedimentation_claim_enabled(
app, source_id
),
source_claim_enabled=lambda source_id: _sedimentation_claim_enabled(app, source_id),
)
app.state.memory_write_drain = memory_write_drain
app.state.memory_write_drain_task = watched_task(
@@ -1190,10 +1188,7 @@ def create_app(
for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True):
if (
record not in chunk
or not any(
item.role == "user" and item.written_by.value == "person"
for item in chunk
)
or not any(item.role == "user" for item in chunk)
or not any(item.role == "assistant" for item in chunk)
):
continue
@@ -3690,9 +3685,7 @@ def create_app(
async with app.state.sedimentation_lock:
previous = store.sedimentation_enabled()
if previous is not request.enabled:
await asyncio.to_thread(
store.set_sedimentation_enabled, request.enabled
)
await asyncio.to_thread(store.set_sedimentation_enabled, request.enabled)
changed = True
diagnostic_event(
level="info",
@@ -4068,6 +4061,7 @@ async def _authorize_answer_generation(
# The explicit question is this operation's input, not a standing permission for history.
if not question.strip() or not grounding:
raise MemoryAnswerError("memory_answer_generation_not_authorized")
def check() -> None:
if _model_profile_store(app).active_identity() != (
profile.profile_id,
@@ -4091,6 +4085,7 @@ async def _authorize_answer_generation(
await recheck()
return recheck
class _LazyASGIApp:
"""Compatibility ASGI app that is built only when a server imports it.
@@ -4473,7 +4468,7 @@ async def _has_large_old_history(adapter: SourceAdapter, *, cutoff: datetime) ->
if observed.tzinfo is None:
observed = observed.replace(tzinfo=UTC)
has_old_data = has_old_data or observed.astimezone(UTC) < cutoff
if event.evidence.role == "user" and event.evidence.written_by is WrittenBy.PERSON:
if event.evidence.role == "user":
user_turns += 1
if user_turns > ONBOARDING_LARGE_HISTORY_USER_TURNS and has_old_data:
return True
@@ -5095,10 +5090,7 @@ def _history_import_gate_released(app: FastAPI) -> bool:
for job in jobs.list_jobs()
if job.operation == "history"
and job.source_id in selected
and (
state.started_at is None
or job.created_at >= state.started_at
)
and (state.started_at is None or job.created_at >= state.started_at)
]
if {job.source_id for job in matching} != selected:
return False
@@ -196,7 +196,7 @@ def read_raw_context(
if _prefers_declared_facts(request.query)
else "1"
)
person_authored = "CASE WHEN e.written_by='person' THEN 1 ELSE 0 END"
person_authored = "CASE WHEN e.role='user' THEN 1 ELSE 0 END"
query_echo = (
"CASE WHEN trim(e.verbatim)=? OR trim(e.verbatim)=? THEN 0 ELSE 1 END"
if _prefers_declared_facts(request.query)
@@ -470,9 +470,8 @@ def revalidate_raw_context(
)
except (ValueError, TypeError):
break
if (row["role"], row["written_by"], created) != (
if (row["role"], created) != (
message.role,
message.written_by,
message.created_at,
):
break
@@ -66,12 +66,12 @@ def _field(text: str) -> ResumeField | None:
return None
def _raw_class(written_by: str, evidence_id: str, accepted: frozenset[str]) -> EvidenceClass:
if written_by == "person":
def _raw_class(role: str, evidence_id: str, accepted: frozenset[str]) -> EvidenceClass:
if role == "user":
return "person_statement"
if written_by == "agent":
if role == "assistant":
return "agent_report"
if written_by == "tool" and evidence_id in accepted:
if role == "tool" and evidence_id in accepted:
return "system_acceptance"
return "tool_output"
@@ -118,7 +118,7 @@ def assemble_resume_packet(
field=_field(message.text) or "unclassified",
text=message.text,
evidence_class=_raw_class(
message.written_by,
message.role,
message.evidence_id,
verified_system_evidence_ids,
),
@@ -345,7 +345,7 @@ def _lexical_raw_rows(
if context_raw._prefers_declared_facts(query.query)
else "1"
)
person_authored = "CASE WHEN e.written_by='person' THEN 1 ELSE 0 END"
person_authored = "CASE WHEN e.role='user' THEN 1 ELSE 0 END"
query_echo = (
"CASE WHEN trim(e.verbatim)=? OR trim(e.verbatim)=? THEN 0 ELSE 1 END"
if context_raw._prefers_declared_facts(query.query)
@@ -507,7 +507,7 @@ def _raw_item(row: sqlite3.Row, start1: int, matched_by: tuple[str, ...]) -> Raw
"source_id": row["source_id"],
"conversation_id": row["conversation_id"],
"role": row["role"],
"written_by": "person" if row["written_by"] == "person" else "agent",
"written_by": "person" if row["role"] == "user" else "agent",
"event_time": event_time.isoformat() if event_time else None,
"spans": [{"start_cp": start, "end_cp": start + len(text), "text": text}],
"matched_by": list(matched_by),
@@ -449,9 +449,7 @@ class HyMemoryBridge:
guards = getattr(client, "_one_memory_write_guards", ())
from one_memory.hy_response_guard import HyReconcileGuard
reconcile_guards = tuple(
guard for guard in guards if isinstance(guard, HyReconcileGuard)
)
reconcile_guards = tuple(guard for guard in guards if isinstance(guard, HyReconcileGuard))
for guard in reconcile_guards:
guard.begin_write()
before = [len(guard.failures) for guard in guards]
@@ -1129,7 +1127,7 @@ class HyMemoryBridge:
self.profile,
api_key=self.api_key,
data_dir=self.data_dir,
http_transport_factory=self._http_transport_factory,
http_transport_factory=self._http_transport_factory,
)
await _joined_thread(initialize)
@@ -1347,9 +1345,7 @@ async def _with_native_mutation_delta(
"""
retired = set(result.retired_memory_ids)
superseded = {
key: tuple(values) for key, values in result.superseded_memory_ids.items()
}
superseded = {key: tuple(values) for key, values in result.superseded_memory_ids.items()}
chat_targets = {
key
for guard in reconcile_guards
@@ -1357,7 +1353,8 @@ async def _with_native_mutation_delta(
if isinstance(key, str) and key
}
for item in raw.get("ops", ()) if isinstance(raw.get("ops"), list) else ():
operations = raw.get("ops")
for item in operations if isinstance(operations, list) else ():
if not isinstance(item, Mapping):
continue
if item.get("action") == "DELETE":
@@ -1365,7 +1362,8 @@ async def _with_native_mutation_delta(
if isinstance(key, str) and key:
retired.add(key)
for item in raw.get("fact_ops", ()) if isinstance(raw.get("fact_ops"), list) else ():
fact_operations = raw.get("fact_ops")
for item in fact_operations if isinstance(fact_operations, list) else ():
if not isinstance(item, Mapping) or item.get("action") != "SUPERSEDE":
continue
old, new = item.get("fact_id"), item.get("superseded_by")
@@ -1386,9 +1384,7 @@ async def _with_native_mutation_delta(
if isinstance(replacement, str) and replacement:
superseded[key] = (replacement,)
elif isinstance(replacement, list):
values = tuple(
value for value in replacement if isinstance(value, str) and value
)
values = tuple(value for value in replacement if isinstance(value, str) and value)
if values:
superseded[key] = values
@@ -7,7 +7,7 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
from one_memory.input_sanitization import sanitize_model_input
@@ -61,25 +61,7 @@ class HyWriteBindings:
seed.workspace,
)
or record.content_kind is not EvidenceContentKind.RAW_CONVERSATION
or (record.role, record.written_by)
not in {
# D98: the same change as the turn assembly. A sub agent's
# question is a `user` message marked written by the
# agent; excluding it here would make the source context
# disagree with what was actually sent.
("user", WrittenBy.PERSON),
# A host that leaves the author blank still sent a turn.
# WorkBuddy records 1078 of them across 1002 conversations,
# and refusing the write over that answered every one of
# those tasks with `model_content_outcome_unknown` on the
# second attempt: a memory that may have been written and
# can never be confirmed. The author is missing, not the
# turn; scope is still checked above.
("user", WrittenBy.UNKNOWN),
("user", WrittenBy.AGENT),
("assistant", WrittenBy.AGENT),
("tool", WrittenBy.TOOL),
}
or record.role not in {"user", "assistant", "tool"}
):
raise ValueError("hy_memory_write_verification_context_scope")
self.all_sources: list[dict[str, Any]] = [
@@ -79,6 +79,8 @@ class EvidenceInput(BaseModel):
source_ordinal: str | None = Field(default=None, min_length=1, max_length=256)
content_kind: EvidenceContentKind = EvidenceContentKind.RAW_CONVERSATION
role: str = Field(min_length=1, max_length=128)
# Legacy import/storage metadata only. Protocol `role` is authoritative;
# this value must not affect identity, admission, scheduling or ownership.
written_by: WrittenBy
created_at: datetime | None = None
verbatim: str = Field(min_length=1, max_length=MAX_VERBATIM_CHARACTERS)
@@ -149,17 +151,22 @@ class EvidenceInput(BaseModel):
)
)
else:
# `role` is the protocol-level speaker identity. `written_by` was a
# parser guess and differed between two imports of the same host
# message (notably sub-agent/user and unknown/user), minting two
# Evidence rows. Keep it out of every new identity.
#
# No workspace and no source_ordinal in the derived key: the hook
# lane has neither (it reports no workspace and a constant ordinal),
# so including either gave the same message a different key per lane
# and stored it twice. Both are record metadata here, not identity.
identity = "\0".join(
(
"derived-v3",
"derived-v4",
self.user_id,
self.source_id,
self.conversation_id,
self.written_by,
self.role,
self.content_hash,
)
)
@@ -168,6 +175,54 @@ class EvidenceInput(BaseModel):
# two rows. The capture time is record metadata, not identity.
return hashlib.sha256(identity.encode()).hexdigest()
@property
def written_by_derived_v3_idempotency_key(self) -> str:
"""The last identity that treated the parser's author guess as canonical.
New builds key by protocol role. During an upgrade this key lets a
rescan find and reuse the row already stored by derived-v3 instead of
inserting the same host message under a second Evidence id.
"""
identity = "\0".join(
(
"derived-v3",
self.user_id,
self.source_id,
self.conversation_id,
self.written_by,
self.content_hash,
)
)
return hashlib.sha256(identity.encode()).hexdigest()
@property
def written_by_derived_v3_idempotency_keys(self) -> tuple[str, ...]:
"""Every legacy author guess this protocol role may have carried."""
guesses = {
"user": (
WrittenBy.PERSON,
WrittenBy.AGENT,
WrittenBy.UNKNOWN,
),
"assistant": (WrittenBy.AGENT, WrittenBy.UNKNOWN, WrittenBy.PERSON),
"tool": (WrittenBy.TOOL, WrittenBy.UNKNOWN),
}.get(self.role, tuple(WrittenBy))
ordered = (self.written_by, *guesses)
keys: list[str] = []
for written_by in dict.fromkeys(ordered):
identity = "\0".join(
(
"derived-v3",
self.user_id,
self.source_id,
self.conversation_id,
written_by,
self.content_hash,
)
)
keys.append(hashlib.sha256(identity.encode()).hexdigest())
return tuple(keys)
@property
def legacy_message_idempotency_key(self) -> str | None:
"""The pre-workspace key for a message-shaped input, when one exists.
@@ -212,7 +267,13 @@ class EvidenceInput(BaseModel):
"""
return self.derived_v2_idempotency_key("derived-v2", workspace_agnostic=False)
def derived_v2_idempotency_key(self, label: str, *, workspace_agnostic: bool) -> str:
def derived_v2_idempotency_key(
self,
label: str,
*,
workspace_agnostic: bool,
written_by: WrittenBy | None = None,
) -> str:
identity = "\0".join(
(
label,
@@ -220,7 +281,7 @@ class EvidenceInput(BaseModel):
self.source_id,
*([] if workspace_agnostic else [self.workspace or "\uffffnull"]),
self.conversation_id,
self.written_by,
written_by or self.written_by,
self.created_at.isoformat() if self.created_at else "",
self.source_ordinal or "",
# v2 hashed the cleaned body; v1 predates raw_verbatim entirely,
@@ -230,6 +291,26 @@ class EvidenceInput(BaseModel):
)
return hashlib.sha256(identity.encode()).hexdigest()
@property
def older_written_by_idempotency_keys(self) -> tuple[str, ...]:
keys: list[str] = []
for written_by in WrittenBy:
keys.append(
self.derived_v2_idempotency_key(
"derived-v1",
workspace_agnostic=True,
written_by=written_by,
)
)
keys.append(
self.derived_v2_idempotency_key(
"derived-v2",
workspace_agnostic=False,
written_by=written_by,
)
)
return tuple(keys)
@property
def evidence_id(self) -> str:
return str(uuid5(EVIDENCE_NAMESPACE, self.idempotency_key))
@@ -249,6 +330,7 @@ class EvidenceRecord(BaseModel):
source_ordinal: str | None
content_kind: EvidenceContentKind
role: str
# Retained while old databases and export formats still carry the column.
written_by: WrittenBy
created_at: datetime | None
verbatim: str
@@ -141,11 +141,18 @@ def parse_codex_record(
if role == "user":
written_by = _user_author(record, payload, session)
# Machine inputs may quote a whole transcript, including user_query tags.
# Session provenance is not proof of who wrote any particular future turn.
if written_by is WrittenBy.PERSON:
# Protocol role is authoritative; host author metadata does not change
# how a user message is cleaned.
if not (
content.startswith(
"The following is the Codex agent history whose request action you are assessing."
)
and ">>> TRANSCRIPT START" in content
and ">>> TRANSCRIPT END" in content
):
content = strip_host_wrappers(content)
if not content:
return ParsedSkip(ScanIssueCode.NO_TEXT)
if not content:
return ParsedSkip(ScanIssueCode.NO_TEXT)
return ParsedMessage(
conversation_id=conversation_from_relative_path(context.relative_path),
message_id=first_string(payload.get("id"), record.get("id")),
@@ -421,10 +421,7 @@ class CursorUnifiedSourceAdapter:
if (
isinstance(event, ScannedEvidence)
and event.evidence.conversation_id in transcript_ids
and not (
event.evidence.written_by is WrittenBy.AGENT
and is_host_generated_user_turn(event.evidence.verbatim)
)
and not is_host_generated_user_turn(event.evidence.verbatim)
):
yield ScanSkip(
cursor=event.cursor,
@@ -449,10 +446,7 @@ def _shadowed_by_transcript(event: ScanEvent, transcript_ids: set[str]) -> ScanE
if (
isinstance(event, ScannedEvidence)
and event.evidence.conversation_id in transcript_ids
and not (
event.evidence.written_by is WrittenBy.AGENT
and is_host_generated_user_turn(event.evidence.verbatim)
)
and not is_host_generated_user_turn(event.evidence.verbatim)
):
return ScanSkip(
cursor=event.cursor,
@@ -57,7 +57,8 @@ _UNANSWERED_PREFIXES = (
def _is_visible_user_row(role: str, written_by: str, verbatim: str) -> bool:
if role != "user" or written_by != WrittenBy.PERSON.value:
del written_by # Legacy storage metadata; protocol role is authoritative.
if role != "user":
return False
return not verbatim.lstrip().lower().startswith(_UNANSWERED_PREFIXES)
@@ -468,16 +469,54 @@ class EvidenceStore:
if anchored is not None:
self._write_origins(connection, anchored, evidence.origin_meta, now)
workspace, project = self._resolve_conversation_attribution(connection, evidence)
connection.execute(
"""
UPDATE evidence
SET workspace = COALESCE(workspace, ?),
project = COALESCE(project, ?),
updated_at = ?
WHERE evidence_id = ?
""",
(workspace, project, now, anchored),
)
existing = connection.execute(
"SELECT role,revision FROM evidence WHERE evidence_id=?",
(anchored,),
).fetchone()
if existing is None:
raise RuntimeError("anchored evidence disappeared")
if str(existing["role"]) != evidence.role:
was_revised = True
revision = int(existing["revision"]) + 1
connection.execute(
"""
UPDATE evidence
SET workspace=COALESCE(workspace,?),
project=COALESCE(project,?),
content_kind=?,role=?,created_at=COALESCE(?,created_at),
verbatim=?,content_hash=?,revision=?,updated_at=?
WHERE evidence_id=?
""",
(
workspace,
project,
evidence.content_kind,
evidence.role,
evidence.created_at.isoformat() if evidence.created_at else None,
evidence.verbatim,
evidence.content_hash,
revision,
now,
anchored,
),
)
self._insert_current_version(
connection,
anchored,
revision=revision,
changed_at=now,
)
else:
connection.execute(
"""
UPDATE evidence
SET workspace=COALESCE(workspace,?),
project=COALESCE(project,?),
updated_at=?
WHERE evidence_id=?
""",
(workspace, project, now, anchored),
)
record = self._get_with_connection(connection, anchored)
if record is None:
raise RuntimeError("anchored evidence disappeared")
@@ -485,7 +524,11 @@ class EvidenceStore:
from one_memory import ingest_journal
ingest_journal.record(connection, anchored, cause="import")
return PutEvidenceResult(record=record, was_created=False, was_revised=False)
return PutEvidenceResult(
record=record,
was_created=False,
was_revised=was_revised,
)
insert = connection.execute(
"""
INSERT INTO evidence (
@@ -803,27 +846,20 @@ class EvidenceStore:
and same_scope[0]["idempotency_key"] != evidence.idempotency_key
):
return str(same_scope[0]["evidence_id"])
elif (legacy_id := claim_legacy_key(evidence.legacy_derived_idempotency_key)) is not None:
return legacy_id
if not evidence.message_id and not evidence.correlation_id:
# BUG-03 moved the derived key off the capture time and onto the
# pre-cleaning text. A replay of a row stored under the old scheme
# has to be recognized as the same message.
replayed = connection.execute(
"""
SELECT evidence_id FROM evidence
WHERE idempotency_key = ? AND user_id = ? AND source_id = ?
AND conversation_id = ?
""",
elif (
legacy_id := next(
(
evidence.prior_derived_idempotency_key,
evidence.user_id,
evidence.source_id,
evidence.conversation_id,
anchored
for key in (
*evidence.written_by_derived_v3_idempotency_keys,
*evidence.older_written_by_idempotency_keys,
)
if (anchored := claim_legacy_key(key)) is not None
),
).fetchone()
if replayed is not None:
return str(replayed["evidence_id"])
None,
)
) is not None:
return legacy_id
if (
EvidenceOrigin.HOOK in evidence.origins
and evidence.message_id is None
@@ -838,7 +874,7 @@ class EvidenceStore:
ON hook.evidence_id=e.evidence_id AND hook.origin='hook'
WHERE e.user_id=? AND e.source_id=? AND e.conversation_id=?
AND (e.workspace IS ? OR e.workspace IS NULL OR ? IS NULL)
AND e.role=? AND e.written_by=? AND e.content_hash=?
AND e.role=? AND e.content_hash=?
AND e.message_id IS NOT NULL AND hook.evidence_id IS NULL
AND ABS((julianday(e.created_at)-julianday(?))*86400.0) <= 5.0
ORDER BY ABS((julianday(e.created_at)-julianday(?))*86400.0), e.evidence_id
@@ -851,7 +887,6 @@ class EvidenceStore:
evidence.workspace,
evidence.workspace,
evidence.role,
evidence.written_by,
evidence.content_hash,
evidence.created_at.isoformat(),
evidence.created_at.isoformat(),
@@ -872,7 +907,7 @@ class EvidenceStore:
AND (e.workspace IS ? OR e.workspace IS NULL OR ? IS NULL)
-- Unary plus keeps the coverage index from looking better than
-- the message index, which narrows to this exact text.
AND e.role = ? AND +e.written_by = ? AND e.content_hash = ?
AND e.role = ? AND e.content_hash = ?
AND e.message_id IS NULL
""",
(
@@ -882,7 +917,6 @@ class EvidenceStore:
evidence.workspace,
evidence.workspace,
evidence.role,
evidence.written_by,
evidence.content_hash,
),
).fetchall()
@@ -1247,7 +1281,7 @@ class EvidenceStore:
rows = connection.execute(
"""
SELECT evidence_id FROM evidence
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
WHERE user_id = 'local' AND role = 'user'
ORDER BY imported_at, evidence_id
LIMIT ?
""",
@@ -1301,7 +1335,7 @@ class EvidenceStore:
rows = connection.execute(
f"""
SELECT evidence_id FROM evidence
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
WHERE user_id = 'local' AND role = 'user'
AND ({predicates})
ORDER BY imported_at, evidence_id
LIMIT ?
@@ -1333,7 +1367,7 @@ class EvidenceStore:
rows = connection.execute(
f"""
SELECT evidence_id FROM evidence
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
WHERE user_id = 'local' AND role = 'user'
AND workspace IS NOT NULL
AND ({predicates})
ORDER BY imported_at, evidence_id
@@ -1476,7 +1510,7 @@ class EvidenceStore:
)
values: list[object] = [user_id, *(source_ids or ())]
visible_user = """
role='user' AND written_by='person' AND NOT (
role='user' AND NOT (
lower(ltrim(verbatim)) LIKE '<codex_internal_context%' OR
lower(ltrim(verbatim)) LIKE '<environment_context>%' OR
lower(ltrim(verbatim)) LIKE '<heartbeat>%' OR
@@ -1485,7 +1519,7 @@ class EvidenceStore:
)
"""
selected_visible_user = """
e.role='user' AND e.written_by='person' AND NOT (
e.role='user' AND NOT (
lower(ltrim(e.verbatim)) LIKE '<codex_internal_context%' OR
lower(ltrim(e.verbatim)) LIKE '<environment_context>%' OR
lower(ltrim(e.verbatim)) LIKE '<heartbeat>%' OR
@@ -1638,7 +1672,7 @@ class EvidenceStore:
if seed is None:
return ConversationTurnPage((), {}, 0, offset, limit)
visible_user = """
role='user' AND written_by='person' AND NOT (
role='user' AND NOT (
lower(ltrim(verbatim)) LIKE '<codex_internal_context%' OR
lower(ltrim(verbatim)) LIKE '<environment_context>%' OR
lower(ltrim(verbatim)) LIKE '<heartbeat>%' OR
@@ -1731,7 +1765,7 @@ class EvidenceStore:
AND first.conversation_id = evidence.conversation_id
AND first.workspace IS evidence.workspace
AND (
(first.role = 'user' AND first.written_by = 'person')
first.role = 'user'
OR first.content_kind = 'memory_content'
)
ORDER BY julianday(COALESCE(first.created_at, first.imported_at)),
@@ -2125,8 +2159,6 @@ class EvidenceStore:
(
existing["content_kind"] != evidence.content_kind,
existing["role"] != evidence.role,
evidence.written_by is not WrittenBy.UNKNOWN
and existing["written_by"] != evidence.written_by,
existing["content_hash"] != evidence.content_hash,
)
)
@@ -85,15 +85,14 @@ def _state(connection: sqlite3.Connection, evidence_id: str) -> dict[str, str |
value = evidence.model_dump(mode="json")
# The stored normalized content_hash is insufficient for exact-content leases.
content = _digest(value["verbatim"])
material = _digest(
{key: value[key] for key in ("verbatim", "role", "written_by", "content_kind")}
)
material = _digest({key: value[key] for key in ("verbatim", "role", "content_kind")})
origins = _digest(value["origin_meta"])
identity = _digest(
{
key: item
for key, item in value.items()
if key not in {"verbatim", "origin_meta", "imported_at", "updated_at"}
if key
not in {"verbatim", "written_by", "origin_meta", "imported_at", "updated_at"}
}
)
return dict(
@@ -210,7 +210,8 @@ def record_capture_rejection(
payload: dict[str, object] = {"source_id": source_id, "reason": reason.value}
if marker.exists() and now - marker.stat().st_mtime < window_seconds:
payload = json.loads(marker.read_text(encoding="utf8"))
payload["count"] = int(payload.get("count", 1)) + 1
previous = payload.get("count", 1)
payload["count"] = int(previous) + 1 if isinstance(previous, (str, int, float)) else 2
else:
payload["count"] = 1
payload["window_started_at"] = datetime.fromtimestamp(now, UTC).isoformat()
@@ -235,7 +236,7 @@ def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed
):
if (
record in chunk
and any(item.role == "user" and item.written_by is WrittenBy.PERSON for item in chunk)
and any(item.role == "user" for item in chunk)
and any(item.role == "assistant" for item in chunk)
):
intake.submit_records(chunk, priority="live")
@@ -160,7 +160,9 @@ class MatterDiscovery:
task_by_id: dict[str, MemoryTask] = {}
for chunk in chunks:
for record in chunk:
task = await asyncio.to_thread(processor.tasks.task_for_evidence, record.evidence_id)
task = await asyncio.to_thread(
processor.tasks.task_for_evidence, record.evidence_id
)
if task is not None and task.status is MemoryTaskStatus.COMPLETED:
task_by_id[task.task_id] = task
tasks = tuple(sorted(task_by_id.values(), key=lambda task: (task.created_at, task.task_id)))
@@ -270,7 +270,7 @@ async def reconcile_current(
(
r
for r in records
if r.written_by == "person"
if r.role == "user"
and r.created_at
and (r.source_id, r.conversation_id) in related
),
@@ -51,14 +51,19 @@ def validate_claim(claim: Claim, records: Mapping[str, EvidenceRecord | SampleRe
text = record.text if isinstance(record, SampleRecord) else record.verbatim
if not citation.quote.strip() or citation.quote not in text:
raise ValueError("citation quote is not in the source")
authors.add(str(record.written_by))
author = {
"user": "person",
"assistant": "agent",
"tool": "tool",
}.get(record.role, "unknown")
authors.add(author)
created = record.created_at
citations.append(
citation.model_copy(
update={
"revision": record.revision,
"source_id": record.source_id,
"written_by": str(record.written_by),
"written_by": author,
"created_at": created.isoformat() if isinstance(created, datetime) else created,
}
)
@@ -290,7 +290,7 @@ def read_memory_library(
COALESCE(
MIN(
CASE WHEN e.role = 'user'
AND e.written_by IN ('person', 'user')
AND e.role = 'user'
THEN e.created_at END
),
MIN(e.created_at)
@@ -53,7 +53,7 @@ from one_memory.hy_memory_bridge import (
from one_memory.hy_runtime_config import hy_memory_version
from one_memory.hy_source_context import HySourceContext, HySupportedMemory
from one_memory.imports.access import opaque_workspace_id
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord
from one_memory.imports.runner import live_sync_arrival
from one_memory.imports.storage import EvidenceStore
from one_memory.input_sanitization import sanitize_model_input
@@ -187,7 +187,6 @@ UNIFIED_HY_AGENT_ID = "one-memory"
PRE_SEND_REFUSAL_CODES = frozenset(
{
"hy_memory_input_exceeds_window",
"hy_memory_write_verification_context_scope",
}
)
_BACKFILL_BATCH_SIZE = 8
@@ -297,6 +296,7 @@ class MemoryTaskRunner:
*,
timeout: float | None = None,
before_hy_invocation: Callable[[], Awaitable[None]] | None = None,
input_records: tuple[EvidenceRecord, ...] | None = None,
) -> MemoryTask:
started = monotonic()
diagnostic_event(
@@ -307,13 +307,28 @@ class MemoryTaskRunner:
task_id=task_id,
)
try:
body = (
self._run_without_content_permission(task_id)
if before_hy_invocation is None
else self._run_without_content_permission(
task_id, before_hy_invocation=before_hy_invocation
if input_records is None:
body = (
self._run_without_content_permission(task_id)
if before_hy_invocation is None
else self._run_without_content_permission(
task_id,
before_hy_invocation=before_hy_invocation,
)
)
else:
body = (
self._run_without_content_permission(
task_id,
input_records=input_records,
)
if before_hy_invocation is None
else self._run_without_content_permission(
task_id,
before_hy_invocation=before_hy_invocation,
input_records=input_records,
)
)
)
# The ceiling bounds how long this task holds its harness's slot:
# a harness is serial, so one hung task starves every turn behind
# it. It is insurance against a single bad call, not a way to run
@@ -398,6 +413,7 @@ class MemoryTaskRunner:
task_id: str,
*,
before_hy_invocation: Callable[[], Awaitable[None]] | None = None,
input_records: tuple[EvidenceRecord, ...] | None = None,
) -> MemoryTask:
# Every store call here is synchronous, and several are writes that
# wait on the write lock for up to five seconds. Made from the loop,
@@ -451,7 +467,9 @@ class MemoryTaskRunner:
step_started = monotonic()
frozen = await in_the_background(self.tasks.inputs, task.task_id)
chunk: tuple[EvidenceRecord, ...] | None
if frozen:
if input_records is not None:
chunk = input_records
elif frozen:
try:
chunk = await in_the_background(self._frozen_chunk, task)
except HyMemoryBridgeError as error:
@@ -484,8 +502,11 @@ class MemoryTaskRunner:
self._source_context, task, request.source_context
)
request = request.model_copy(update={"source_context": context})
uses_frozen_input = bool(frozen) or input_records is not None
grant_token = (
await in_the_background(self._grant_token, task, chunk[0]) if frozen else None
await in_the_background(self._grant_token, task, chunk[0])
if uses_frozen_input
else None
)
memory_contents = tuple(
sanitize_model_input(record.verbatim)
@@ -528,19 +549,24 @@ class MemoryTaskRunner:
agent_id=request.agent_id,
),
)
if frozen:
await in_the_background(self._frozen_chunk, task)
if uses_frozen_input:
if input_records is None:
await in_the_background(self._frozen_chunk, task)
if grant_token != await in_the_background(self._grant_token, task, chunk[0]):
raise HyMemoryBridgeError("hy_memory_source_changed")
if request.source_context is not None:
await in_the_background(self._check_source_context, request.source_context)
await in_the_background(
self._check_source_context,
request.source_context,
skip_current=input_records is not None,
)
# One thread hop for the whole batch rather than a dozen writes
# made from the loop.
step_started = monotonic()
await in_the_background(
self._record_outputs,
task=task,
chunk=self._original_records(chunk),
chunk=chunk,
outputs=outputs,
request=request,
derivation=derivation,
@@ -778,8 +804,18 @@ class MemoryTaskRunner:
raise HyMemoryBridgeError("hy_memory_evidence_changed") from error
return context
def _check_source_context(self, context: HySourceContext) -> None:
for original in (*context.current, *context.history, *context.supporting):
def _check_source_context(
self,
context: HySourceContext,
*,
skip_current: bool = False,
) -> None:
checked = (
(*context.history, *context.supporting)
if skip_current
else (*context.current, *context.history, *context.supporting)
)
for original in checked:
record = self.evidence.get(original.evidence_id)
if record is None or not matches_evidence_ref(record, evidence_ref(original)):
raise HyMemoryBridgeError("hy_memory_evidence_changed")
@@ -880,7 +916,7 @@ class MemoryTaskRunner:
# HY's own attribution. An undecided memory counts as the person's:
# HY's prompt defaults that way.
from_agent = output.owner == "agent" or all(
record.written_by is WrittenBy.AGENT
record.role == "assistant"
and record.content_kind is EvidenceContentKind.RAW_CONVERSATION
for record in support
)
@@ -1063,11 +1099,7 @@ class MemoryTaskRunner:
record.content_kind is EvidenceContentKind.RAW_CONVERSATION for record in chunk
)
source = next(
(
record
for record in chunk
if record.role == "user" and record.written_by is WrittenBy.PERSON
),
(record for record in chunk if record.role == "user"),
chunk[0],
)
request_turns = tuple(
@@ -1076,12 +1108,9 @@ class MemoryTaskRunner:
if (
record.content_kind is EvidenceContentKind.RAW_CONVERSATION
and (
# D98: any `user` message goes, whoever the parser says
# wrote it. A sub agent's question is ordinary
# user-assistant data; the author field cannot tell who
# wrote a message, and filtering on it dropped the sub
# agent's turns. `written_by` still labels the display and
# decides owner — it no longer decides what is sent.
# Protocol role is authoritative. A sub-agent question is
# an ordinary user message regardless of a host parser's
# former author guess.
record.role == "user"
# The assistant's side goes too. Without it 「就按你说的那个来」
# arrives with nothing to resolve it against, and HY's own
@@ -1092,7 +1121,7 @@ class MemoryTaskRunner:
or record.role == "assistant"
# Tool calls go as they happened; whether they make the
# turn a coding turn is HY's call, not ours.
or (record.role == "tool" and record.written_by is WrittenBy.TOOL)
or record.role == "tool"
)
)
or (not has_raw and record.content_kind is EvidenceContentKind.MEMORY_CONTENT)
@@ -1376,7 +1405,6 @@ class MemoryProcessingSupervisor:
async with self._explicit_processing_lock(records):
if self._sedimentation_stopped():
raise HyMemoryBridgeError("model_sedimentation_disabled")
records = await in_the_background(self._original_records, records)
active = await in_the_background(self.profiles.active, self.secrets)
if active is None or (profile_id is not None and profile_id != active.profile_id):
raise HyMemoryBridgeError("model_profile_not_active")
@@ -1391,13 +1419,6 @@ class MemoryProcessingSupervisor:
raise HyMemoryBridgeError("hy_memory_source_disabled")
conversations: dict[tuple[str, str, str | None], list[EvidenceRecord]] = {}
for record in records:
stored = await in_the_background(self.evidence.get, record.evidence_id)
if (
stored is None
or stored.revision != record.revision
or stored.content_hash != record.content_hash
):
raise ValueError("hy_memory_evidence_changed")
conversations.setdefault(
(record.source_id, record.conversation_id, record.workspace), []
).append(record)
@@ -1414,7 +1435,7 @@ class MemoryProcessingSupervisor:
for conversation in conversations.values()
for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True)
]
prepared = []
prepared: list[tuple[MemoryTask, tuple[EvidenceRecord, ...]]] = []
for chunk in chunks:
task = await in_the_background(
self._enqueue_chunk,
@@ -1422,12 +1443,12 @@ class MemoryProcessingSupervisor:
active,
freeze_inputs=True,
)
prepared.append(task)
prepared.append((task, chunk))
runner = self.runner_factory()
completed = []
if on_progress:
on_progress(tuple(prepared), 0)
for task in prepared:
on_progress(tuple(task for task, _chunk in prepared), 0)
for task, chunk in prepared:
if task.status is not MemoryTaskStatus.COMPLETED:
# Another importer may already own this task. Never race its claim.
task = await in_the_background(self.tasks.require, task.task_id)
@@ -1456,23 +1477,41 @@ class MemoryProcessingSupervisor:
work_id=task.task_id,
scope_key=chunk[0].source_id,
):
task = (
await runner.run(task.task_id, timeout=timeout)
if before_hy_invocation is None
else await runner.run(
task.task_id,
timeout=timeout,
before_hy_invocation=before_hy_invocation,
if isinstance(runner, MemoryTaskRunner):
task = (
await runner.run(
task.task_id,
timeout=timeout,
input_records=chunk,
)
if before_hy_invocation is None
else await runner.run(
task.task_id,
timeout=timeout,
before_hy_invocation=before_hy_invocation,
input_records=chunk,
)
)
else:
task = (
await runner.run(task.task_id, timeout=timeout)
if before_hy_invocation is None
else await runner.run(
task.task_id,
timeout=timeout,
before_hy_invocation=before_hy_invocation,
)
)
)
finally:
_unmark_direct_task(self.tasks.database_path, task.task_id)
if task.status is not MemoryTaskStatus.COMPLETED:
raise HyMemoryBridgeError(task.last_error_code or "hy_memory_incomplete")
await in_the_background(runner._frozen_chunk, task)
completed.append(task)
if on_progress:
on_progress(tuple(prepared), len(completed))
on_progress(
tuple(prepared_task for prepared_task, _chunk in prepared),
len(completed),
)
expected = {record.evidence_id for record in records if _is_hy_turn_record(record)}
actual = {
row["evidence_id"] for task in completed for row in self.tasks.inputs(task.task_id)
@@ -3261,9 +3300,7 @@ class MemoryProcessingSupervisor:
if not tail:
return False
last = tail[-1]
return bool(
last.role == "user" and last.written_by is WrittenBy.PERSON and _opens_turn(last)
)
return bool(last.role == "user" and _opens_turn(last))
def _admitted_chunks(
self,
@@ -3608,7 +3645,6 @@ def _is_memory_note(record: EvidenceRecord) -> bool:
meta = record.origin_meta.get(EvidenceOrigin.MANUAL_IMPORT)
return (
record.source_id == "one-memory-manual"
and record.written_by is WrittenBy.PERSON
and meta is not None
and meta.raw_meta.get("input_mode") == "direct_memory_note"
)
@@ -3903,14 +3939,12 @@ def _merge_historical_chunks(
def _chunk_input_ref(chunk: tuple[EvidenceRecord, ...]) -> str:
person_records = tuple(
item for item in chunk if item.role == "user" and item.written_by is WrittenBy.PERSON
)
return person_records[-1].evidence_id if person_records else chunk[-1].evidence_id
user_records = tuple(item for item in chunk if item.role == "user")
return user_records[-1].evidence_id if user_records else chunk[-1].evidence_id
def _opens_turn(record: EvidenceRecord) -> bool:
return record.role == "user" and record.written_by is WrittenBy.PERSON
return record.role == "user"
def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
@@ -3927,10 +3961,8 @@ def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
def _is_hy_turn_record(record: EvidenceRecord) -> bool:
"""What HY receives of a turn: the input turns and the tool calls made in it."""
if record.role == "tool":
return (
record.written_by is WrittenBy.TOOL
and record.content_kind is EvidenceContentKind.RAW_CONVERSATION
and bool(record.verbatim.strip())
return record.content_kind is EvidenceContentKind.RAW_CONVERSATION and bool(
record.verbatim.strip()
)
return _is_hy_input_turn(record)
@@ -3947,11 +3979,8 @@ def _is_hy_input_turn(record: EvidenceRecord) -> bool:
# from a model.
if record.role == "assistant":
return True
# D98: the author no longer decides. A sub agent's question arrives as a
# `user` message marked written by the agent — it is ordinary
# user-assistant data, and dropping it sent Claude Code transcripts to HY
# with every question but the first missing. `written_by` labels the
# display and decides owner; it cannot tell who actually wrote a message.
# A sub-agent question is still a protocol `user` message. Host-specific
# author guesses do not decide whether it reaches HY.
if record.role != "user":
return False
if source.casefold().startswith(_HARNESS_PREFIXES):
@@ -27,7 +27,9 @@ from datetime import datetime
from enum import StrEnum
from typing import Annotated, Final, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
from one_memory.imports.models import EvidenceRecord
#: Bumped when a derivation rule changes, so old and new ids never compare equal.
SCHEMA_VERSION: Final[str] = "ms1"
@@ -167,7 +169,6 @@ class CanonicalMessage(BaseModel):
message_id: str | None = Field(default=None, min_length=1, max_length=512)
ordinal: int = Field(ge=0)
role: str = Field(min_length=1, max_length=128)
written_by: str = Field(min_length=1, max_length=64)
content: str = Field(min_length=1)
workspace: str | None = Field(default=None, max_length=1024)
@@ -214,12 +215,13 @@ class WriteRequest(BaseModel):
source_id: str | None = Field(default=None, min_length=1, max_length=128)
batch_identity: BatchIdentity
evidence_ids: list[str] = Field(min_length=1)
# New jobs freeze the complete input in their own payload. The default is
# solely for reading pre-upgrade rows whose payload only carried ids.
input_records: tuple[EvidenceRecord, ...] = ()
idempotency_key: str = Field(min_length=1, max_length=256)
priority: Literal["live", "normal"] = "normal"
event_at: datetime | None = None
time_quality: Literal["exact", "anchored", "session", "approximate"] = (
"approximate"
)
time_quality: Literal["exact", "anchored", "session", "approximate"] = "approximate"
@field_validator("evidence_ids")
@classmethod
@@ -232,6 +234,16 @@ class WriteRequest(BaseModel):
raise ValueError("evidence_ids must be unique")
return value
@field_validator("input_records")
@classmethod
def _input_records_match_ids(
cls, value: tuple[EvidenceRecord, ...], info: ValidationInfo
) -> tuple[EvidenceRecord, ...]:
evidence_ids = info.data.get("evidence_ids")
if value and evidence_ids != [record.evidence_id for record in value]:
raise ValueError("input_records must match evidence_ids in order")
return value
@property
def request_key(self) -> str:
"""Stable dedup key. Same batch + same intent must not enqueue twice."""
@@ -30,7 +30,7 @@ from uuid import uuid4
from one_memory.background_work import in_the_background
from one_memory.database_access import main_database
from one_memory.diagnostic_log import diagnostic_event
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
from one_memory.imports.storage import EvidenceStore
from one_memory.memory_server.contracts import WriteRequest
from one_memory.memory_server.data_safety import MIN_FREE_BYTES # noqa: F401 - re-exported
@@ -250,16 +250,9 @@ MODEL_NOT_READY_CODES = frozenset(
#: `failed`, not `unknown` (which would wrongly imply a possibly-applied
#: write) and not a retry (which would fail identically).
#:
#: `hy_memory_write_verification_context_scope` belongs here for the same
#: reason: `HyWriteBindings` is built inside `ingest` *before* the payload
#: reaches HY, and it is the only thing that raises this code. The durable
#: marker is already set by then, so classifying it as `unknown` claimed a
#: write may have landed when HY was never called — a sticky row that blocked
#: the queue behind it and left no reason to act on. See issue #44.
PRE_SEND_REFUSAL_CODES = frozenset(
{
"hy_memory_input_exceeds_window",
"hy_memory_write_verification_context_scope",
}
)
@@ -290,11 +283,6 @@ _VALID_JOB_STATES = frozenset(
#: A row in one of these states can never be moved back to a live one.
_TERMINAL_WRITE_STATES = frozenset({COMPLETED, FAILED, DEAD, CANCELLED})
#: A job in one of these states can never run again, so its payload is dead
#: weight. `unknown` is deliberately absent: it is non-terminal and still needs
#: its intent for reconciliation. `running` is obviously still live.
TERMINAL_PAYLOAD_STATES = (COMPLETED, FAILED, DEAD, CANCELLED)
#: Columns a current schema declares but an older `memory_write_jobs` may lack.
#: `CREATE TABLE IF NOT EXISTS` is a no-op for an existing table, so without
#: this an upgraded install would keep an older queue that every claim/lease
@@ -404,6 +392,42 @@ def _matches_payload_except_priority(stored: str, payload: str) -> bool:
return old == new
def _matches_legacy_reference_replay(stored: str, payload: str) -> bool:
"""Match an old id-only caller to the same job's frozen payload."""
try:
old = json.loads(stored)
new = json.loads(payload)
except (TypeError, ValueError):
return False
if not isinstance(old, dict) or not isinstance(new, dict):
return False
old_inputs = old.get("input_records")
new_inputs = new.get("input_records")
if bool(old_inputs) == bool(new_inputs):
return False
old.pop("input_records", None)
new.pop("input_records", None)
return old == new
def _matches_reaped_legacy_payload(stored: str, payload: str) -> bool:
"""Compare a current frozen request with a pre-snapshot reaped payload."""
try:
old = json.loads(stored)
current = json.loads(payload)
except (TypeError, ValueError):
return False
if (
not isinstance(old, dict)
or not isinstance(current, dict)
or not isinstance(old.get(REAPED_PAYLOAD_MARKER), str)
):
return False
current.pop("input_records", None)
legacy_payload = json.dumps(current, sort_keys=True)
return bool(old[REAPED_PAYLOAD_MARKER] == _payload_digest(legacy_payload))
class DurableQueueSchemaUnavailableError(RuntimeError):
"""The durable queue schema has not been created yet.
@@ -456,6 +480,7 @@ class DurableWriteQueue:
if not initialize:
return
with main_database(database_path).write() as connection, connection:
connection.row_factory = sqlite3.Row
# First create the base table if absent. Then widen an older table
# before replaying indexes that reference the new columns; a bare
# CREATE INDEX would fail on a pre-priority queue.
@@ -491,8 +516,7 @@ class DurableWriteQueue:
"UPDATE memory_write_jobs SET source_key=session_key WHERE source_key=''"
)
legacy_rows = connection.execute(
"SELECT job_id FROM memory_write_jobs WHERE queue_seq=0 "
"ORDER BY created_at, rowid"
"SELECT job_id FROM memory_write_jobs WHERE queue_seq=0 ORDER BY created_at, rowid"
).fetchall()
next_seq = int(
connection.execute(
@@ -539,10 +563,9 @@ class DurableWriteQueue:
)
return
evidence_columns = {
str(row[1])
for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
str(row[1]) for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
}
if not {"created_at", "imported_at", "role", "written_by"} <= evidence_columns:
if not {"created_at", "imported_at", "role"} <= evidence_columns:
connection.execute(
"UPDATE memory_write_jobs SET event_at=created_at WHERE event_at IS NULL"
)
@@ -556,7 +579,7 @@ class DurableWriteQueue:
FROM memory_write_job_evidence AS linked
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
WHERE linked.job_id = memory_write_jobs.job_id
AND e.role = 'user' AND e.written_by = 'person'
AND e.role = 'user'
),
(
SELECT MIN(COALESCE(e.created_at, e.imported_at))
@@ -666,6 +689,7 @@ class DurableWriteQueue:
replacing the earlier intent.
"""
self._refuse_if_below_reserve()
request = self._freeze_request_inputs(request)
payload = json.dumps(
request.model_dump(
mode="json",
@@ -691,16 +715,22 @@ class DurableWriteQueue:
(key,),
).fetchone()
if existing is not None:
# A replay of work that already finished is a hit, not a
# conflict. Terminal rows store a reaped digest instead of the
# original payload, and the caller's key is derived from the
# evidence's revision, which can move on between replays while
# the payload's evidence ids stay the same. Refusing that pair
# turned every startup replay of one settled job into an
# exception: 236 of them in a single afternoon, all one key.
already_settled = (
str(existing["state"]) == COMPLETED and int(existing["hy_invoked"]) == 1
)
stored_payload = str(existing["payload_json"])
priority_replay = _matches_payload_except_priority(stored_payload, payload)
legacy_reaped_match = bool(
already_settled
and _matches_reaped_legacy_payload(stored_payload, payload)
)
if (
not _matches_stored_payload(stored_payload, payload)
and not priority_replay
and not _matches_legacy_reference_replay(stored_payload, payload)
and not legacy_reaped_match
):
raise ValueError(f"idempotency key {key} already exists with different content")
if already_settled:
return DurableWriteRecord(
job_id=str(existing["job_id"]),
@@ -711,14 +741,6 @@ class DurableWriteQueue:
updated_at=str(existing["updated_at"]),
detail="already completed; replay treated as a hit",
)
priority_replay = _matches_payload_except_priority(
str(existing["payload_json"]), payload
)
if (
not _matches_stored_payload(str(existing["payload_json"]), payload)
and not priority_replay
):
raise ValueError(f"idempotency key {key} already exists with different content")
if priority_replay and request.priority == "live":
connection.execute(
"UPDATE memory_write_jobs SET priority='live', updated_at=? "
@@ -803,9 +825,7 @@ class DurableWriteQueue:
0,
None,
event_at,
request.time_quality
if request.event_at is not None
else resolved_quality,
request.time_quality if request.event_at is not None else resolved_quality,
now,
now,
),
@@ -835,34 +855,36 @@ class DurableWriteQueue:
updated_at=now,
)
def _event_time_for_request(
self, request: WriteRequest
) -> tuple[datetime | None, str]:
if self.evidence is None:
return None, "approximate"
records = [
record
for evidence_id in request.evidence_ids
if (record := self.evidence.get(evidence_id)) is not None
]
def _freeze_request_inputs(self, request: WriteRequest) -> WriteRequest:
if request.input_records or self.evidence is None:
return request
records: list[EvidenceRecord] = []
for evidence_id in request.evidence_ids:
record = self.evidence.get(evidence_id)
if record is None:
return request
records.append(record)
return request.model_copy(update={"input_records": tuple(records)})
def _event_time_for_request(self, request: WriteRequest) -> tuple[datetime | None, str]:
records = list(request.input_records)
if not records and self.evidence is not None:
records = [
record
for evidence_id in request.evidence_ids
if (record := self.evidence.get(evidence_id)) is not None
]
if not records:
return None, "approximate"
opener = next(
(
record
for record in records
if record.role == "user"
and record.written_by is WrittenBy.PERSON
),
(record for record in records if record.role == "user"),
records[0],
)
if opener.created_at is None:
return opener.imported_at, "approximate"
origin = opener.origin_meta.get(EvidenceOrigin.FILE)
basis = origin.raw_meta.get("time_basis") if origin is not None else None
turn_event_at = (
origin.raw_meta.get("turn_event_at") if origin is not None else None
)
turn_event_at = origin.raw_meta.get("turn_event_at") if origin is not None else None
if isinstance(turn_event_at, str):
try:
return datetime.fromisoformat(turn_event_at), "exact"
@@ -981,11 +1003,55 @@ class DurableWriteQueue:
raise ReconcileSchemaUnavailableError(
"evidence table is unavailable; run migrations before reconcile"
)
where = (
"WHERE evidence.imported_at IS NOT NULL "
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence ref "
"WHERE ref.evidence_id=evidence.evidence_id)"
has_versions = (
connection.execute(
"SELECT 1 FROM sqlite_master "
"WHERE type='table' AND name='evidence_versions'"
).fetchone()
is not None
)
evidence_columns = {
str(row[1])
for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
}
if {"revision", "content_hash", "updated_at"} <= evidence_columns:
revision_time = (
"COALESCE((SELECT version.changed_at FROM evidence_versions version "
"WHERE version.evidence_id=evidence.evidence_id "
"AND version.revision=evidence.revision),"
"evidence.updated_at,evidence.imported_at)"
if has_versions
else "COALESCE(evidence.updated_at,evidence.imported_at)"
)
where = (
"WHERE evidence.imported_at IS NOT NULL "
"AND NOT EXISTS ("
"SELECT 1 FROM memory_write_job_evidence linked "
"JOIN memory_write_jobs job ON job.job_id=linked.job_id, "
"json_each(job.payload_json,'$.input_records') frozen "
"WHERE linked.evidence_id=evidence.evidence_id "
"AND json_extract(frozen.value,'$.evidence_id')=evidence.evidence_id "
"AND json_extract(frozen.value,'$.revision')=evidence.revision "
"AND json_extract(frozen.value,'$.content_hash')=evidence.content_hash"
") "
# Pre-snapshot builds only recorded an Evidence id. Treat
# that as coverage only when its Job was created after this
# exact revision.
"AND NOT EXISTS ("
"SELECT 1 FROM memory_write_job_evidence legacy "
"JOIN memory_write_jobs legacy_job ON legacy_job.job_id=legacy.job_id "
"WHERE legacy.evidence_id=evidence.evidence_id "
"AND COALESCE(json_array_length("
"json_extract(legacy_job.payload_json,'$.input_records')),0)=0 "
f"AND legacy_job.created_at>=({revision_time}))"
)
else:
# A genuinely old/minimal schema has no revision identity.
where = (
"WHERE evidence.imported_at IS NOT NULL "
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence legacy "
"WHERE legacy.evidence_id=evidence.evidence_id)"
)
params: tuple[Any, ...] = ()
cursor_clause = ""
if after_cursor is not None:
@@ -1285,45 +1351,10 @@ class DurableWriteQueue:
return state
def clear_terminal_payloads(self, *, limit: int = 512) -> int:
"""Replace terminal rows' request bodies with a content digest.
A completed/failed/dead/cancelled row can never run again, so the full
serialized `WriteRequest` is unbounded dead weight. A digest keeps the
idempotency contract intact: a replay of the same intent still resolves
to the same row, while the same key with different content still
conflicts. `unknown` is left alone because it is non-terminal and its
intent is needed for reconciliation.
Bounded by `limit` so a large backlog drains over several calls instead
of one long write on the single writer.
"""
"""Deprecated compatibility hook; durable inputs are immutable audit data."""
if type(limit) is not int or limit < 1:
raise ValueError("limit must be positive")
placeholders = ",".join("?" for _ in TERMINAL_PAYLOAD_STATES)
with (
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
connection,
):
rows = connection.execute(
"SELECT job_id, payload_json FROM memory_write_jobs "
f"WHERE state IN ({placeholders}) "
"AND instr(payload_json, ?) = 0 "
"LIMIT ?",
(*TERMINAL_PAYLOAD_STATES, REAPED_PAYLOAD_MARKER, limit),
).fetchall()
if not rows:
return 0
for row in rows:
connection.execute(
"UPDATE memory_write_jobs SET payload_json=? "
f"WHERE job_id=? AND state IN ({placeholders})",
(
_reaped_payload(_payload_digest(str(row["payload_json"]))),
str(row["job_id"]),
*TERMINAL_PAYLOAD_STATES,
),
)
return len(rows)
return 0
def readmit_parked(self, *, states: tuple[str, ...] = PARKED_STATES) -> int:
"""Move parked jobs back to `pending` for a fresh claim.
@@ -1870,9 +1901,12 @@ class DurableWriteRegistry:
def evidence_for_request(store: EvidenceStore, request: WriteRequest) -> tuple[EvidenceRecord, ...]:
"""The Evidence a stored write points at, in the request's own order.
A write may only point at Evidence that still exists; a missing id is a
rejection rather than a silent partial send.
Current jobs are self-contained: their immutable records are the model
input even if a Harness later rewinds or edits its live transcript. The
store lookup remains only for payloads written by pre-upgrade builds.
"""
if request.input_records:
return request.input_records
records = []
for evidence_id in request.evidence_ids:
record = store.get(evidence_id)
@@ -1979,10 +2013,14 @@ async def execute_job(
# Never persist the raw message: it may be a provider response or the
# user's own text, and `last_error` is shown over HTTP as `detail`.
code = safe_error_code(error)
if code in {
"model_content_source_disabled",
"model_sedimentation_disabled",
} and not hy_invoked:
if (
code
in {
"model_content_source_disabled",
"model_sedimentation_disabled",
}
and not hy_invoked
):
await in_the_background(queue.release_unstarted, job_id, owner=owner)
return
# The task layer classifies these before any bridge is opened, so HY
@@ -1994,14 +2032,8 @@ async def execute_job(
# Everything else may have crossed into HY, and HY has no idempotency
# key, so it is `unknown` rather than a blind retry.
if code in PRE_SEND_REFUSAL_CODES:
# Provably before the bridge, even though the marker is already
# set: the write bindings are built — and the input window
# checked — before the payload reaches HY, and this code is raised
# from that construction alone. Nothing was sent, so this is
# terminal rather than unknown, and the marker is cleared so
# reclaim does not treat a row HY never saw as possibly-sent.
# Otherwise a refused source context pinned the queue behind it
# with no reason shown and no way forward. See issue #44.
# The frozen input window is checked before the payload reaches
# HY. This one refusal therefore has a proven pre-send boundary.
state, invoked_marker = FAILED, False
elif hy_invoked:
# Error names do not prove execution phase: permission and
@@ -2103,7 +2135,6 @@ class DurableWriteDrain:
#: sustained backlog must not keep finished rows' request bodies in the
#: business database indefinitely. Its own gate so neither interval can
#: starve the other.
self._last_terminal_reap: datetime | None = None
#: `waiting_for_model` re-admission is also time-based recoverability:
#: a model outage clears on its own, and a queue that never goes idle
#: must still re-admit the write once the wait has elapsed.
@@ -2154,7 +2185,7 @@ class DurableWriteDrain:
now: datetime | None = None,
budget_ms: int | None = None,
) -> tuple[str, ...]:
"""Claim at most one FIFO head per enabled Harness."""
"""Claim event-time heads, with one running job per enabled Harness."""
if type(limit) is not int or limit < 1:
raise ValueError("limit must be positive")
now = now or datetime.now(UTC)
@@ -2196,9 +2227,7 @@ class DurableWriteDrain:
(PENDING,),
).fetchall()
enabled_sources = {
str(row[0])
for row in source_rows
if self.source_writes_permitted(str(row[0]))
str(row[0]) for row in source_rows if self.source_writes_permitted(str(row[0]))
}
if not enabled_sources:
return ()
@@ -2218,8 +2247,7 @@ class DurableWriteDrain:
source_clause = f"AND candidate.source_key IN ({placeholders}) "
source_params = tuple(sorted(enabled_sources))
window = connection.execute(
"SELECT window_anchor,window_end FROM memory_write_schedule_state "
"WHERE singleton=1"
"SELECT window_anchor,window_end FROM memory_write_schedule_state WHERE singleton=1"
).fetchone()
if window is not None:
anchor_text, end_text = str(window[0]), str(window[1])
@@ -2227,14 +2255,12 @@ class DurableWriteDrain:
active_source_params: tuple[Any, ...] = ()
if enabled_sources is not None:
placeholders = ",".join("?" for _ in enabled_sources)
active_source_clause = (
f"AND active.source_key IN ({placeholders}) "
)
active_source_clause = f"AND active.source_key IN ({placeholders}) "
active_source_params = tuple(sorted(enabled_sources))
# The contract retires a window when every *eligible*
# pending/running job in it has settled or parked. A pending row
# that FIFO already holds back (an unexpired lease, a busy
# Harness, or an earlier unsent sibling) is not eligible, so it
# that scheduling already holds back (an unexpired lease, a
# busy Harness, or an earlier event in its Session) is not eligible, so it
# must not keep the anchor alive: those blockers usually sit
# outside this ten-minute range and can never be claimed from
# inside it, which pinned the anchor and starved the whole
@@ -2245,18 +2271,18 @@ class DurableWriteDrain:
"WHERE active.event_at>=? AND active.event_at<? AND ("
"active.state=? OR (active.state=? "
"AND (active.lease_expires_at IS NULL "
"OR active.lease_expires_at < ?) "
+ active_source_clause
+ "AND NOT EXISTS ("
"OR active.lease_expires_at < ?) " + active_source_clause + "AND NOT EXISTS ("
"SELECT 1 FROM memory_write_jobs AS held "
"WHERE held.source_key=active.source_key AND held.state=? "
"AND held.lease_owner IS NOT NULL "
"AND held.lease_expires_at>=?) "
"AND NOT EXISTS ("
"SELECT 1 FROM memory_write_jobs AS earlier "
"WHERE earlier.source_key=active.source_key "
"WHERE earlier.session_key=active.session_key "
f"AND earlier.state IN ({blocking}) "
"AND earlier.queue_seq<active.queue_seq) "
"AND (earlier.event_at<active.event_at OR "
"(earlier.event_at=active.event_at "
"AND earlier.queue_seq<active.queue_seq))) "
")) LIMIT 1",
(
anchor_text,
@@ -2271,9 +2297,7 @@ class DurableWriteDrain:
),
).fetchone()
if active is None:
connection.execute(
"DELETE FROM memory_write_schedule_state WHERE singleton=1"
)
connection.execute("DELETE FROM memory_write_schedule_state WHERE singleton=1")
window = None
if window is None:
anchor = connection.execute(
@@ -2281,18 +2305,18 @@ class DurableWriteDrain:
"FROM memory_write_jobs AS candidate "
"WHERE candidate.state=? "
"AND (candidate.lease_expires_at IS NULL "
"OR candidate.lease_expires_at < ?) "
+ source_clause
+ "AND NOT EXISTS ("
"OR candidate.lease_expires_at < ?) " + source_clause + "AND NOT EXISTS ("
"SELECT 1 FROM memory_write_jobs AS held "
"WHERE held.source_key=candidate.source_key AND held.state=? "
"AND held.lease_owner IS NOT NULL "
"AND held.lease_expires_at>=?) "
"AND NOT EXISTS ("
"SELECT 1 FROM memory_write_jobs AS earlier "
"WHERE earlier.source_key=candidate.source_key "
"WHERE earlier.session_key=candidate.session_key "
f"AND earlier.state IN ({blocking}) "
"AND earlier.queue_seq<candidate.queue_seq) "
"AND (earlier.event_at<candidate.event_at OR "
"(earlier.event_at=candidate.event_at "
"AND earlier.queue_seq<candidate.queue_seq))) "
"ORDER BY candidate.event_at,candidate.queue_seq LIMIT 1",
(
PENDING,
@@ -2321,17 +2345,19 @@ class DurableWriteDrain:
"FROM memory_write_jobs AS candidate "
"WHERE candidate.state=? "
"AND (candidate.lease_expires_at IS NULL OR candidate.lease_expires_at < ?) "
+ source_clause +
"AND candidate.event_at>=? AND candidate.event_at<? "
+ source_clause
+ "AND candidate.event_at>=? AND candidate.event_at<? "
"AND NOT EXISTS ("
"SELECT 1 FROM memory_write_jobs AS held "
"WHERE held.source_key=candidate.source_key AND held.state=? "
"AND held.lease_owner IS NOT NULL AND held.lease_expires_at>=?) "
"AND NOT EXISTS ("
"SELECT 1 FROM memory_write_jobs AS earlier "
"WHERE earlier.source_key=candidate.source_key "
"WHERE earlier.session_key=candidate.session_key "
f"AND earlier.state IN ({blocking}) "
"AND earlier.queue_seq<candidate.queue_seq) "
"AND (earlier.event_at<candidate.event_at OR "
"(earlier.event_at=candidate.event_at "
"AND earlier.queue_seq<candidate.queue_seq))) "
"ORDER BY candidate.event_at,candidate.queue_seq",
(
PENDING,
@@ -2345,12 +2371,17 @@ class DurableWriteDrain:
),
).fetchall()
selected: list[str] = []
selected_sources: set[str] = set()
for row in rows:
source_key = str(row["source_key"])
if source_key in selected_sources:
continue
if self.source_claim_enabled is not None and not self.source_claim_enabled(
str(row["source_key"])
source_key
):
continue
selected.append(str(row["job_id"]))
selected_sources.add(source_key)
if len(selected) == min(limit, self.max_concurrent_sources):
break
if not selected:
@@ -2579,19 +2610,6 @@ class DurableWriteDrain:
self.queue.reclaim_expired,
operation_name="reclaim_expired",
)
# Terminal-payload reaping is a bounded retention write: a
# finished row's full request body must not sit in the business
# database forever just because the queue never went idle. Its
# own interval keeps it off every poll.
if (
self._last_terminal_reap is None
or tick_started - self._last_terminal_reap >= self._reap_interval
):
self._last_terminal_reap = tick_started
await self._run_idle_maintenance(
self.queue.clear_terminal_payloads,
operation_name="clear_terminal_payloads",
)
# A model outage clears on its own, so `waiting_for_model` jobs
# are re-admitted on a timer. That must not depend on idleness: a
# queue busy for longer than the wait window would otherwise keep
@@ -1525,7 +1525,7 @@ async def _053_memory_write_event_time(cursor: Any) -> None:
FROM memory_write_job_evidence AS linked
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
WHERE linked.job_id = memory_write_jobs.job_id
AND e.role = 'user' AND e.written_by = 'person'
AND e.role = 'user'
),
(
SELECT MIN(COALESCE(e.created_at, e.imported_at))
@@ -1571,6 +1571,22 @@ async def _053_memory_write_event_time(cursor: Any) -> None:
)
async def _054_legacy_possibly_sent_tasks(cursor: Any) -> None:
"""Preserve the old sent marker before removing its retired permission table."""
columns = await cursor.columns("memory_task_continuous_permissions")
if "task_id" not in columns:
return
await cursor.execute(
"""
UPDATE memory_tasks
SET hy_invocation_count=CASE
WHEN hy_invocation_count<1 THEN 1 ELSE hy_invocation_count END
WHERE task_id IN (SELECT task_id FROM memory_task_continuous_permissions)
"""
)
await cursor.execute("DROP TABLE memory_task_continuous_permissions")
def model_profiles_ddl(dialect: str = "mysql") -> str:
"""Model profiles and their version history.
@@ -3022,4 +3038,9 @@ MIGRATIONS: tuple[Migration, ...] = (
_053_memory_write_event_time,
("memory_write_schedule_state",),
),
Migration(
54,
"legacy_possibly_sent_tasks",
_054_legacy_possibly_sent_tasks,
),
)
@@ -107,6 +107,8 @@ def build_request(
first_round: int,
last_round: int,
evidence_ids: list[str],
input_records: tuple[EvidenceRecord, ...] = (),
source_id: str | None = None,
priority: Literal["live", "normal"] = "normal",
idempotency_key: str | None = None,
) -> WriteRequest:
@@ -125,6 +127,8 @@ def build_request(
internal_session_id=internal_session_id,
batch_identity=batch,
evidence_ids=evidence_ids,
input_records=input_records,
source_id=source_id,
idempotency_key=idempotency_key or batch.batch_id,
priority=priority,
)
@@ -271,6 +275,8 @@ def build_request_from_evidence(
first_round=first_round,
last_round=last_round,
evidence_ids=list(ordered),
input_records=tuple(ordered.values()),
source_id=seed.source_id,
priority=priority,
idempotency_key=idempotency_key,
)
@@ -22,7 +22,7 @@ from datetime import datetime
from hashlib import sha256
from typing import Literal
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
from one_memory.memory_server.contracts import (
BatchIdentity,
WriteRequest,
@@ -126,6 +126,11 @@ class UnifiedWriteIntake:
) -> WriteRequest:
session_id = internal_session_id_for(record.conversation_id, record.source_id)
event_at, time_quality = _job_time((record,))
request_key = (
f"entry:{record.evidence_id}"
if record.revision == 1
else (f"entry:v2:{record.evidence_id}:{record.revision}:{record.content_hash}")
)
return WriteRequest(
internal_session_id=session_id,
source_id=record.source_id,
@@ -135,7 +140,8 @@ class UnifiedWriteIntake:
last_round=last_round,
),
evidence_ids=[record.evidence_id],
idempotency_key=f"entry:{record.evidence_id}",
input_records=(record,),
idempotency_key=request_key,
priority=priority,
event_at=event_at,
time_quality=time_quality,
@@ -176,6 +182,7 @@ class UnifiedWriteIntake:
last_round=1,
),
evidence_ids=[record.evidence_id for record in records],
input_records=records,
idempotency_key=request_id,
priority=priority,
event_at=event_at,
@@ -204,20 +211,14 @@ def _job_time(
"""Use the turn opener's wall clock, with an explicit fallback quality."""
opener = next(
(
record
for record in records
if record.role == "user" and record.written_by is WrittenBy.PERSON
),
(record for record in records if record.role == "user"),
records[0],
)
if opener.created_at is None:
return opener.imported_at, "approximate"
file_origin = opener.origin_meta.get(EvidenceOrigin.FILE)
basis = file_origin.raw_meta.get("time_basis") if file_origin is not None else None
turn_event_at = (
file_origin.raw_meta.get("turn_event_at") if file_origin is not None else None
)
turn_event_at = file_origin.raw_meta.get("turn_event_at") if file_origin is not None else None
if isinstance(turn_event_at, str):
try:
return datetime.fromisoformat(turn_event_at), "exact"
@@ -17,7 +17,7 @@ from pydantic import BaseModel, ConfigDict, Field
from one_memory.database import take_the_write_lock_first
from one_memory.database_access import main_database
from one_memory.imports.models import EvidenceRecord
from one_memory.imports.models import EvidenceRecord, WrittenBy
from one_memory.memory_server.migrations import memory_tasks_ddl
from one_memory.model_profiles import ModelProfile, ModelProfileSnapshot
@@ -174,29 +174,60 @@ def evidence_ref(record: EvidenceRecord) -> TaskEvidenceRef:
"segment_end": int(marker["end"]),
"segment_hash": str(marker["hash"]),
}
identity = record.model_dump(mode="json", exclude={"origin_meta", "imported_at", "updated_at"})
return {
"evidence_id": record.evidence_id,
"revision": record.revision,
"content_hash": record.content_hash,
"identity_hash": sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest(),
"identity_hash": _evidence_identity_hash(record, include_written_by=False),
}
def _evidence_identity_hash(record: EvidenceRecord, *, include_written_by: bool) -> str:
excluded = {"origin_meta", "imported_at", "updated_at"}
if not include_written_by:
excluded.add("written_by")
identity = record.model_dump(mode="json", exclude=excluded)
return sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
def _identity_hash_matches(record: EvidenceRecord, expected: object) -> bool:
candidates = {_evidence_identity_hash(record, include_written_by=False)}
# Tasks created by pre-role-only builds keep their original proof. The
# legacy metadata could also have changed in place without a revision, so
# accept the old hash under every historical author guess.
for written_by in WrittenBy:
candidates.add(
_evidence_identity_hash(
record.model_copy(update={"written_by": written_by}),
include_written_by=True,
)
)
return expected in candidates
def matches_evidence_ref(record: EvidenceRecord, ref: TaskEvidenceRef) -> bool:
if "segment_hash" in ref:
base = evidence_ref(record)
if any(
base.get(key) != ref.get(key)
for key in ("evidence_id", "revision", "content_hash", "identity_hash")
for key in ("evidence_id", "revision", "content_hash")
):
return False
if not _identity_hash_matches(record, ref.get("identity_hash")):
return False
start, end = ref.get("segment_start"), ref.get("segment_end")
if not isinstance(start, int) or not isinstance(end, int) or not (0 <= start < end):
return False
return sha256(record.verbatim[start:end].encode()).hexdigest() == ref["segment_hash"]
current = evidence_ref(record)
return all(current.get(key) == value for key, value in ref.items())
return all(
(
_identity_hash_matches(record, value)
if key == "identity_hash"
else current.get(key) == value
)
for key, value in ref.items()
)
def segment_evidence_record(
@@ -87,7 +87,7 @@ def build_acceptance_pack(
"""
SELECT DISTINCT e.evidence_id
FROM evidence e
WHERE e.role = 'user' AND e.written_by = 'person'
WHERE e.role = 'user'
AND LENGTH(e.verbatim) <= 300
AND (
e.verbatim LIKE '%我是一名%' OR e.verbatim LIKE '%我是一个%'
@@ -118,7 +118,7 @@ def build_acceptance_pack(
for row in connection.execute(
"""
SELECT evidence_id FROM evidence
WHERE role = 'user' AND written_by != 'person'
WHERE role != 'user'
"""
):
strata["attribution_edge"].add(row["evidence_id"])
@@ -138,11 +138,7 @@ def build_acceptance_pack(
quota,
)
if len(selected) < sample_size:
eligible = (
evidence_id
for evidence_id, row in evidence.items()
if row["role"] == "user" and row["written_by"] == "person"
)
eligible = (evidence_id for evidence_id, row in evidence.items() if row["role"] == "user")
_extend_unique(selected, _stable_order(eligible, seed=f"{seed}:fill"), sample_size)
selected = selected[:sample_size]
if len(selected) < sample_size:
@@ -163,7 +159,11 @@ def build_acceptance_pack(
row_id=f"{index:03d}",
evidence_id=evidence_id,
source_id=source["source_id"],
written_by=source["written_by"],
written_by=(
"person"
if source["role"] == "user"
else ("agent" if source["role"] == "assistant" else source["role"])
),
character_count=len(source["verbatim"]),
content_hash=source["content_hash"],
strata=tuple(
@@ -2,7 +2,7 @@
from collections.abc import Sequence
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers
from one_memory.input_sanitization import sanitize_model_input
from one_memory.memory_processing import _is_hy_input_turn
@@ -15,10 +15,7 @@ def identity_content_ready(record: EvidenceRecord) -> bool:
# cleaning requires a source map and remains explicitly uncovered for now.
return (
record.content_kind is EvidenceContentKind.RAW_CONVERSATION
and (
(record.role == "user" and record.written_by is WrittenBy.PERSON)
or (record.role == "assistant" and record.written_by is WrittenBy.AGENT)
)
and record.role in {"user", "assistant"}
and _is_hy_input_turn(record)
and not is_host_generated_user_turn(record.verbatim)
and strip_host_wrappers(record.verbatim) == record.verbatim.strip()
@@ -10,7 +10,6 @@ from one_memory.imports.models import (
MAX_VERBATIM_CHARACTERS,
EvidenceContentKind,
EvidenceRecord,
WrittenBy,
)
from one_memory.imports.sources.parsing import (
is_host_generated_user_turn,
@@ -164,10 +163,10 @@ def prepare_mapped_content(record: EvidenceRecord) -> PreparedMappedContent:
The returned text is transient; only metadata is suitable for an outer manifest.
That manifest must independently bind Evidence identity/revision/hash and grants.
"""
if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or not (
(record.role == "user" and record.written_by is WrittenBy.PERSON)
or (record.role == "assistant" and record.written_by is WrittenBy.AGENT)
):
if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or record.role not in {
"user",
"assistant",
}:
raise MappedContentError("mapped_content_ineligible")
original = record.verbatim
if len(original) > MAX_VERBATIM_CHARACTERS:
@@ -11,7 +11,7 @@ from datetime import UTC
from one_memory import ingest_journal as journal
from one_memory.context_raw import _check, _connection
from one_memory.imports.models import EvidenceRecord, WrittenBy
from one_memory.imports.models import EvidenceRecord
from one_memory.imports.storage import EvidenceStore
from one_memory.retrieval.index_store import RetrievalIndexStore
from one_memory.retrieval.models import EvidenceSpan, RawEvidence
@@ -166,7 +166,7 @@ def materialize_raw_candidates(
source_id=record.source_id,
conversation_id=record.conversation_id,
role=record.role,
written_by="person" if record.written_by is WrittenBy.PERSON else "agent",
written_by="person" if record.role == "user" else "agent",
event_time=event_time,
spans=tuple(spans),
matched_by=("semantic",),
@@ -56,10 +56,10 @@ def build_source_import_receipt(
"SELECT count(*) AS messages, "
"count(DISTINCT conversation_id || char(0) || COALESCE(workspace, '')) "
"AS conversations, "
"coalesce(sum(written_by='person'),0) AS person_messages, "
"coalesce(sum(written_by='agent'),0) AS agent_messages, "
"coalesce(sum(written_by='tool'),0) AS tool_messages, "
"coalesce(sum(written_by NOT IN ('person','agent','tool')),0) "
"coalesce(sum(role='user'),0) AS person_messages, "
"coalesce(sum(role='assistant'),0) AS agent_messages, "
"coalesce(sum(role='tool'),0) AS tool_messages, "
"coalesce(sum(role NOT IN ('user','assistant','tool')),0) "
"AS unknown_author_messages, "
"coalesce(sum(created_at IS NULL),0) AS unknown_time_messages "
"FROM evidence WHERE user_id=? AND source_id=?",
@@ -68,7 +68,9 @@ def build_source_import_receipt(
# Bound text reads before materializing snippets. These are samples, not a
# recency ranking; sorting every large original body is unnecessary here.
examples = db.execute(
"SELECT evidence_id, role, written_by, created_at, "
"SELECT evidence_id, role, CASE role "
"WHEN 'user' THEN 'person' WHEN 'assistant' THEN 'agent' "
"WHEN 'tool' THEN 'tool' ELSE 'unknown' END AS written_by, created_at, "
"substr(verbatim,1,600) AS excerpt, length(verbatim)>600 AS truncated "
"FROM evidence WHERE evidence_id IN ("
"SELECT evidence_id FROM evidence WHERE user_id=? AND source_id=? LIMIT 3)",
@@ -80,10 +82,13 @@ def build_source_import_receipt(
# Qualify before LIMIT: agent/system rows must not consume the three samples.
query_filter = " AND evidence_id=?" if query_evidence_id is not None else ""
queries = db.execute(
f"""SELECT evidence_id, revision AS evidence_revision, role, written_by, created_at,
CASE WHEN user_id=? AND source_id=? AND role='user' AND written_by='person'
f"""SELECT evidence_id, revision AS evidence_revision, role,
CASE role WHEN 'user' THEN 'person' WHEN 'assistant' THEN 'agent'
WHEN 'tool' THEN 'tool' ELSE 'unknown' END AS written_by,
created_at,
CASE WHEN user_id=? AND source_id=? AND role='user'
AND length(verbatim)<=65536
THEN display_query_projection(verbatim,role,written_by)
THEN display_query_projection(verbatim,role,'person')
ELSE NULL END AS projection
FROM evidence WHERE user_id=? AND source_id=?{query_filter}
AND projection IS NOT NULL
+87 -2
View File
@@ -2,6 +2,7 @@ import json
import sqlite3
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from hashlib import sha256
import pytest
from pydantic import ValidationError
@@ -14,6 +15,7 @@ from one_memory.imports.models import (
WrittenBy,
)
from one_memory.imports.storage import EvidenceStore
from one_memory.memory_tasks import evidence_ref, matches_evidence_ref
from one_memory.workspace_resolution import KnownWorkspace
@@ -25,6 +27,7 @@ def evidence(
verbatim: str = "Remember the source, not the guess.",
origin: EvidenceOrigin = EvidenceOrigin.FILE,
written_by: WrittenBy = WrittenBy.PERSON,
role: str = "user",
source_ordinal: str | None = None,
) -> EvidenceInput:
return EvidenceInput(
@@ -35,7 +38,7 @@ def evidence(
source_ordinal=source_ordinal
if source_ordinal is not None
else ("line:1" if message_id is None and correlation_id is None else None),
role="user",
role=role,
written_by=written_by,
created_at=datetime(2026, 8, 31, 12, 0, tzinfo=UTC),
verbatim=verbatim,
@@ -87,7 +90,7 @@ def test_rescanning_an_unchanged_file_does_not_write_the_message_again(tmp_path)
"""A rescan is idempotent because the identity is derived from the content.
`EvidenceInput.idempotency_key` is a hash of the source, conversation,
author, ordinal and content hash, and `evidence_id` is a uuid5 over that
protocol role and content hash, and `evidence_id` is a uuid5 over that
key, so replaying an unchanged file reproduces the same primary key and
`ON CONFLICT` absorbs it. Verified: removing the anchoring logic still
leaves this green, so this pins the contract, not one implementation.
@@ -102,6 +105,87 @@ def test_rescanning_an_unchanged_file_does_not_write_the_message_again(tmp_path)
assert store.count() == 1
def test_derived_identity_uses_role_not_host_author_guess(tmp_path) -> None:
store = EvidenceStore(tmp_path / "one-memory.sqlite3")
first = store.put(evidence(message_id=None, written_by=WrittenBy.UNKNOWN))
replay = store.put(evidence(message_id=None, written_by=WrittenBy.PERSON))
assistant = store.put(evidence(message_id=None, role="assistant", written_by=WrittenBy.AGENT))
assert replay.record.evidence_id == first.record.evidence_id
assert replay.was_created is False
assert assistant.record.evidence_id != first.record.evidence_id
assert store.count() == 2
def test_task_proof_ignores_legacy_written_by_metadata(tmp_path) -> None:
store = EvidenceStore(tmp_path / "one-memory.sqlite3")
record = store.put(evidence()).record
proof = evidence_ref(record)
changed_author = record.model_copy(update={"written_by": WrittenBy.AGENT})
legacy_identity = record.model_dump(
mode="json",
exclude={"origin_meta", "imported_at", "updated_at"},
)
legacy_proof = {
**proof,
"identity_hash": sha256(
json.dumps(legacy_identity, sort_keys=True).encode()
).hexdigest(),
}
assert matches_evidence_ref(changed_author, proof) is True
assert matches_evidence_ref(changed_author, legacy_proof) is True
def test_rescan_reuses_a_row_stored_under_written_by_derived_v3(tmp_path) -> None:
database = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(database)
item = evidence(message_id=None, written_by=WrittenBy.UNKNOWN)
original = store.put(item).record
with sqlite3.connect(database) as connection, connection:
connection.execute(
"UPDATE evidence SET idempotency_key=? WHERE evidence_id=?",
(item.written_by_derived_v3_idempotency_key, original.evidence_id),
)
replay = store.put(evidence(message_id=None, written_by=WrittenBy.PERSON))
assert replay.was_created is False
assert replay.record.evidence_id == original.evidence_id
assert store.count() == 1
def test_legacy_identity_role_change_revises_the_existing_evidence(tmp_path) -> None:
database = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(database)
old = evidence(
message_id=None,
role="assistant",
written_by=WrittenBy.AGENT,
)
original = store.put(old).record
with sqlite3.connect(database) as connection, connection:
connection.execute(
"UPDATE evidence SET idempotency_key=? WHERE evidence_id=?",
(old.written_by_derived_v3_idempotency_key, original.evidence_id),
)
replay = store.put(
evidence(
message_id=None,
role="user",
written_by=WrittenBy.AGENT,
)
)
assert replay.record.evidence_id == original.evidence_id
assert replay.was_revised is True
assert replay.record.revision == original.revision + 1
assert replay.record.role == "user"
assert store.count() == 1
def test_changed_text_at_one_position_is_a_new_message_not_a_silent_edit(tmp_path) -> None:
"""An edited line becomes new evidence, never an overwrite of the old one.
@@ -146,6 +230,7 @@ def test_personal_candidate_prefilter_excludes_unrelated_user_turns(tmp_path) ->
evidence(
message_id="assistant",
verbatim="我偏好简洁文案。",
role="assistant",
written_by=WrittenBy.AGENT,
)
)
@@ -248,18 +248,8 @@ def test_profile_rendering_keeps_per_field_evidence_without_dropping_native_prof
assert bindings.profile_support("The user's name is 陈.", {"name": "陈"}) == ()
def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
"""The author must match the role: only user/person, assistant/agent, tool/tool.
The objective requires 作者 to be truthful. The other scope tests only ever
build well-formed pairs, so this pins the pair check itself: an assistant
turn attributed to a person, or a tool turn attributed to one, must be
refused with `context_scope` rather than silently trusted.
`user` + AGENT is deliberately accepted since D98: the host parser marks a
sub agent's question that way, and that turn is sent like any other. What
is refused is the pair that cannot occur, not the one we now allow.
"""
def test_bound_context_uses_protocol_role_not_author_guess(tmp_path):
"""Host author guesses never reject a valid user/assistant/tool message."""
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
store = EvidenceStore(tmp_path / "evidence.db")
@@ -268,8 +258,7 @@ def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
assistant = source(store, "reply", "assistant", "好的,继续推进。")
wrong_author = assistant.model_copy(update={"written_by": WrittenBy.PERSON})
with pytest.raises(ValueError, match="context_scope"):
HyWriteBindings(HySourceContext(current=(wrong_author,)))
HyWriteBindings(HySourceContext(current=(wrong_author,)))
# `source()` maps non-user roles to AGENT, so a tool turn needs its own build.
tool = store.put(
@@ -284,7 +273,10 @@ def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
)
).record
assert HyWriteBindings(HySourceContext(current=(tool,))) # valid tool/tool pair
assert HyWriteBindings(HySourceContext(current=(tool,)))
wrong_tool = tool.model_copy(update={"written_by": WrittenBy.PERSON})
HyWriteBindings(HySourceContext(current=(wrong_tool,)))
unsupported = assistant.model_copy(update={"role": "system"})
with pytest.raises(ValueError, match="context_scope"):
HyWriteBindings(HySourceContext(current=(wrong_tool,)))
HyWriteBindings(HySourceContext(current=(unsupported,)))
+12 -1
View File
@@ -111,7 +111,7 @@ def test_import_metadata_changes_invalidate_without_authorizing_content(setup, c
@pytest.mark.parametrize("cause", ["import", "maintenance"])
@pytest.mark.parametrize(
"column,value",
[("verbatim", "changed raw body"), ("written_by", "agent"), ("role", "assistant")],
[("verbatim", "changed raw body"), ("role", "assistant")],
)
def test_material_change_eligibility_depends_on_trusted_cause(setup, cause, column, value):
db, store, _ = setup
@@ -126,6 +126,17 @@ def test_material_change_eligibility_depends_on_trusted_cause(setup, cause, colu
assert new.kind == ("revise" if cause == "import" else "metadata")
def test_legacy_written_by_change_is_not_a_material_event(setup):
db, store, _ = setup
db.execute("BEGIN IMMEDIATE")
key = put(store, db)
journal.initialize(db)
old = journal.current(db, key)
db.execute("UPDATE evidence SET written_by='agent' WHERE evidence_id=?", (key,))
assert journal.record(db, key, cause="import") == old
def test_complete_origin_metadata_changes_and_observation_noise(setup):
db, store, _ = setup
db.execute("BEGIN IMMEDIATE")
@@ -1780,7 +1780,7 @@ def test_process_records_bounds_a_hung_model_call_by_the_profile_timeout(
with sqlite3.connect(tmp_path / "one-memory.sqlite3") as db:
db.executescript(
"""
CREATE TABLE source_grants (
CREATE TABLE IF NOT EXISTS source_grants (
user_id TEXT NOT NULL,
source_id TEXT NOT NULL,
root_path TEXT NOT NULL,
@@ -2038,6 +2038,126 @@ def test_oversized_single_turn_segments_are_each_processed(tmp_path) -> None:
assert len(governance.list(user_id="local")) == 2
def test_direct_processing_uses_the_supplied_frozen_revision(tmp_path) -> None:
stores = _setup(tmp_path)
evidence, profiles, tasks, governance, secrets, _profile = stores
old_user = _put(
evidence,
message_id="rewound-user",
role="user",
written_by=WrittenBy.PERSON,
content="old branch",
)
reply = _put(
evidence,
message_id="rewound-reply",
role="assistant",
written_by=WrittenBy.AGENT,
content="old reply",
)
revised = _put(
evidence,
message_id="rewound-user",
role="user",
written_by=WrittenBy.PERSON,
content="new branch",
)
assert revised.evidence_id == old_user.evidence_id
assert revised.revision == old_user.revision + 1
with sqlite3.connect(tasks.database_path) as connection, connection:
connection.execute(
"""
CREATE TABLE source_grants (
user_id TEXT NOT NULL,
source_id TEXT NOT NULL,
root_path TEXT NOT NULL,
requested_path TEXT NOT NULL,
root_identity TEXT NOT NULL,
candidate_kind TEXT NOT NULL,
granted_at TEXT NOT NULL,
revoked_at TEXT,
revision INTEGER NOT NULL,
PRIMARY KEY (user_id, source_id)
)
"""
)
connection.execute(
"""
INSERT INTO source_grants (
user_id,source_id,root_path,requested_path,root_identity,
candidate_kind,granted_at,revoked_at,revision
) VALUES (?,?,?,?,?,?,?,NULL,1)
""",
(
"local",
"cursor",
"/authorized",
"/authorized",
"test-root",
"default",
datetime.now(UTC).isoformat(),
),
)
bridge = FakeBridge(outputs=())
supervisor = MemoryProcessingSupervisor(
tasks=tasks,
profiles=profiles,
secrets=secrets,
evidence=evidence,
governance=governance,
data_dir=tmp_path,
runner_factory=lambda: _runner(tmp_path, stores, bridge),
)
completed = asyncio.run(supervisor.process_records((old_user, reply)))
assert len(completed) == 1
assert [turn.content for turn in bridge.requests[0].turns] == [
"old branch",
"old reply",
]
def test_frozen_current_still_revalidates_history_after_the_model_call(tmp_path) -> None:
from one_memory.hy_source_context import HySourceContext
stores = _setup(tmp_path)
evidence, _profiles, tasks, _governance, _secrets, _profile = stores
current = _put(
evidence,
message_id="frozen-current",
role="user",
written_by=WrittenBy.PERSON,
content="old current",
)
history = _put(
evidence,
message_id="frozen-history",
role="assistant",
written_by=WrittenBy.AGENT,
content="stable history",
)
_put(
evidence,
message_id="frozen-current",
role="user",
written_by=WrittenBy.PERSON,
content="new current",
)
runner = _runner(tmp_path, stores, FakeBridge(outputs=()))
context = HySourceContext(current=(current,), history=(history,))
runner._check_source_context(context, skip_current=True)
with sqlite3.connect(tasks.database_path) as connection, connection:
connection.execute(
"DELETE FROM evidence WHERE evidence_id=?",
(history.evidence_id,),
)
with pytest.raises(HyMemoryBridgeError, match="hy_memory_evidence_changed"):
runner._check_source_context(context, skip_current=True)
def test_oversized_assistant_segments_keep_the_question_as_context(tmp_path) -> None:
evidence = EvidenceStore(tmp_path / "one-memory.sqlite3")
question = _put(
@@ -125,8 +125,7 @@ def test_write_job_state_covers_unknown_and_terminal() -> None:
} == {state for state in WriteJobState if not state.is_terminal}
def test_canonical_message_requires_author_and_ordinal() -> None:
"""A message without a known author or position cannot be ordered later."""
def test_canonical_message_requires_a_nonnegative_ordinal() -> None:
with pytest.raises(ValueError):
CanonicalMessage(
harness_id="cursor",
@@ -134,6 +133,5 @@ def test_canonical_message_requires_author_and_ordinal() -> None:
message_id=None,
ordinal=-1,
role="user",
written_by="person",
content="hello",
)
@@ -10,6 +10,7 @@ import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from hashlib import sha256
import pytest
@@ -38,11 +39,13 @@ def _request(
evidence: tuple[str, ...] = ("e1",),
key: str = "idem-1",
source: str | None = None,
session: str | None = None,
event_at: datetime | None = None,
) -> WriteRequest:
batch = BatchIdentity(internal_session_id="sess-1", first_round=1, last_round=5)
session_id = session or f"sess-{source or '1'}"
batch = BatchIdentity(internal_session_id=session_id, first_round=1, last_round=5)
return WriteRequest(
internal_session_id="sess-1",
internal_session_id=session_id,
batch_identity=batch,
evidence_ids=list(evidence),
idempotency_key=key,
@@ -94,13 +97,8 @@ def test_acceptance_is_written_to_the_business_database(tmp_path, monkeypatch) -
]
def test_the_stored_payload_is_references_only(tmp_path) -> None:
"""The job payload must point at Evidence, never carry its text.
A queue row is durable and much longer-lived than the API call that made
it, so storing verbatim content here would be a second copy of the user's
private text. The payload is the request's identifiers only.
"""
def test_an_isolated_legacy_queue_keeps_the_empty_snapshot_compatibility_shape(tmp_path) -> None:
"""Queues without an Evidence authority exist only for legacy/unit compatibility."""
import json
path = tmp_path / "one-memory.sqlite3"
@@ -112,10 +110,12 @@ def test_the_stored_payload_is_references_only(tmp_path) -> None:
"internal_session_id",
"batch_identity",
"evidence_ids",
"input_records",
"idempotency_key",
"priority",
"source_id",
}
assert payload["input_records"] == []
assert payload["evidence_ids"] == ["ev-1", "ev-2"]
@@ -216,20 +216,17 @@ def test_a_replay_is_idempotent_even_after_its_evidence_is_removed(tmp_path) ->
assert replay.enqueued is False
assert len(_rows(path)) == 1
def test_a_settled_job_replays_as_a_hit_not_a_conflict(tmp_path) -> None:
"""Replaying a completed key must return the job, never raise.
Real trigger: 236 startup replays of one key, all
`ValueError: idempotency key ... already exists with different content`.
The stored payload had been reaped to a digest sentinel, and the caller's
key is derived from the evidence's revision, which can move on between
replays while the payload's evidence ids stay put. Refusing that pair
turned a normal startup replay into an exception every time.
"""
"""A historical reaped payload matches only its exact pre-snapshot intent."""
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
first = queue.submit(_request(evidence=("e1",), key="idem-settled"))
request = _request(evidence=("e1",), key="idem-settled")
first = queue.submit(request)
legacy = request.model_dump(
mode="json",
exclude={"event_at", "time_quality", "input_records"},
)
digest = sha256(json.dumps(legacy, sort_keys=True).encode()).hexdigest()
# Settle it the way the drain does: terminal state, HY invoked, and the
# payload reaped down to its digest so no body is kept.
@@ -238,19 +235,40 @@ def test_a_settled_job_replays_as_a_hit_not_a_conflict(tmp_path) -> None:
"UPDATE memory_write_jobs SET state=?, hy_invoked=1, payload_json=? WHERE job_id=?",
(
COMPLETED,
json.dumps({"_one_memory_reaped_payload": "0" * 64}),
json.dumps({"_one_memory_reaped_payload": digest}),
first.job_id,
),
)
# A payload that cannot match the reaped digest, under the same key.
replay = queue.submit(_request(evidence=("e1",), key="idem-settled"))
replay = queue.submit(request)
assert replay.job_id == first.job_id
assert replay.state == COMPLETED
assert replay.enqueued is False
assert len(_rows(path)) == 1
with pytest.raises(ValueError, match="different content"):
queue.submit(
_request(
evidence=("e1",),
key="idem-settled",
source="cursor",
session="different-session",
)
)
def test_a_real_completed_job_still_rejects_the_same_key_with_different_input(
tmp_path,
) -> None:
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
first = queue.submit(_request(evidence=("e1",), key="completed-conflict"))
queue.settle(first.job_id, state=COMPLETED, hy_invoked=True)
with pytest.raises(ValueError, match="different content"):
queue.submit(_request(evidence=("e2",), key="completed-conflict"))
def test_a_killed_worker_leaves_a_possibly_sent_job_unknown_after_restart(tmp_path) -> None:
"""A real process death after HY was invoked must not become retryable.
@@ -1144,10 +1162,13 @@ def test_unresolved_unknown_is_automatically_retried_only_once(tmp_path) -> None
first_retry = queue.get(record.request_id)
assert first_retry.state == PENDING
with sqlite3.connect(path) as connection:
assert connection.execute(
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
(record.job_id,),
).fetchone()[0] == 1
assert (
connection.execute(
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
(record.job_id,),
).fetchone()[0]
== 1
)
drain = DurableWriteDrain(queue, execute=_never_execute, owner="retry-owner")
assert drain.claim(limit=1) == (record.job_id,)
@@ -1161,10 +1182,13 @@ def test_unresolved_unknown_is_automatically_retried_only_once(tmp_path) -> None
)
with sqlite3.connect(path) as connection:
assert connection.execute(
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
(record.job_id,),
).fetchone()[0] == 2
assert (
connection.execute(
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
(record.job_id,),
).fetchone()[0]
== 2
)
assert (
queue.retry_first_unknown_after_reconcile(
older_than=datetime.now(UTC) + timedelta(minutes=1)
@@ -2541,21 +2565,16 @@ def test_settle_with_an_owner_does_not_touch_an_ownerless_row(tmp_path, caplog)
assert not any("durable_write_settle_missed_row" in message for message in messages), messages
def test_terminal_payloads_are_reaped_but_replay_still_detects_conflicts(tmp_path) -> None:
"""Payload cleanup must not weaken idempotency.
Once a job is terminal the serialized request is dead weight, so it is
replaced by a digest. Replaying the same intent must still resolve to the
same row, and the same key with different content must still be refused.
"""
def test_terminal_payloads_are_retained_and_replay_still_detects_conflicts(tmp_path) -> None:
"""Terminal input is immutable audit data and idempotency remains strict."""
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
record = queue.submit(_request(evidence=("e1",)))
queue.settle(record.job_id, state="completed")
assert queue.clear_terminal_payloads() == 1
assert queue.clear_terminal_payloads() == 0
stored = _rows(path)[0]["payload_json"]
assert "e1" not in stored and "_one_memory_reaped_payload" in stored
assert "e1" in stored and "_one_memory_reaped_payload" not in stored
assert _rows(path)[0]["evidence_ids_json"] == '["e1"]'
# Same intent still resolves to the existing row, not a second write.
@@ -2569,15 +2588,15 @@ def test_terminal_payloads_are_reaped_but_replay_still_detects_conflicts(tmp_pat
queue.submit(_request(evidence=("e2",)))
def test_terminal_payload_reaping_is_bounded_and_idempotent(tmp_path) -> None:
def test_terminal_payload_cleanup_is_a_compatibility_noop(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
for index in range(3):
record = queue.submit(_request(evidence=("e1",), key=f"idem-{index}"))
queue.settle(record.job_id, state="dead")
assert queue.clear_terminal_payloads(limit=2) == 2
assert queue.clear_terminal_payloads(limit=2) == 1
assert queue.clear_terminal_payloads(limit=2) == 0
assert queue.clear_terminal_payloads(limit=2) == 0
assert queue.clear_terminal_payloads(limit=2) == 0
@@ -2596,8 +2615,8 @@ def test_unknown_and_pending_payloads_are_never_reaped(tmp_path) -> None:
assert "e2" in payloads[unknown.request_id]
def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_path) -> None:
"""Payload cleanup must not make a completed Evidence look unqueued.
def test_retained_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_path) -> None:
"""A completed Evidence remains queued while its full input is retained.
The reconcile primitive scans the durable queue by Evidence id. If it
derived that set from `payload_json`, reaping a terminal row would erase
@@ -2631,7 +2650,7 @@ def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_
queue = DurableWriteQueue(path)
record = queue.submit(_request(evidence=(evidence.evidence_id,), key="idem-reaped"))
queue.settle(record.job_id, state="completed")
assert queue.clear_terminal_payloads() == 1
assert queue.clear_terminal_payloads() == 0
assert queue.unsubmitted_evidence_ids() == ()
orphan = store.put(
@@ -2649,19 +2668,17 @@ def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_
assert queue.unsubmitted_evidence_ids() == (orphan.evidence_id,)
def test_payload_for_a_reaped_terminal_row_returns_none(tmp_path) -> None:
"""A stale claim of a reaped row must not resurrect a dead intent."""
def test_payload_for_a_terminal_row_keeps_the_immutable_input(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
record = queue.submit(_request())
queue.settle(record.job_id, state="completed")
queue.clear_terminal_payloads()
assert queue.payload_for(record.job_id) is None
assert queue.payload_for(record.job_id) == _request()
def test_drain_reaps_terminal_payloads_when_idle(tmp_path) -> None:
"""The reaper must actually run in the worker, not only in tests."""
def test_drain_retains_terminal_payloads_when_idle(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
record = queue.submit(_request())
@@ -2686,19 +2703,11 @@ def test_drain_reaps_terminal_payloads_when_idle(tmp_path) -> None:
asyncio.run(scenario())
assert executed == []
assert "_one_memory_reaped_payload" in _rows(path)[0]["payload_json"]
assert "_one_memory_reaped_payload" not in _rows(path)[0]["payload_json"]
def test_the_drain_reaps_terminal_payloads_even_while_never_idle(tmp_path) -> None:
"""Terminal payloads must be reaped even if the queue never goes idle.
Regression: `clear_terminal_payloads` ran only on the `executed == 0` idle
branch, so a sustained backlog kept every finished row's full serialized
request body (which holds the written evidence references) in the business
database indefinitely — a disk/privacy-retention invariant starved exactly
when the queue is busiest. Reaping is a bounded maintenance write and must
be gated by its own interval, not by idleness.
"""
def test_the_drain_retains_terminal_payloads_while_never_idle(tmp_path) -> None:
"""A sustained backlog must not mutate an earlier job's input snapshot."""
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
finished = queue.submit(_request(key="finished"))
@@ -2726,9 +2735,8 @@ def test_the_drain_reaps_terminal_payloads_even_while_never_idle(tmp_path) -> No
payload = _rows(path)
by_key = {row["idempotency_key"]: row["payload_json"] for row in payload}
assert "_one_memory_reaped_payload" in by_key["finished"], (
"a completed row's payload must be reaped even under a standing backlog"
)
assert "_one_memory_reaped_payload" not in by_key["finished"]
assert '"e1"' in by_key["finished"]
def test_the_drain_readmits_model_waiters_even_while_never_idle(tmp_path) -> None:
@@ -3177,8 +3185,8 @@ def test_claim_budget_ab_against_a_held_writer(tmp_path) -> None:
def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> None:
"""An empty queue must not issue a write per idle poll.
`clear_terminal_payloads` and `readmit_model_waiters` both write. On an
empty queue the loop wakes every `idle_pause`; running them each time would
`readmit_model_waiters` writes. On an empty queue the loop wakes every
`idle_pause`; running it each time would
contend with imports for the single writer for no benefit. They share a
coarse interval gate, so a 20-poll window performs at most a couple of
writes.
@@ -3187,22 +3195,14 @@ def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> No
queue = DurableWriteQueue(path)
drain = DurableWriteDrain(queue, execute=_never_execute)
reaps = 0
readmits = 0
real_reap = queue.clear_terminal_payloads
real_readmit = queue.readmit_model_waiters
def counted_reap(*args, **kwargs):
nonlocal reaps
reaps += 1
return real_reap(*args, **kwargs)
def counted_readmit(*args, **kwargs):
nonlocal readmits
readmits += 1
return real_readmit(*args, **kwargs)
monkeypatch.setattr(queue, "clear_terminal_payloads", counted_reap)
monkeypatch.setattr(queue, "readmit_model_waiters", counted_readmit)
monkeypatch.setattr(drain, "claim", lambda **_kw: ())
@@ -3216,7 +3216,6 @@ def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> No
# Well over 20 polls happened; maintenance ran at most once (the interval
# gate), not once per poll.
assert reaps <= 1
assert readmits <= 1
@@ -3347,7 +3346,7 @@ def test_idle_maintenance_database_failure_is_sampled_without_database_error(
def locked(*_args, **_kwargs):
raise sqlite3.OperationalError("database is locked")
monkeypatch.setattr(drain.queue, "clear_terminal_payloads", locked)
monkeypatch.setattr(drain.queue, "readmit_model_waiters", locked)
monkeypatch.setattr(drain, "claim", lambda **_kw: ())
async def scenario() -> None:
@@ -3363,8 +3362,8 @@ def test_idle_maintenance_database_failure_is_sampled_without_database_error(
def programming_error(*_args, **_kwargs):
raise RuntimeError("maintenance bug")
monkeypatch.setattr(drain.queue, "clear_terminal_payloads", programming_error)
drain._last_terminal_reap = None # noqa: SLF001 - force the next maintenance tick
monkeypatch.setattr(drain.queue, "readmit_model_waiters", programming_error)
drain._last_model_readmit = None # noqa: SLF001 - force the next maintenance tick
asyncio.run(scenario())
@@ -4288,9 +4287,7 @@ def test_event_time_window_claims_cross_harness_jobs_within_ten_minutes(
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
first = queue.submit(
_request(key="window-a", source="cursor", event_at=anchor)
)
first = queue.submit(_request(key="window-a", source="cursor", event_at=anchor))
inside = queue.submit(
_request(
key="window-b",
@@ -4334,9 +4331,7 @@ def test_event_time_window_anchor_does_not_slide_while_a_job_is_running(
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
first = queue.submit(
_request(key="fixed-a", source="cursor", event_at=anchor)
)
first = queue.submit(_request(key="fixed-a", source="cursor", event_at=anchor))
inside = queue.submit(
_request(
key="fixed-b",
@@ -4815,18 +4810,10 @@ def test_readmit_paths_never_requeue_a_possibly_sent_row(tmp_path) -> None:
assert DurableWriteDrain(queue, execute=_never_execute).claim(limit=5) == ()
def test_a_refused_source_context_fails_instead_of_parking_as_unknown(
def test_a_refused_source_context_is_nonblocking_unknown(
tmp_path,
) -> None:
"""A refusal raised before the call must not read as a possibly-sent write.
Issue #44: an OpenCode import carried `role=user, written_by=unknown`, so
`HyWriteBindings` — built inside `ingest`, ahead of the payload reaching HY
— refused the batch. The invocation marker is recorded before that call, so
the row was parked as `unknown`: a state that says "HY may hold a write we
cannot confirm" about a request that was never sent. It kept the reason off
the row and stopped the queue behind it.
"""
"""An ambiguous call-boundary failure is retained but never blocks FIFO."""
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
record = queue.submit(_request(key="refused-scope", source="opencode"))
@@ -4848,14 +4835,12 @@ def test_a_refused_source_context_fails_instead_of_parking_as_unknown(
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
state, invoked, last_error = db.execute(
"SELECT state, hy_invoked, last_error FROM memory_write_jobs "
"WHERE job_id=?",
"SELECT state, hy_invoked, last_error FROM memory_write_jobs WHERE job_id=?",
(record.job_id,),
).fetchone()
assert state == FAILED
# Nothing was sent, so reclaim must not treat this as possibly-sent.
assert invoked == 0
assert state == UNKNOWN
assert invoked == 1
assert last_error == "hy_memory_write_verification_context_scope"
assert DurableWriteDrain(queue, execute=_never_execute).claim(limit=1) == ()
@@ -4873,9 +4858,7 @@ def test_a_failure_after_the_call_still_parks_as_unknown(tmp_path) -> None:
record.job_id,
queue=queue,
supervisor=_Recorder(
error=HyMemoryBridgeError(
"hy_memory_write_verification_authority_invalid"
),
error=HyMemoryBridgeError("hy_memory_write_verification_authority_invalid"),
invoke_hy=True,
),
evidence=_EvidenceStore({"e1"}),
@@ -4893,40 +4876,27 @@ def test_a_failure_after_the_call_still_parks_as_unknown(tmp_path) -> None:
assert invoked == 1
def test_a_window_holding_only_blocked_rows_advances_instead_of_starving(
def test_event_time_not_enqueue_order_drives_the_window(
tmp_path,
) -> None:
"""A window whose remaining rows FIFO holds back must be retired.
Live shape: the anchor sat on 2026-10-07T03:24Z while the four pending rows
inside it were each blocked by 900+ earlier siblings of the same Harness
whose own event times sat outside the range (the earliest was 2026-03-12).
None of them could ever be claimed from inside that window, and because the
window only asked whether the range still held *some* pending row it never
retired: 3.8k queued turns sat at zero completions until restart, which
does not help because the anchor is persisted.
"""
"""A later-enqueued old turn runs before an earlier-enqueued new turn."""
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
# The anchor itself: another Harness, claimable, inside the range.
opener = queue.submit(
_request(key="starve-opener", source="codex", event_at=anchor)
)
# Same Harness as the blocked row but an hour later: outside the range, and
# it sorts earlier in queue_seq, so FIFO holds the row below behind it.
older_seq_newer_time = queue.submit(
enqueued_first = queue.submit(
_request(
key="starve-head",
key="newer-event",
source="workbuddy",
session="newer-session",
event_at=anchor + timedelta(hours=1),
)
)
blocked = queue.submit(
enqueued_second = queue.submit(
_request(
key="starve-blocked",
key="older-event",
source="workbuddy",
session="older-session",
event_at=anchor + timedelta(minutes=5),
)
)
@@ -4938,36 +4908,70 @@ def test_a_window_holding_only_blocked_rows_advances_instead_of_starving(
max_concurrent_sources=10,
)
# The blocked row is inside the window but FIFO-refused, so it is not
# claimed alongside the opener.
assert drain.claim(limit=10) == (opener.job_id,)
assert drain.claim(limit=10) == (enqueued_second.job_id,)
queue.settle(
opener.job_id,
enqueued_second.job_id,
state=COMPLETED,
hy_invoked=True,
owner="starve-owner",
)
# The window now holds exactly one row and that row can never be claimed
# from inside it. Retire the window and re-anchor on the blocker.
assert drain.claim(limit=10) == (older_seq_newer_time.job_id,)
assert drain.claim(limit=10) == (enqueued_first.job_id,)
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
window = db.execute(
"SELECT window_anchor FROM memory_write_schedule_state "
"WHERE singleton=1"
"SELECT window_anchor FROM memory_write_schedule_state WHERE singleton=1"
).fetchone()
assert window is not None
assert str(window[0]) == (
anchor + timedelta(hours=1)
).astimezone(UTC).isoformat()
assert str(window[0]) == (anchor + timedelta(hours=1)).astimezone(UTC).isoformat()
def test_same_session_uses_event_time_then_queue_sequence(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
base = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
later = queue.submit(
_request(key="same-session-later", source="cursor", event_at=base + timedelta(hours=1))
)
earlier = queue.submit(_request(key="same-session-earlier", source="cursor", event_at=base))
drain = DurableWriteDrain(
queue,
execute=_never_execute,
owner="event-owner",
max_concurrent_sources=10,
)
assert drain.claim(limit=10) == (earlier.job_id,)
queue.settle(
older_seq_newer_time.job_id,
earlier.job_id,
state=COMPLETED,
hy_invoked=True,
owner="starve-owner",
owner="event-owner",
)
# Once the blocker is gone the deferred row keeps its original event time
# and is reached on the next pass.
assert drain.claim(limit=10) == (blocked.job_id,)
assert drain.claim(limit=10) == (later.job_id,)
def test_equal_event_time_uses_queue_sequence_inside_one_session(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
queue = DurableWriteQueue(path)
event_at = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
first = queue.submit(
_request(key="equal-first", source="cursor", event_at=event_at)
)
second = queue.submit(
_request(key="equal-second", source="cursor", event_at=event_at)
)
drain = DurableWriteDrain(
queue,
execute=_never_execute,
owner="equal-owner",
max_concurrent_sources=10,
)
assert drain.claim(limit=10) == (first.job_id,)
queue.settle(
first.job_id,
state=COMPLETED,
hy_invoked=True,
owner="equal-owner",
)
assert drain.claim(limit=10) == (second.job_id,)
@@ -2103,7 +2103,7 @@ def test_migration_49_resets_only_cursor_history_when_conversation_time_is_missi
result = asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
assert result.applied == (49, 50, 51, 52, 53)
assert result.applied == (49, 50, 51, 52, 53, 54)
with main_database(path).read() as connection:
assert connection.execute("SELECT COUNT(*) FROM source_cursors").fetchone()[0] == 0
assert connection.execute("SELECT COUNT(*) FROM live_source_cursors").fetchone()[0] == 1
@@ -2136,7 +2136,7 @@ def test_upgrade_drops_obsolete_hy_node_permissions_without_touching_memories(
result = asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
assert result.applied == (51, 52, 53)
assert result.applied == (51, 52, 53, 54)
with sqlite3.connect(path) as connection:
tables = {
row[0]
@@ -2217,3 +2217,55 @@ def test_migration_53_backfills_write_job_event_time_from_person_turn(
"SELECT 1 FROM sqlite_master "
"WHERE type='table' AND name='memory_write_schedule_state'"
).fetchone() == (1,)
def test_migration_54_preserves_legacy_possibly_sent_tasks(tmp_path) -> None:
import sqlite3
from one_memory.memory_server.migration_runner import SqliteMigrationStore
path = tmp_path / "one-memory.sqlite3"
through_53 = tuple(item for item in MIGRATIONS if item.version <= 53)
asyncio.run(migrate(SqliteMigrationStore(path), through_53))
with sqlite3.connect(path) as connection, connection:
connection.execute(
"""
INSERT INTO memory_tasks (
task_id,request_id,user_id,kind,input_ref,status,
profile_id,profile_revision,profile_snapshot_json,
attempts,created_at,updated_at,hy_invocation_count
) VALUES (?,?,?,?,?,?,?,?,?,0,?,?,0)
""",
(
"legacy-sent",
"legacy-sent-request",
"local",
"extract",
"evidence",
"running",
"profile",
1,
"{}",
"2026-10-18T00:00:00+00:00",
"2026-10-18T00:00:00+00:00",
),
)
connection.execute(
"CREATE TABLE memory_task_continuous_permissions "
"(task_id TEXT PRIMARY KEY)"
)
connection.execute(
"INSERT INTO memory_task_continuous_permissions VALUES (?)",
("legacy-sent",),
)
asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
with sqlite3.connect(path) as connection:
assert connection.execute(
"SELECT hy_invocation_count FROM memory_tasks WHERE task_id='legacy-sent'"
).fetchone() == (1,)
assert connection.execute(
"SELECT 1 FROM sqlite_master "
"WHERE type='table' AND name='memory_task_continuous_permissions'"
).fetchone() is None
@@ -2,6 +2,8 @@
from __future__ import annotations
import asyncio
import json
import sqlite3
from datetime import UTC, datetime
@@ -9,13 +11,23 @@ import pytest
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
from one_memory.imports.storage import EvidenceStore
from one_memory.memory_server.durable_queue import DurableWriteQueue
from one_memory.memory_server.durable_queue import (
DurableWriteDrain,
DurableWriteQueue,
execute_job,
)
from one_memory.memory_server.unified_entry import (
UnifiedWriteIntake,
)
def _record(store: EvidenceStore, *, conversation_id: str = "c1", message_id: str = "m1"):
def _record(
store: EvidenceStore,
*,
conversation_id: str = "c1",
message_id: str = "m1",
verbatim: str = "durable intake",
):
return store.put(
EvidenceInput(
source_id="cursor",
@@ -24,7 +36,7 @@ def _record(store: EvidenceStore, *, conversation_id: str = "c1", message_id: st
role="user",
written_by=WrittenBy.PERSON,
created_at=datetime.now(UTC),
verbatim="durable intake",
verbatim=verbatim,
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
)
).record
@@ -172,6 +184,136 @@ def test_replaying_the_same_evidence_does_not_create_a_second_row(tmp_path) -> N
assert replay.enqueued is False
def test_rewound_harness_revision_appends_a_job_with_immutable_input(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(path)
intake = UnifiedWriteIntake(DurableWriteQueue(path, evidence=store))
old = _record(store, verbatim="old branch")
old_job = intake.submit_evidence(old, first_round=1, last_round=1)
revised = _record(store, verbatim="new branch")
new_job = intake.submit_evidence(revised, first_round=1, last_round=1)
assert revised.evidence_id == old.evidence_id
assert revised.revision == old.revision + 1
assert new_job.record.job_id != old_job.record.job_id
assert intake.queue.payload_for(old_job.record.job_id).input_records[0].verbatim == "old branch"
assert intake.queue.payload_for(new_job.record.job_id).input_records[0].verbatim == "new branch"
assert intake.queue.unsubmitted_evidence_ids() == ()
with sqlite3.connect(path) as connection:
assert connection.execute("SELECT COUNT(*) FROM memory_write_jobs").fetchone()[0] == 2
def test_revision_committed_before_job_crash_is_discovered_for_reconcile(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(path)
queue = DurableWriteQueue(path, evidence=store)
intake = UnifiedWriteIntake(queue)
old = _record(store, verbatim="old branch")
intake.submit_evidence(old, first_round=1, last_round=1)
revised = _record(store, verbatim="new branch")
assert revised.evidence_id == old.evidence_id
assert revised.revision == old.revision + 1
assert queue.unsubmitted_evidence_ids() == (revised.evidence_id,)
def test_late_old_revision_snapshot_does_not_cover_the_current_revision(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(path)
queue = DurableWriteQueue(path, evidence=store)
intake = UnifiedWriteIntake(queue)
old = _record(store, verbatim="old branch")
revised = _record(store, verbatim="new branch")
intake.submit_evidence(old, first_round=1, last_round=1)
assert revised.evidence_id == old.evidence_id
assert revised.revision == old.revision + 1
assert queue.unsubmitted_evidence_ids() == (revised.evidence_id,)
def test_old_branch_job_executes_its_frozen_revision_after_harness_rewind(tmp_path) -> None:
path = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(path)
queue = DurableWriteQueue(path, evidence=store)
intake = UnifiedWriteIntake(queue)
old = _record(store, verbatim="old branch")
old_job = intake.submit_evidence(old, first_round=1, last_round=1)
revised = _record(store, verbatim="new branch")
intake.submit_evidence(revised, first_round=1, last_round=1)
drain = DurableWriteDrain(queue, execute=lambda _job_id: asyncio.sleep(0), owner="frozen")
assert drain.claim(limit=1) == (old_job.record.job_id,)
captured = []
class Supervisor:
async def process_records(self, records, *, before_hy_invocation):
captured.extend(records)
await before_hy_invocation()
asyncio.run(
execute_job(
old_job.record.job_id,
queue=queue,
supervisor=Supervisor(),
evidence=store,
owner="frozen",
)
)
assert [record.verbatim for record in captured] == ["old branch"]
assert queue.get(old_job.request_id).state == "completed"
def test_reopening_reads_an_old_id_only_job_without_rewriting_it(tmp_path) -> None:
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
path = tmp_path / "one-memory.sqlite3"
store = EvidenceStore(path)
record = _record(store)
queue = DurableWriteQueue(path, evidence=store)
legacy = queue.submit(
WriteRequest(
internal_session_id=record.conversation_id,
source_id=record.source_id,
batch_identity=BatchIdentity(
internal_session_id=record.conversation_id,
first_round=1,
last_round=1,
),
evidence_ids=[record.evidence_id],
idempotency_key="legacy-id-only",
)
)
with sqlite3.connect(path) as connection, connection:
payload = json.loads(
connection.execute(
"SELECT payload_json FROM memory_write_jobs WHERE job_id=?",
(legacy.job_id,),
).fetchone()[0]
)
payload.pop("input_records", None)
connection.execute(
"UPDATE memory_write_jobs SET payload_json=? WHERE job_id=?",
(json.dumps(payload, sort_keys=True), legacy.job_id),
)
assert queue.payload_for(legacy.job_id).input_records == ()
reopened = DurableWriteQueue(path, evidence=store)
assert reopened.payload_for(legacy.job_id).input_records == ()
with sqlite3.connect(path) as connection:
persisted = json.loads(
connection.execute(
"SELECT payload_json FROM memory_write_jobs WHERE job_id=?",
(legacy.job_id,),
).fetchone()[0]
)
assert "input_records" not in persisted
def test_committed_evidence_without_submit_is_the_red_state(tmp_path) -> None:
"""The exact crash window the C0 wiring has to close.
+2 -8
View File
@@ -1500,14 +1500,8 @@ def test_restart_splits_a_sent_continuous_task_from_an_unsent_one(tmp_path) -> N
input_ref="evidence-unsent",
profile=profile,
)
with sqlite3.connect(store.database_path) as connection, connection:
connection.execute(
"CREATE TABLE memory_task_continuous_permissions (task_id TEXT PRIMARY KEY)"
)
connection.execute(
"INSERT INTO memory_task_continuous_permissions VALUES (?)", (sent.task_id,)
)
store.claim(sent.task_id)
_sent_task, sent_token = store.claim(sent.task_id)
store.record_hy_invocation(sent.task_id, runner_token=sent_token)
store.claim(unsent.task_id)
recovered = store.recover_running()
@@ -132,19 +132,22 @@ def test_raw_authors_keep_exact_body_and_do_not_mutate_evidence(role, author):
@pytest.mark.parametrize(
"change",
[
{"role": "user", "written_by": WrittenBy.AGENT},
{"role": "assistant", "written_by": WrittenBy.PERSON},
{"role": "tool", "written_by": WrittenBy.TOOL},
{"role": "system"},
{"written_by": WrittenBy.UNKNOWN},
{"content_kind": EvidenceContentKind.MEMORY_CONTENT},
],
)
def test_raw_and_author_qualification_rejects_without_tokenizing(change):
def test_raw_role_qualification_rejects_without_tokenizing(change):
with pytest.raises(MappedContentError, match="mapped_content_ineligible"):
prepare_mapped_content(record("ordinary body", **change))
@pytest.mark.parametrize("author", list(WrittenBy))
def test_host_author_guess_does_not_change_protocol_role_qualification(author):
content = prepare_mapped_content(record("ordinary body", role="user", written_by=author))
assert content.state == "ready"
@pytest.mark.parametrize(
"text,reason",
[
@@ -25,7 +25,6 @@ def _message(
message_id=message_id or f"m{ordinal}",
ordinal=ordinal,
role=role,
written_by="person" if role == "user" else "model",
content=content,
)
@@ -39,7 +39,11 @@ def record(
source_id=source,
conversation_id=f"c{index % 2}",
message_id=str(index),
role="assistant" if author == WrittenBy.AGENT else "user",
role=(
"assistant"
if author == WrittenBy.AGENT
else ("tool" if author == WrittenBy.TOOL else "user")
),
written_by=author,
created_at=time,
verbatim=text or f"original {index}",
@@ -83,8 +87,8 @@ def test_receipt_counts_saved_records_without_inventing_dates_or_other_sources(t
data = response.json()
assert data["source_id"] == "claude-code"
assert data["messages"] == 5 and data["conversations"] == 2
assert data["person_messages"] == 2 and data["agent_messages"] == 1
assert data["unknown_author_messages"] == 1 and data["tool_messages"] == 1
assert data["person_messages"] == 3 and data["agent_messages"] == 1
assert data["unknown_author_messages"] == 0 and data["tool_messages"] == 1
assert data["unknown_time_messages"] == 4
assert len(data["examples"]) == 3
assert "excluded-source" not in response.text and "excluded-user" not in response.text
@@ -34,8 +34,8 @@ def test_a_heartbeat_is_not_a_question():
assert last_row_opens_an_unanswered_turn(("user", "person", "<heartbeat>ping")) is False
def test_a_message_written_by_the_agent_as_user_is_not_a_question():
assert last_row_opens_an_unanswered_turn(("user", "agent", "继续")) is False
def test_protocol_user_role_opens_a_turn_regardless_of_legacy_author_guess():
assert last_row_opens_an_unanswered_turn(("user", "agent", "继续")) is True
def test_a_conversation_with_no_rows_is_not_unanswered():
@@ -15,7 +15,7 @@ fixed ten-minute event-time window.
- Parse native Harness timestamps.
- Parse Cursor `<timestamp>` envelopes before stripping them.
- Mark fallback quality explicitly.
- Derive each chunk's time from its person-authored user opener.
- Derive each chunk's time from its protocol `user` opener.
2. **Durable schema**
- Add `event_at` and `time_quality` to `memory_write_jobs`.
@@ -25,7 +25,8 @@ fixed ten-minute event-time window.
3. **Windowed Drain**
- Anchor at the oldest eligible pending event.
- Admit only `[anchor, anchor + 10 minutes)`.
- Keep one running job per Harness and FIFO by `queue_seq`.
- Keep one running job per Harness; order globally by `event_at`.
- Use `queue_seq` only to break equal-time ties within one Session.
- Advance only after the current window has no eligible pending/running
work.
@@ -16,12 +16,12 @@ The row stores:
- `event_at`: UTC event time used for scheduling;
- `time_quality`: `exact`, `anchored`, `session`, or `approximate`;
- `queue_seq`: the existing monotonic tie-breaker and replay order;
- `queue_seq`: a deterministic tie-breaker only when event times are equal;
- `session_key` and `source_key`: the existing session/Harness identities.
Time selection is:
1. the person-authored user turn timestamp;
1. the protocol `user` turn timestamp;
2. the nearest session timestamp anchor;
3. the session/file timestamp;
4. `imported_at` or write creation time.
@@ -58,8 +58,10 @@ Within one window:
- different Harnesses may run concurrently, up to the existing concurrency
limit;
- one Harness still has at most one running job;
- earlier `queue_seq` values within a Harness block later values;
- jobs are selected by `(event_at, queue_seq, job_id)`;
- each Session is ordered by `(event_at, queue_seq)`;
- different Sessions from one Harness never block each other's event order;
- `source_key` limits one running job per Harness but does not define FIFO;
- jobs are selected globally by `(event_at, queue_seq, job_id)`;
- `live` affects service admission only and does not jump the time window.
When the active window is empty, the next oldest eligible pending job becomes
@@ -91,16 +93,31 @@ batch are not held.
- Claimed but not invoked: return to pending without consuming an attempt.
- Already invoked: the current provider call may finish; it is never blindly
cancelled or retried.
- Unknown outcomes retain the existing reconcile-before-retry contract.
- Unknown outcomes retain their complete frozen input, never block another
Session or Harness, and follow the bounded reconcile-before-retry contract.
## Immutable job input and branches
Every new `memory_write_jobs.payload_json` stores the complete immutable
Evidence records used by the write, not only references to another table.
Execution and unknown reconciliation use that snapshot. Terminal jobs retain
it as audit data.
When a Harness rewinds and continues from an earlier point, already stored
Evidence and jobs are not deleted. Shared unchanged messages deduplicate;
changed or new branch records append new jobs. A later revision of a reused
host message id receives a new write id while the earlier job keeps its
original content.
## Upgrade compatibility
Legacy jobs are assigned:
1. the latest person-user Evidence time linked to the job;
1. the latest `role=user` Evidence time linked to the job;
2. otherwise the earliest linked Evidence time;
3. otherwise the original job creation time.
They are marked `approximate` and retain `queue_seq` as a deterministic
tie-breaker. No payload, Evidence identity, or idempotency key is rewritten.
tie-breaker. Old id-only payloads remain readable; all newly submitted jobs
are self-contained.