Compare commits
35
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
db6fd1b8e3 | ||
|
|
5176683937 | ||
|
|
6150ceb9cc | ||
|
|
6054f9921a | ||
|
|
ba70b84b00 | ||
|
|
fb71eafa35 | ||
|
|
cdd549fdd1 | ||
|
|
0df84c086c | ||
|
|
dc0ba49fbe | ||
|
|
ce705e9e4a | ||
|
|
334b14472b | ||
|
|
38baa9eebf | ||
|
|
0083a355a5 | ||
|
|
c049facb70 | ||
|
|
c681fdcf24 | ||
|
|
d0a52c9863 | ||
|
|
01259d01f5 | ||
|
|
0e436e9b70 | ||
|
|
ec99618e1f | ||
|
|
c53c874c21 | ||
|
|
80f21d29a7 | ||
|
|
2cc2f02450 | ||
|
|
5be27fd87f | ||
|
|
7aaf0f8557 | ||
|
|
887583b858 | ||
|
|
fcdb2307e2 | ||
|
|
9ed744b455 | ||
|
|
1b90f04a50 | ||
|
|
0822e7137e | ||
|
|
bcc0751060 | ||
|
|
183b61e736 | ||
|
|
e038006c7f | ||
|
|
16953ed49e | ||
|
|
4ff948c7e1 | ||
|
|
346f2653f2 |
No files matched your search
@@ -1123,6 +1123,7 @@ class Extractor:
|
||||
max_tokens=self._llm_config.agent_max_tokens,
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
audit_stage="extract",
|
||||
)
|
||||
self._call_count += 1
|
||||
total_tokens += response.tokens_used
|
||||
@@ -1233,6 +1234,7 @@ class Extractor:
|
||||
max_tokens=self._llm_config.agent_max_tokens,
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
audit_stage="extract",
|
||||
)
|
||||
self._call_count += 1
|
||||
total_tokens += response.tokens_used
|
||||
|
||||
@@ -34,6 +34,7 @@ import os
|
||||
import weakref
|
||||
|
||||
from ..config import MemoryConfig
|
||||
from ..utils.model_call_context import model_call_stage
|
||||
from ..core.workload import llm_slot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -348,6 +349,7 @@ class LLMProvider:
|
||||
stop: List[str] = None,
|
||||
tools: Optional[List[Dict[str, Any]]] = None,
|
||||
tool_choice: Optional[Any] = None,
|
||||
audit_stage: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> LLMResponse:
|
||||
"""
|
||||
@@ -386,15 +388,16 @@ class LLMProvider:
|
||||
len(prompt), _content_fingerprint(prompt),
|
||||
)
|
||||
|
||||
response = await self._call_llm(
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
stop=stop,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
**kwargs
|
||||
)
|
||||
with model_call_stage(audit_stage):
|
||||
response = await self._call_llm(
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
stop=stop,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
self._total_calls += 1
|
||||
self._total_tokens += response.tokens_used
|
||||
@@ -595,6 +598,7 @@ class LLMProvider:
|
||||
temperature: Optional[float] = None,
|
||||
tools: Optional[List[Dict[str, Any]]] = None,
|
||||
tool_choice: Optional[Any] = None,
|
||||
audit_stage: Optional[str] = None,
|
||||
) -> LLMResponse:
|
||||
"""
|
||||
多轮 messages 调用(支持 tool results 回传)。
|
||||
@@ -622,13 +626,14 @@ class LLMProvider:
|
||||
f"msgs={len(messages)} tools={len(tools) if tools else 0}"
|
||||
)
|
||||
|
||||
response = await self._chat_completions(
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
with model_call_stage(audit_stage):
|
||||
response = await self._chat_completions(
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
|
||||
self._total_calls += 1
|
||||
self._total_tokens += response.tokens_used
|
||||
|
||||
@@ -669,6 +669,7 @@ class MemoryReconciler:
|
||||
response = await llm.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=500,
|
||||
audit_stage="reconcile",
|
||||
)
|
||||
text = response.content.strip()
|
||||
if "```json" in text:
|
||||
@@ -1383,6 +1384,7 @@ class MemoryReconciler:
|
||||
response = await llm.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=dyn_max_tokens,
|
||||
audit_stage="reconcile",
|
||||
)
|
||||
except Exception as e:
|
||||
# LLM 彻底不可用(provider 已按 max_retries 重试过,网关 500 /
|
||||
|
||||
@@ -919,6 +919,7 @@ class CodingMemoryExtractor:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
audit_stage="coding_extract",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[coding-extract] LLM call failed: {e}")
|
||||
|
||||
@@ -571,6 +571,7 @@ class WorkingFactReconciler:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
audit_stage="coding_fact_reconcile",
|
||||
)
|
||||
return self._parse_ops(resp.content)
|
||||
|
||||
@@ -637,6 +638,7 @@ class WorkingFactReconciler:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=_CHAIN_SUMMARY_MAX_TOKENS,
|
||||
audit_stage="coding_fact_summary",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[fact-reconcile] chain summary failed: {e}")
|
||||
|
||||
@@ -29,6 +29,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_JUDGE_MAX_TURNS = 20
|
||||
_JUDGE_HEAD_TURNS = 5
|
||||
_JUDGE_USER_CONTENT_LIMIT = 1_000
|
||||
_JUDGE_USER_CONTENT_HEAD = 200
|
||||
_JUDGE_USER_CONTENT_TAIL = 200
|
||||
_JUDGE_MAX_QUERY_CHARS = 300
|
||||
_JUDGE_MAX_QUERIES = 20
|
||||
|
||||
@@ -186,6 +189,7 @@ async def classify_messages_scene(
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
audit_stage="coding_judge",
|
||||
)
|
||||
_elapsed = round((time.perf_counter() - _t0) * 1000, 2)
|
||||
result = _parse_scene_json(resp.content)
|
||||
@@ -631,9 +635,13 @@ def _format_turns(summary: List[Dict[str, Any]]) -> str:
|
||||
rows = []
|
||||
for t in summary:
|
||||
user = t.get("user", "") or ""
|
||||
# 单 turn 用户文本截到 200 字符
|
||||
if len(user) > 200:
|
||||
user = user[:200] + "..."
|
||||
if len(user) > _JUDGE_USER_CONTENT_LIMIT:
|
||||
omitted = len(user) - _JUDGE_USER_CONTENT_HEAD - _JUDGE_USER_CONTENT_TAIL
|
||||
user = (
|
||||
user[:_JUDGE_USER_CONTENT_HEAD]
|
||||
+ f"\n[...{omitted} chars omitted...]\n"
|
||||
+ user[-_JUDGE_USER_CONTENT_TAIL:]
|
||||
)
|
||||
tools = t.get("tools", []) or []
|
||||
rows.append(
|
||||
f"- turn {t.get('turn', '?')}: user={user!r}, tools={tools}"
|
||||
|
||||
@@ -310,6 +310,7 @@ class CodingMemoryReconciler:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
audit_stage="coding_reconcile",
|
||||
)
|
||||
return self._parse_ops(resp.content), prompt, resp.content, resp
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Context propagated to host-provided HTTP transports for cost accounting."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar, Token
|
||||
|
||||
_model_call_stage: ContextVar[str] = ContextVar(
|
||||
"hy_memory_model_call_stage",
|
||||
default="other",
|
||||
)
|
||||
|
||||
|
||||
def current_model_call_stage() -> str:
|
||||
return _model_call_stage.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def model_call_stage(stage: str | None) -> Iterator[None]:
|
||||
normalized = (stage or "other").strip().lower() or "other"
|
||||
token: Token[str] = _model_call_stage.set(normalized)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_model_call_stage.reset(token)
|
||||
@@ -80,7 +80,11 @@ class PipelineLogWriter:
|
||||
split_namespace_user,
|
||||
)
|
||||
|
||||
tokens = (prompt_tokens or 0) + (completion_tokens or 0) + (total_tokens or 0)
|
||||
# Providers define total_tokens as prompt+completion. Adding all
|
||||
# three fields doubled every cost estimate in this JSONL.
|
||||
tokens = (total_tokens or 0) or (
|
||||
(prompt_tokens or 0) + (completion_tokens or 0)
|
||||
)
|
||||
namespace, uid = split_namespace_user(user_id or "")
|
||||
if not is_llm_content_logging_enabled():
|
||||
prompt = redact_content(prompt)
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
"dataLicense": "CC0-1.0",
|
||||
"SPDXID": "SPDXRef-DOCUMENT",
|
||||
"name": "one-memory",
|
||||
"documentNamespace": "https://one-memory.local/spdx/2026-10-17T11:07:57Z",
|
||||
"documentNamespace": "https://one-memory.local/spdx/2026-10-17T13:52:48Z",
|
||||
"creationInfo": {
|
||||
"created": "2026-10-17T11:07:57Z",
|
||||
"created": "2026-10-17T13:52:48Z",
|
||||
"creators": [
|
||||
"Tool: one-memory-generate-sbom"
|
||||
]
|
||||
@@ -2335,7 +2335,7 @@
|
||||
{
|
||||
"SPDXID": "SPDXRef-Package-hy-memory",
|
||||
"name": "hy-memory",
|
||||
"versionInfo": "2.0.3",
|
||||
"versionInfo": "2.0.4",
|
||||
"downloadLocation": "NOASSERTION",
|
||||
"filesAnalyzed": false,
|
||||
"licenseConcluded": "Apache-2.0",
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
MemHub itself is licensed under Apache-2.0 (see `LICENSE`).
|
||||
It bundles the runtime dependencies listed below.
|
||||
Generated from the installed dependency tree on 2026-10-17T11:07:57Z
|
||||
Generated from the installed dependency tree on 2026-10-17T13:52:48Z
|
||||
by `python3 scripts/generate_sbom.py`. Do not edit by hand.
|
||||
|
||||
Per-package license texts are not reproduced here yet; the license
|
||||
column is filled in as part of release review, and the SBOM records
|
||||
`NOASSERTION` until then rather than guessing.
|
||||
|
||||
## hy-memory 2.0.3
|
||||
## hy-memory 2.0.4
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@one-memory/cli",
|
||||
"version": "1.0.3",
|
||||
"version": "1.0.4",
|
||||
"description": "MemHub bootstrap CLI for installing and managing the local-first shared memory runtime",
|
||||
"license": "Apache-2.0",
|
||||
"repository": {
|
||||
|
||||
@@ -384,6 +384,101 @@ describe("unified context search", () => {
|
||||
expect(streams.stdout()).not.toContain("must-not-print");
|
||||
});
|
||||
|
||||
it("counts characters as code points so an emoji hit stays grounded", async () => {
|
||||
// Core reports used_characters with Python's len(), which counts code
|
||||
// points: this text is 10 characters there and 12 UTF-16 units here.
|
||||
const text = "消费保 📍 扩展 📡";
|
||||
expect(text.length).toBe(12);
|
||||
expect(Array.from(text).length).toBe(10);
|
||||
transport.post.mockResolvedValue({
|
||||
status: 200,
|
||||
data: {
|
||||
...RESPONSE,
|
||||
paths: {
|
||||
...RESPONSE.paths,
|
||||
memory: {
|
||||
...RESPONSE.paths.memory,
|
||||
status: "empty",
|
||||
returned_count: 0,
|
||||
},
|
||||
},
|
||||
items: [
|
||||
{
|
||||
...(RESPONSE.items[0] as Record<string, unknown>),
|
||||
messages: [
|
||||
{
|
||||
...(
|
||||
RESPONSE.items[0] as {
|
||||
messages: Array<Record<string, unknown>>;
|
||||
}
|
||||
).messages[0],
|
||||
text,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
used_characters: 10,
|
||||
},
|
||||
});
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
expect(await run(["recall", "test", "--json"])).toBe(0);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(JSON.parse(streams.stdout())).toMatchObject({
|
||||
state: "grounded",
|
||||
used_characters: 10,
|
||||
items: [{ kind: "raw", messages: [{ text }] }],
|
||||
});
|
||||
});
|
||||
|
||||
it("still rejects a response whose character total does not match", async () => {
|
||||
// The same text counted as UTF-16 units is exactly what Core does not mean,
|
||||
// so the cross-check must keep failing instead of accepting both totals.
|
||||
const text = "消费保 📍 扩展 📡";
|
||||
transport.post.mockResolvedValue({
|
||||
status: 200,
|
||||
data: {
|
||||
...RESPONSE,
|
||||
paths: {
|
||||
...RESPONSE.paths,
|
||||
memory: {
|
||||
...RESPONSE.paths.memory,
|
||||
status: "empty",
|
||||
returned_count: 0,
|
||||
},
|
||||
},
|
||||
items: [
|
||||
{
|
||||
...(RESPONSE.items[0] as Record<string, unknown>),
|
||||
messages: [
|
||||
{
|
||||
...(
|
||||
RESPONSE.items[0] as {
|
||||
messages: Array<Record<string, unknown>>;
|
||||
}
|
||||
).messages[0],
|
||||
text,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
used_characters: 12,
|
||||
},
|
||||
});
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
expect(await run(["recall", "test", "--json"])).toBe(1);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(JSON.parse(streams.stdout())).toMatchObject({
|
||||
state: "invalid",
|
||||
code: "response_invalid",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports endpoint denial without exposing the response body", async () => {
|
||||
transport.post.mockResolvedValue({
|
||||
status: 403,
|
||||
|
||||
@@ -181,9 +181,9 @@ function validateContextResponse(
|
||||
items.push(item);
|
||||
characters +=
|
||||
item.kind === "memory"
|
||||
? String(item.content).length
|
||||
? characterCount(item.content)
|
||||
: (item.messages as Array<Record<string, unknown>>).reduce(
|
||||
(sum, message) => sum + String(message.text).length,
|
||||
(sum, message) => sum + characterCount(message.text),
|
||||
0,
|
||||
);
|
||||
}
|
||||
@@ -330,6 +330,17 @@ function record(value: unknown): Record<string, unknown> | null {
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count characters the way Core does. Python's len() counts Unicode code
|
||||
* points; String.prototype.length counts UTF-16 code units, so an emoji outside
|
||||
* the BMP is one character to Core and two here. Comparing the two totals made
|
||||
* every response containing an emoji fail the used_characters cross-check and
|
||||
* surface as response_invalid.
|
||||
*/
|
||||
function characterCount(value: unknown): number {
|
||||
return Array.from(String(value)).length;
|
||||
}
|
||||
|
||||
function boundedText(
|
||||
value: unknown,
|
||||
maximum: number,
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
||||
|
||||
[project]
|
||||
name = "one-memory"
|
||||
version = "1.0.3"
|
||||
version = "1.0.4"
|
||||
description = "MemHub local-first memory core"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -12,7 +12,7 @@ dependencies = [
|
||||
"fastapi>=0.141.1",
|
||||
"filelock>=3.32.4",
|
||||
# Public SDK version validated against MemHub's permission and recall contracts.
|
||||
"hy-memory==2.0.3",
|
||||
"hy-memory==2.0.4",
|
||||
# Retrieval's derived vector index directly consumes these locked versions.
|
||||
"chromadb==1.5.9",
|
||||
"numpy==2.5.2",
|
||||
|
||||
@@ -100,7 +100,13 @@ from one_memory.imports.access import (
|
||||
SourceAccessError,
|
||||
root_identity_matches,
|
||||
)
|
||||
from one_memory.imports.adapters import SourceAdapter, SyncCompleteness
|
||||
from one_memory.imports.adapters import (
|
||||
ScanFailure,
|
||||
ScannedEvidence,
|
||||
ScanOptions,
|
||||
SourceAdapter,
|
||||
SyncCompleteness,
|
||||
)
|
||||
from one_memory.imports.candidates import (
|
||||
SourceCandidateError,
|
||||
SourceCandidateSelection,
|
||||
@@ -111,7 +117,11 @@ from one_memory.imports.connections import (
|
||||
ConnectionIntentReceipt,
|
||||
SourceConnectionStore,
|
||||
)
|
||||
from one_memory.imports.grants import SourceGrantReceipt, SourceGrantStore
|
||||
from one_memory.imports.grants import (
|
||||
SourceAuthorizationState,
|
||||
SourceGrantReceipt,
|
||||
SourceGrantStore,
|
||||
)
|
||||
from one_memory.imports.host_packages import CapabilityState, HostPackageCapabilities
|
||||
from one_memory.imports.jobs import (
|
||||
ImportIssueRecord,
|
||||
@@ -190,6 +200,7 @@ from one_memory.model_profiles import (
|
||||
ModelProfileStore,
|
||||
ModelProfileUnavailableError,
|
||||
)
|
||||
from one_memory.model_request_audit import AuditedModelTransport, ModelRequestAuditStore
|
||||
from one_memory.model_request_budget import BudgetedModelTransport, ModelRequestBudget
|
||||
from one_memory.onboarding import OnboardingState, OnboardingStore
|
||||
from one_memory.recall import GovernedRecallTarget, RecallReceipt, RecallRequest
|
||||
@@ -281,6 +292,9 @@ class SourceResponse(BaseModel):
|
||||
candidate_revision: int = 0
|
||||
candidate_change_allowed: bool = False
|
||||
candidate_change_reason: str | None = None
|
||||
#: Why `authorized` is false, so the UI can ask for the one thing that
|
||||
#: would fix it instead of offering every recovery at once.
|
||||
authorization_state: SourceAuthorizationState = SourceAuthorizationState.OK
|
||||
|
||||
|
||||
class PreviewSourceCandidateRequest(BaseModel):
|
||||
@@ -313,6 +327,22 @@ class SourceSyncState(BaseModel):
|
||||
sync_enabled: bool
|
||||
|
||||
|
||||
class SedimentationRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
enabled: StrictBool
|
||||
|
||||
|
||||
class SedimentationState(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
enabled: bool
|
||||
#: What the switch means for the model bill and for what still runs. Read
|
||||
#: by the app so the setting can say so without hardcoding the wording.
|
||||
pending_retained: bool = True
|
||||
imports_unaffected: bool = True
|
||||
|
||||
|
||||
class DetectionResponse(BaseModel):
|
||||
source_id: str
|
||||
detected: bool
|
||||
@@ -339,7 +369,7 @@ class ConnectSourcesRequest(BaseModel):
|
||||
import_upload_id: UUID | None = None
|
||||
|
||||
|
||||
ONBOARDING_LARGE_HISTORY_SESSIONS = 200
|
||||
ONBOARDING_LARGE_HISTORY_USER_TURNS = 200
|
||||
ONBOARDING_HISTORY_WINDOW = timedelta(days=3)
|
||||
|
||||
|
||||
@@ -886,6 +916,7 @@ def create_app(
|
||||
)
|
||||
app.state.import_job_store = ImportJobStore(database_path)
|
||||
app.state.onboarding_store = OnboardingStore(database_path)
|
||||
app.state.history_import_gate_released = False
|
||||
app.state.extraction_settings_store = ExtractionSettingsStore(database_path)
|
||||
app.state.source_grant_store = SourceGrantStore(
|
||||
database_path,
|
||||
@@ -908,6 +939,7 @@ def create_app(
|
||||
app.state.source_permission_locks = {
|
||||
source_id: asyncio.Lock() for source_id in app.state.source_catalog
|
||||
}
|
||||
app.state.sedimentation_lock = asyncio.Lock()
|
||||
_follow_workbuddy_home(app, runtime_settings.source_home)
|
||||
app.state.source_detection = {}
|
||||
app.state.secret_store = secret_store or KeyringSecretStore()
|
||||
@@ -974,6 +1006,7 @@ def create_app(
|
||||
else None
|
||||
)
|
||||
app.state.model_request_budget = model_request_budget
|
||||
app.state.model_request_audit_store = ModelRequestAuditStore(database_path)
|
||||
app.state.memory_answerer = MemoryAnswerer(
|
||||
governance=app.state.memory_governance_store,
|
||||
filter_memories=app.state.source_read_access.memories,
|
||||
@@ -1016,10 +1049,17 @@ def create_app(
|
||||
profile=profile,
|
||||
api_key=api_key,
|
||||
data_dir=data_dir,
|
||||
http_transport_factory=(
|
||||
(lambda: BudgetedModelTransport(model_request_budget))
|
||||
if model_request_budget is not None
|
||||
else None
|
||||
http_transport_factory=lambda: AuditedModelTransport(
|
||||
app.state.model_request_audit_store,
|
||||
profile_id=profile.profile_id,
|
||||
profile_revision=profile.revision,
|
||||
provider_kind=str(profile.provider_kind),
|
||||
model=profile.model,
|
||||
inner=(
|
||||
BudgetedModelTransport(model_request_budget)
|
||||
if model_request_budget is not None
|
||||
else None
|
||||
),
|
||||
),
|
||||
),
|
||||
# `None` means "whatever the active profile allows". Pinned to
|
||||
@@ -1032,6 +1072,7 @@ def create_app(
|
||||
max_concurrent_tasks=runtime_settings.precipitation_max_concurrent_tasks,
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
service_levels=app.state.service_levels,
|
||||
sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(),
|
||||
)
|
||||
# C2 owns the single production budget (handle 5s + reserve 250ms,
|
||||
# total 5.25s, lease 10s, renew 1s, claim 250ms). The composition
|
||||
@@ -1064,8 +1105,12 @@ def create_app(
|
||||
mark_invoked: Callable[[], Awaitable[bool]],
|
||||
) -> bool:
|
||||
"""Serialize pause with the last boundary before an HY call."""
|
||||
lock = app.state.source_permission_locks.setdefault(source_id, asyncio.Lock())
|
||||
async with lock:
|
||||
source_lock = app.state.source_permission_locks.setdefault(
|
||||
source_id, asyncio.Lock()
|
||||
)
|
||||
async with app.state.sedimentation_lock, source_lock:
|
||||
if not _config_store(app).sedimentation_enabled():
|
||||
raise RuntimeError("model_sedimentation_disabled")
|
||||
if not _source_writes_on(app, source_id):
|
||||
raise RuntimeError("model_content_source_disabled")
|
||||
return await mark_invoked()
|
||||
@@ -1103,8 +1148,18 @@ def create_app(
|
||||
+ 10
|
||||
),
|
||||
reconcile_unknown=reconcile_unknown_writes,
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
source_claim_enabled=lambda source_id: _config_store(app).sync_enabled(source_id),
|
||||
source_writes_permitted=lambda source_id: _sedimentation_claim_enabled(
|
||||
app, source_id
|
||||
),
|
||||
# The claim gate has to be the same question the write gate
|
||||
# asks, not a second answer to it. It used to be
|
||||
# `sync_enabled`, the per-app toggle, and `one-memory-manual` is
|
||||
# not a connected app: it has no entry in the configuration
|
||||
# document, so that predicate answers False for it while the
|
||||
# write gate admits it by its grant. Every manual note's job was
|
||||
# therefore skipped on every drain tick — the note stayed
|
||||
# "saved, waiting" with no task behind it, forever.
|
||||
source_claim_enabled=lambda source_id: _sedimentation_claim_enabled(app, source_id),
|
||||
)
|
||||
app.state.memory_write_drain = memory_write_drain
|
||||
app.state.memory_write_drain_task = watched_task(
|
||||
@@ -1133,10 +1188,7 @@ def create_app(
|
||||
for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True):
|
||||
if (
|
||||
record not in chunk
|
||||
or not any(
|
||||
item.role == "user" and item.written_by.value == "person"
|
||||
for item in chunk
|
||||
)
|
||||
or not any(item.role == "user" for item in chunk)
|
||||
or not any(item.role == "assistant" for item in chunk)
|
||||
):
|
||||
continue
|
||||
@@ -1749,6 +1801,30 @@ def create_app(
|
||||
"events": events,
|
||||
}
|
||||
|
||||
@app.get("/api/v1/diagnostics/model-usage", dependencies=protected)
|
||||
def diagnostic_model_usage(
|
||||
since: datetime | None = Query(default=None), # noqa: B008
|
||||
request_id: str | None = Query( # noqa: B008
|
||||
default=None, min_length=1, max_length=512
|
||||
),
|
||||
limit: int = Query(default=200, ge=1, le=2_000), # noqa: B008
|
||||
) -> dict[str, object]:
|
||||
"""Return content-free precipitation usage for provider reconciliation."""
|
||||
|
||||
normalized_since = (
|
||||
(since if since.tzinfo is not None else since.replace(tzinfo=UTC))
|
||||
if since is not None
|
||||
else None
|
||||
)
|
||||
report = app.state.model_request_audit_store.report(
|
||||
since=normalized_since.astimezone(UTC).isoformat()
|
||||
if normalized_since is not None
|
||||
else None,
|
||||
request_id=request_id,
|
||||
limit=limit,
|
||||
)
|
||||
return {"schema_version": 1, **report}
|
||||
|
||||
@app.get(
|
||||
"/api/v1/runtime/identity",
|
||||
response_model=HealthResponse,
|
||||
@@ -1937,6 +2013,11 @@ def create_app(
|
||||
candidate_change_reason=app.state.source_candidate_store.change_reason(
|
||||
registration
|
||||
),
|
||||
authorization_state=grant_store.authorization_state(
|
||||
user_id="local",
|
||||
source_id=source_id,
|
||||
candidate=registration.candidate.path,
|
||||
),
|
||||
)
|
||||
for source_id, registration in _catalog(app).items()
|
||||
]
|
||||
@@ -2248,24 +2329,39 @@ def create_app(
|
||||
raise HTTPException(status_code=409, detail=error.code) from None
|
||||
if grant is None:
|
||||
raise HTTPException(status_code=409, detail="grant_required")
|
||||
await _prepare_source_resume(source_id)
|
||||
try:
|
||||
await _prepare_source_resume(source_id)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_prepare_failed") from error
|
||||
else:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
try:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_prepare_failed") from error
|
||||
try:
|
||||
await asyncio.to_thread(_config_store(app).set_sync, source_id, request.enabled)
|
||||
except BaseException:
|
||||
except BaseException as error:
|
||||
if request.enabled:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
else:
|
||||
await _prepare_source_resume(source_id)
|
||||
await _activate_source_resume(source_id)
|
||||
await _refresh_live_watch(app)
|
||||
raise
|
||||
raise _sync_refused("sync_config_write_failed") from error
|
||||
try:
|
||||
if request.enabled:
|
||||
await _activate_source_resume(source_id)
|
||||
await _refresh_live_watch(app)
|
||||
try:
|
||||
await _activate_source_resume(source_id)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_resume_failed") from error
|
||||
try:
|
||||
await _refresh_live_watch(app)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_watcher_failed") from error
|
||||
except BaseException:
|
||||
# Rolls back the desired state and settles the watcher exactly as
|
||||
# before; the only change is that the raise above now carries a
|
||||
# code saying which stage failed.
|
||||
await asyncio.to_thread(_config_store(app).set_sync, source_id, previous)
|
||||
if request.enabled:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
@@ -2396,6 +2492,8 @@ def create_app(
|
||||
async def connect_sources(
|
||||
request: ConnectSourcesRequest,
|
||||
) -> ConnectSourcesResponse:
|
||||
if request.onboarding:
|
||||
app.state.history_import_gate_released = False
|
||||
source_ids = tuple(dict.fromkeys(request.source_ids))
|
||||
registrations = [_require_registration(app, source_id) for source_id in source_ids]
|
||||
if request.request_id is None:
|
||||
@@ -3561,6 +3659,48 @@ def create_app(
|
||||
def put_extraction_settings(request: ExtractionWindowUpdate) -> ExtractionSettings:
|
||||
return _memory_processing_supervisor(app).set_extraction_window(request.window_preset)
|
||||
|
||||
@app.get(
|
||||
"/api/v1/sedimentation",
|
||||
response_model=SedimentationState,
|
||||
dependencies=protected,
|
||||
)
|
||||
def get_sedimentation() -> SedimentationState:
|
||||
"""Whether the model may still turn conversations into memories."""
|
||||
return SedimentationState(enabled=_config_store(app).sedimentation_enabled())
|
||||
|
||||
@app.put(
|
||||
"/api/v1/sedimentation",
|
||||
response_model=SedimentationState,
|
||||
dependencies=protected,
|
||||
)
|
||||
async def put_sedimentation(request: SedimentationRequest) -> SedimentationState:
|
||||
"""Start or stop model-driven sedimentation, keeping the configuration.
|
||||
|
||||
Stopping is the user's cost control: nothing further goes to the model
|
||||
and already queued work waits rather than failing, so starting again
|
||||
resumes it. Importing conversations is not affected either way.
|
||||
"""
|
||||
store = _config_store(app)
|
||||
changed = False
|
||||
async with app.state.sedimentation_lock:
|
||||
previous = store.sedimentation_enabled()
|
||||
if previous is not request.enabled:
|
||||
await asyncio.to_thread(store.set_sedimentation_enabled, request.enabled)
|
||||
changed = True
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"enabled": request.enabled},
|
||||
)
|
||||
if changed and request.enabled:
|
||||
# Work parked by the stop is picked up again rather than waiting
|
||||
# for the next natural dispatch, but not on this request's time:
|
||||
# see _resume_sedimentation.
|
||||
_resume_sedimentation(app)
|
||||
return SedimentationState(enabled=store.sedimentation_enabled())
|
||||
|
||||
@app.post(
|
||||
"/api/v1/memory-processing/manual-extract",
|
||||
response_model=ManualExtractReceipt,
|
||||
@@ -3921,6 +4061,7 @@ async def _authorize_answer_generation(
|
||||
# The explicit question is this operation's input, not a standing permission for history.
|
||||
if not question.strip() or not grounding:
|
||||
raise MemoryAnswerError("memory_answer_generation_not_authorized")
|
||||
|
||||
def check() -> None:
|
||||
if _model_profile_store(app).active_identity() != (
|
||||
profile.profile_id,
|
||||
@@ -3944,6 +4085,7 @@ async def _authorize_answer_generation(
|
||||
await recheck()
|
||||
return recheck
|
||||
|
||||
|
||||
class _LazyASGIApp:
|
||||
"""Compatibility ASGI app that is built only when a server imports it.
|
||||
|
||||
@@ -3975,6 +4117,16 @@ class _LazyASGIApp:
|
||||
app = _LazyASGIApp()
|
||||
|
||||
|
||||
def _sync_refused(code: str) -> HTTPException:
|
||||
"""A sync refusal that says which stage refused.
|
||||
|
||||
Turning the switch on takes four steps: authorize, prepare, write, resume.
|
||||
The UI used to be told only that the switch had sprung back, and every
|
||||
stage needs a different fix, so each one gets its own code.
|
||||
"""
|
||||
return HTTPException(status_code=409, detail=code)
|
||||
|
||||
|
||||
def _granted_roots(
|
||||
source_ids: Iterable[str],
|
||||
grant_store: SourceGrantStore,
|
||||
@@ -4026,6 +4178,7 @@ def _source_response(
|
||||
totals: tuple[int, int],
|
||||
*,
|
||||
candidate_change_reason: str | None = None,
|
||||
authorization_state: SourceAuthorizationState = SourceAuthorizationState.OK,
|
||||
) -> SourceResponse:
|
||||
location = inspect_source_location(
|
||||
registration.descriptor.source_id, registration.candidate.path
|
||||
@@ -4238,7 +4391,7 @@ async def _onboarding_history_after(
|
||||
home: Path,
|
||||
now: datetime | None = None,
|
||||
) -> datetime | None:
|
||||
"""Return one shared cutoff when any selected harness has large history.
|
||||
"""Return one shared cutoff when any selected harness has large, old history.
|
||||
|
||||
The cutoff is decided before any history job is created, then persisted on
|
||||
every accepted job. This is deliberately global: independently limiting
|
||||
@@ -4247,6 +4400,8 @@ async def _onboarding_history_after(
|
||||
"""
|
||||
if not request.onboarding:
|
||||
return None
|
||||
anchor = now or datetime.now(UTC)
|
||||
cutoff = anchor - ONBOARDING_HISTORY_WINDOW
|
||||
revisions = request.source_candidate_revisions or {}
|
||||
for registration in registrations:
|
||||
source_id = registration.descriptor.source_id
|
||||
@@ -4274,29 +4429,52 @@ async def _onboarding_history_after(
|
||||
home=home,
|
||||
)
|
||||
adapter = registration.create_adapter(grant)
|
||||
count_sessions = cast(
|
||||
Callable[..., Awaitable[int]] | None,
|
||||
getattr(adapter, "count_sessions", None),
|
||||
)
|
||||
if count_sessions is not None:
|
||||
session_count = await count_sessions(limit=ONBOARDING_LARGE_HISTORY_SESSIONS + 1)
|
||||
else:
|
||||
pending = _install_state(adapter)
|
||||
if pending is None:
|
||||
continue
|
||||
session_count = (await pending).session_count
|
||||
if await _has_large_old_history(adapter, cutoff=cutoff):
|
||||
return cutoff
|
||||
except (OSError, SourceAccessError):
|
||||
# A supported source whose metadata count cannot be completed is
|
||||
# treated conservatively. The later connection still owns the
|
||||
# precise authorization error, while onboarding never falls back
|
||||
# to an unbounded history import.
|
||||
session_count = ONBOARDING_LARGE_HISTORY_SESSIONS + 1
|
||||
if session_count > ONBOARDING_LARGE_HISTORY_SESSIONS:
|
||||
anchor = now or datetime.now(UTC)
|
||||
return anchor - ONBOARDING_HISTORY_WINDOW
|
||||
return cutoff
|
||||
return None
|
||||
|
||||
|
||||
async def _has_large_old_history(adapter: SourceAdapter, *, cutoff: datetime) -> bool:
|
||||
"""Probe one authorized Harness with the same parsers the import will use.
|
||||
|
||||
Stop as soon as both policy facts are proven: more than 200 genuine person
|
||||
turns and at least one record older than the three-day cutoff. Counting
|
||||
parsed Evidence instead of transcript files prevents one giant coding
|
||||
session from bypassing the limit, while reusing the adapters keeps host
|
||||
envelopes and tool-shaped user records out of the count.
|
||||
"""
|
||||
|
||||
user_turns = 0
|
||||
has_old_data = False
|
||||
async for event in adapter.scan(ScanOptions()):
|
||||
if isinstance(event, ScanFailure):
|
||||
# A partial probe must never turn into an unbounded first import.
|
||||
return True
|
||||
if not isinstance(event, ScannedEvidence):
|
||||
continue
|
||||
observed = event.evidence.created_at
|
||||
if observed is None and event.cursor.observed_modified_ns > 0:
|
||||
observed = datetime.fromtimestamp(
|
||||
event.cursor.observed_modified_ns / 1_000_000_000,
|
||||
tz=UTC,
|
||||
)
|
||||
if observed is not None:
|
||||
if observed.tzinfo is None:
|
||||
observed = observed.replace(tzinfo=UTC)
|
||||
has_old_data = has_old_data or observed.astimezone(UTC) < cutoff
|
||||
if event.evidence.role == "user":
|
||||
user_turns += 1
|
||||
if user_turns > ONBOARDING_LARGE_HISTORY_USER_TURNS and has_old_data:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _install_state(adapter: object) -> Awaitable[SourceInstallState] | None:
|
||||
"""Ask an adapter for the two-level answer when it has one.
|
||||
|
||||
@@ -4588,7 +4766,7 @@ async def _keep_loop_heartbeat(watchdog: LoopWatchdog) -> None:
|
||||
async def _emit_resource_snapshots(interval_seconds: float = 60.0) -> None:
|
||||
while True:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="core.resources",
|
||||
event="core.resource_snapshot",
|
||||
outcome="observed",
|
||||
@@ -4613,6 +4791,33 @@ def _start_backfill(app: FastAPI) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _resume_sedimentation(app: FastAPI) -> None:
|
||||
"""Pick the work parked by a stop back up, off the caller's clock.
|
||||
|
||||
Turning the switch back on is a decision, not a job the caller waits for:
|
||||
the choice is already stored, and what follows it is maintenance over every
|
||||
unfinished task followed by a scan of the whole queue. On a real backlog
|
||||
that runs for many minutes, so awaiting it here left the PUT unanswered
|
||||
until the client gave up — the renderer showed "启用未能确认" for a switch
|
||||
that had in fact been saved, and the switch only appeared to turn on once
|
||||
the follow-up read came back. Measured: one toggle, twenty-plus minutes,
|
||||
no response, client timeout at thirty seconds.
|
||||
|
||||
Repeated toggles reuse the running walk; it reads the switch as it goes, so
|
||||
a stop part-way through is honoured on the next batch instead of needing a
|
||||
cancel.
|
||||
"""
|
||||
running = cast("asyncio.Task[None] | None", getattr(app.state, "sedimentation_resume", None))
|
||||
if running is not None and not running.done():
|
||||
return
|
||||
app.state.sedimentation_resume = watched_task(
|
||||
asyncio.get_running_loop(),
|
||||
lambda: _memory_processing_supervisor(app).resume_allowed(),
|
||||
component="memory.sedimentation",
|
||||
name="one-memory-sedimentation-resume",
|
||||
)
|
||||
|
||||
|
||||
def _start_visibility_recompute(app: FastAPI) -> None:
|
||||
"""Keep the stored visibility conclusions current, in the background.
|
||||
|
||||
@@ -4856,6 +5061,51 @@ def _source_writes_on(app: FastAPI, source_id: str) -> bool:
|
||||
return _source_processing_enabled(app, source_id)
|
||||
|
||||
|
||||
def _sedimentation_claim_enabled(app: FastAPI, source_id: str) -> bool:
|
||||
"""Claim queued model work only while both global and source gates allow it."""
|
||||
|
||||
return (
|
||||
_config_store(app).sedimentation_enabled()
|
||||
and _history_import_gate_released(app)
|
||||
and _source_writes_on(app, source_id)
|
||||
)
|
||||
|
||||
|
||||
def _history_import_gate_released(app: FastAPI) -> bool:
|
||||
"""Hold first sedimentation until every accepted history import is terminal."""
|
||||
|
||||
if getattr(app.state, "history_import_gate_released", False):
|
||||
return True
|
||||
onboarding = getattr(app.state, "onboarding_store", None)
|
||||
jobs = getattr(app.state, "import_job_store", None)
|
||||
if onboarding is None or jobs is None:
|
||||
return True
|
||||
state = onboarding.get()
|
||||
if state.status != "importing" or not state.selected_source_ids:
|
||||
app.state.history_import_gate_released = True
|
||||
return True
|
||||
selected = set(state.selected_source_ids)
|
||||
matching = [
|
||||
job
|
||||
for job in jobs.list_jobs()
|
||||
if job.operation == "history"
|
||||
and job.source_id in selected
|
||||
and (state.started_at is None or job.created_at >= state.started_at)
|
||||
]
|
||||
if {job.source_id for job in matching} != selected:
|
||||
return False
|
||||
active = {
|
||||
ImportStatus.PENDING,
|
||||
ImportStatus.RUNNING,
|
||||
ImportStatus.PAUSED,
|
||||
ImportStatus.INTERRUPTED,
|
||||
}
|
||||
released = all(job.status not in active for job in matching)
|
||||
if released:
|
||||
app.state.history_import_gate_released = True
|
||||
return released
|
||||
|
||||
|
||||
def _model_prober(app: FastAPI) -> ModelProber:
|
||||
return cast(ModelProber, app.state.model_prober)
|
||||
|
||||
|
||||
@@ -175,6 +175,22 @@ async def in_interactive_read[T](
|
||||
return await in_read_pool(interactive_read_pool(), work, *args, **kwargs)
|
||||
|
||||
|
||||
def _retrieve[T](future: "asyncio.Future[T]") -> None:
|
||||
"""Mark a shielded read's outcome as seen.
|
||||
|
||||
`in_read_pool` shields the wrapped worker, so a caller that gives up and
|
||||
cancels leaves the read running to completion. When that read then fails,
|
||||
nothing awaits the future any more: asyncio reports the orphan at GC as
|
||||
"Future exception was never retrieved", the loop handler turns it into a
|
||||
`core.loop.task_exception` error diagnostic, and a lane that degraded on
|
||||
its own budget -- correctly, and already reported -- looks like a crashed
|
||||
task on top of it. Retrieving here keeps the diagnostics honest. Awaiting
|
||||
still raises for callers that are still there, so no failure is hidden.
|
||||
"""
|
||||
if not future.cancelled():
|
||||
future.exception()
|
||||
|
||||
|
||||
async def in_read_pool[T](
|
||||
pool: ThreadPoolExecutor,
|
||||
work: Callable[..., T],
|
||||
@@ -193,7 +209,9 @@ async def in_read_pool[T](
|
||||
_interactive_reads -= 1
|
||||
|
||||
worker.add_done_callback(released)
|
||||
return await asyncio.shield(asyncio.wrap_future(worker))
|
||||
pending = asyncio.wrap_future(worker)
|
||||
pending.add_done_callback(_retrieve)
|
||||
return await asyncio.shield(pending)
|
||||
|
||||
|
||||
async def wait_for_interactive_reads() -> None:
|
||||
|
||||
@@ -266,7 +266,13 @@ def serve(*, maintenance: bool = False, reader: bool = False) -> None:
|
||||
)
|
||||
application = create_reader_app(reader_settings)
|
||||
print(f"reader_port:{port}", file=sys.stderr, flush=True)
|
||||
uvicorn.run(application, host=settings.host, port=port, reload=False)
|
||||
uvicorn.run(
|
||||
application,
|
||||
host=settings.host,
|
||||
port=port,
|
||||
reload=False,
|
||||
access_log=False,
|
||||
)
|
||||
return
|
||||
|
||||
from one_memory.api import create_app
|
||||
@@ -297,9 +303,21 @@ def serve(*, maintenance: bool = False, reader: bool = False) -> None:
|
||||
)
|
||||
if maintenance:
|
||||
with CoreRuntime(settings, publish_receipt=False):
|
||||
uvicorn.run(application, host=settings.host, port=settings.port, reload=False)
|
||||
uvicorn.run(
|
||||
application,
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
reload=False,
|
||||
access_log=False,
|
||||
)
|
||||
else:
|
||||
uvicorn.run(application, host=settings.host, port=settings.port, reload=False)
|
||||
uvicorn.run(
|
||||
application,
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
reload=False,
|
||||
access_log=False,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -196,7 +196,7 @@ def read_raw_context(
|
||||
if _prefers_declared_facts(request.query)
|
||||
else "1"
|
||||
)
|
||||
person_authored = "CASE WHEN e.written_by='person' THEN 1 ELSE 0 END"
|
||||
person_authored = "CASE WHEN e.role='user' THEN 1 ELSE 0 END"
|
||||
query_echo = (
|
||||
"CASE WHEN trim(e.verbatim)=? OR trim(e.verbatim)=? THEN 0 ELSE 1 END"
|
||||
if _prefers_declared_facts(request.query)
|
||||
@@ -470,9 +470,8 @@ def revalidate_raw_context(
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
break
|
||||
if (row["role"], row["written_by"], created) != (
|
||||
if (row["role"], created) != (
|
||||
message.role,
|
||||
message.written_by,
|
||||
message.created_at,
|
||||
):
|
||||
break
|
||||
|
||||
@@ -66,12 +66,12 @@ def _field(text: str) -> ResumeField | None:
|
||||
return None
|
||||
|
||||
|
||||
def _raw_class(written_by: str, evidence_id: str, accepted: frozenset[str]) -> EvidenceClass:
|
||||
if written_by == "person":
|
||||
def _raw_class(role: str, evidence_id: str, accepted: frozenset[str]) -> EvidenceClass:
|
||||
if role == "user":
|
||||
return "person_statement"
|
||||
if written_by == "agent":
|
||||
if role == "assistant":
|
||||
return "agent_report"
|
||||
if written_by == "tool" and evidence_id in accepted:
|
||||
if role == "tool" and evidence_id in accepted:
|
||||
return "system_acceptance"
|
||||
return "tool_output"
|
||||
|
||||
@@ -118,7 +118,7 @@ def assemble_resume_packet(
|
||||
field=_field(message.text) or "unclassified",
|
||||
text=message.text,
|
||||
evidence_class=_raw_class(
|
||||
message.written_by,
|
||||
message.role,
|
||||
message.evidence_id,
|
||||
verified_system_evidence_ids,
|
||||
),
|
||||
|
||||
@@ -345,7 +345,7 @@ def _lexical_raw_rows(
|
||||
if context_raw._prefers_declared_facts(query.query)
|
||||
else "1"
|
||||
)
|
||||
person_authored = "CASE WHEN e.written_by='person' THEN 1 ELSE 0 END"
|
||||
person_authored = "CASE WHEN e.role='user' THEN 1 ELSE 0 END"
|
||||
query_echo = (
|
||||
"CASE WHEN trim(e.verbatim)=? OR trim(e.verbatim)=? THEN 0 ELSE 1 END"
|
||||
if context_raw._prefers_declared_facts(query.query)
|
||||
@@ -507,7 +507,7 @@ def _raw_item(row: sqlite3.Row, start1: int, matched_by: tuple[str, ...]) -> Raw
|
||||
"source_id": row["source_id"],
|
||||
"conversation_id": row["conversation_id"],
|
||||
"role": row["role"],
|
||||
"written_by": "person" if row["written_by"] == "person" else "agent",
|
||||
"written_by": "person" if row["role"] == "user" else "agent",
|
||||
"event_time": event_time.isoformat() if event_time else None,
|
||||
"spans": [{"start_cp": start, "end_cp": start + len(text), "text": text}],
|
||||
"matched_by": list(matched_by),
|
||||
|
||||
@@ -1039,7 +1039,7 @@ def close_main_databases() -> None:
|
||||
def log_main_database_stats(moment: str) -> None:
|
||||
"""One line in core.log: how many connections are open and who waited."""
|
||||
stats = main_database_stats()
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"main_database_connections moment=%s %s",
|
||||
moment,
|
||||
" ".join(f"{key}={value}" for key, value in stats.items()),
|
||||
|
||||
@@ -19,6 +19,7 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@@ -139,6 +140,10 @@ _DETAIL_KEYS = frozenset(
|
||||
"accounted_ms",
|
||||
"unaccounted_ms",
|
||||
"conversation_hash",
|
||||
# The sedimentation switch: a bounded boolean, the same shape as
|
||||
# `spawned` and `index_complete`. Left out of this list it did not
|
||||
# drop a log line, it turned the switch's own request into a 500.
|
||||
"enabled",
|
||||
}
|
||||
)
|
||||
_trace_id: ContextVar[str | None] = ContextVar("one_memory_trace_id", default=None)
|
||||
@@ -649,12 +654,24 @@ def diagnostic_event(**values: object) -> None:
|
||||
active = _active
|
||||
if active is None:
|
||||
return
|
||||
if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(
|
||||
logging.DEBUG
|
||||
):
|
||||
return
|
||||
try:
|
||||
active.write(**values) # type: ignore[arg-type]
|
||||
except OSError:
|
||||
# Diagnostics must not become a new product outage when a disk is full
|
||||
# or the log directory is temporarily unavailable.
|
||||
return
|
||||
except ValueError:
|
||||
# Same promise, same reason, for a rejected payload instead of a failed
|
||||
# write. An unlisted `details` key once turned every external recall
|
||||
# into a 500 and then killed the process, and the sedimentation switch
|
||||
# into a permanent failure. The payload itself is still wrong and is
|
||||
# still caught in CI by tests/test_diagnostic_payload_keys.py; what
|
||||
# must not happen is the caller paying for it.
|
||||
return
|
||||
|
||||
|
||||
def current_session_id() -> str | None:
|
||||
|
||||
@@ -33,6 +33,7 @@ from one_memory.hy_runtime_config import (
|
||||
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
|
||||
from one_memory.hy_store_concurrency import native_store_read_slot
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind, attach_disable_thinking
|
||||
from one_memory.model_request_audit import audit_precipitation_request
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
|
||||
#: D102: how relevant HY has to call a hit before it counts. Raised from 0.25
|
||||
@@ -143,6 +144,22 @@ class HyMemoryBridgeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES = 512
|
||||
|
||||
|
||||
def coding_tool_result_max_bytes() -> int:
|
||||
"""Return the per-tool payload ceiling used by MemHub's HY client."""
|
||||
|
||||
raw = os.environ.get("MEMORY_CODING_TOOL_RESULT_MAX_BYTES")
|
||||
if raw is None:
|
||||
return MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
return value if value > 0 else MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
|
||||
|
||||
def _exclusive[**P, T](
|
||||
method: Callable[P, Coroutine[Any, Any, T]],
|
||||
) -> Callable[P, Coroutine[Any, Any, T]]:
|
||||
@@ -414,8 +431,9 @@ class HyMemoryBridge:
|
||||
|
||||
@_exclusive
|
||||
async def ingest(self, request: HyMemoryIngestRequest) -> HyMemoryIngestResult:
|
||||
async with self._write_lock:
|
||||
return await self._ingest(request)
|
||||
with audit_precipitation_request(request.request_id):
|
||||
async with self._write_lock:
|
||||
return await self._ingest(request)
|
||||
|
||||
async def _ingest(self, request: HyMemoryIngestRequest) -> HyMemoryIngestResult:
|
||||
started = monotonic()
|
||||
@@ -431,9 +449,7 @@ class HyMemoryBridge:
|
||||
guards = getattr(client, "_one_memory_write_guards", ())
|
||||
from one_memory.hy_response_guard import HyReconcileGuard
|
||||
|
||||
reconcile_guards = tuple(
|
||||
guard for guard in guards if isinstance(guard, HyReconcileGuard)
|
||||
)
|
||||
reconcile_guards = tuple(guard for guard in guards if isinstance(guard, HyReconcileGuard))
|
||||
for guard in reconcile_guards:
|
||||
guard.begin_write()
|
||||
before = [len(guard.failures) for guard in guards]
|
||||
@@ -544,8 +560,9 @@ class HyMemoryBridge:
|
||||
request: HyMemoryIngestRequest,
|
||||
contents: Sequence[str],
|
||||
) -> HyMemoryIngestResult:
|
||||
async with self._write_lock:
|
||||
return await self._ingest_extracted(request, contents)
|
||||
with audit_precipitation_request(request.request_id):
|
||||
async with self._write_lock:
|
||||
return await self._ingest_extracted(request, contents)
|
||||
|
||||
async def _ingest_extracted(
|
||||
self,
|
||||
@@ -1110,7 +1127,7 @@ class HyMemoryBridge:
|
||||
self.profile,
|
||||
api_key=self.api_key,
|
||||
data_dir=self.data_dir,
|
||||
http_transport_factory=self._http_transport_factory,
|
||||
http_transport_factory=self._http_transport_factory,
|
||||
)
|
||||
|
||||
await _joined_thread(initialize)
|
||||
@@ -1137,7 +1154,7 @@ class HyMemoryBridge:
|
||||
)
|
||||
raise HyMemoryBridgeError("hy_memory_init_failed") from error
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="hy.client",
|
||||
event="hy.client_initialization.completed",
|
||||
outcome="succeeded",
|
||||
@@ -1328,9 +1345,7 @@ async def _with_native_mutation_delta(
|
||||
"""
|
||||
|
||||
retired = set(result.retired_memory_ids)
|
||||
superseded = {
|
||||
key: tuple(values) for key, values in result.superseded_memory_ids.items()
|
||||
}
|
||||
superseded = {key: tuple(values) for key, values in result.superseded_memory_ids.items()}
|
||||
chat_targets = {
|
||||
key
|
||||
for guard in reconcile_guards
|
||||
@@ -1338,7 +1353,8 @@ async def _with_native_mutation_delta(
|
||||
if isinstance(key, str) and key
|
||||
}
|
||||
|
||||
for item in raw.get("ops", ()) if isinstance(raw.get("ops"), list) else ():
|
||||
operations = raw.get("ops")
|
||||
for item in operations if isinstance(operations, list) else ():
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
if item.get("action") == "DELETE":
|
||||
@@ -1346,7 +1362,8 @@ async def _with_native_mutation_delta(
|
||||
if isinstance(key, str) and key:
|
||||
retired.add(key)
|
||||
|
||||
for item in raw.get("fact_ops", ()) if isinstance(raw.get("fact_ops"), list) else ():
|
||||
fact_operations = raw.get("fact_ops")
|
||||
for item in fact_operations if isinstance(fact_operations, list) else ():
|
||||
if not isinstance(item, Mapping) or item.get("action") != "SUPERSEDE":
|
||||
continue
|
||||
old, new = item.get("fact_id"), item.get("superseded_by")
|
||||
@@ -1367,9 +1384,7 @@ async def _with_native_mutation_delta(
|
||||
if isinstance(replacement, str) and replacement:
|
||||
superseded[key] = (replacement,)
|
||||
elif isinstance(replacement, list):
|
||||
values = tuple(
|
||||
value for value in replacement if isinstance(value, str) and value
|
||||
)
|
||||
values = tuple(value for value in replacement if isinstance(value, str) and value)
|
||||
if values:
|
||||
superseded[key] = values
|
||||
|
||||
@@ -1697,6 +1712,7 @@ def build_hy_memory_config(
|
||||
config.history.db_path = str(memory_root / "history.sqlite3")
|
||||
config.coding.enable = True
|
||||
config.coding.db_path = str(memory_root / "coding.sqlite3")
|
||||
config.coding.tool_result_max_bytes = coding_tool_result_max_bytes()
|
||||
config.recall.episode_recall_enabled = True
|
||||
config.recall.episode_summary_enabled = False
|
||||
config.recall.dynamic_episode_enabled = True
|
||||
|
||||
@@ -14,7 +14,7 @@ from pathlib import Path
|
||||
from typing import Literal, cast
|
||||
|
||||
HY_MEMORY_DISTRIBUTION = "hy-memory"
|
||||
HY_MEMORY_VERSION = "2.0.3"
|
||||
HY_MEMORY_VERSION = "2.0.4"
|
||||
HY_MEMORY_NAMESPACE = "governed-v2"
|
||||
DiagnosticLogLevel = Literal["error", "warning", "info", "debug"]
|
||||
|
||||
@@ -31,11 +31,12 @@ _COLLECTION_LOGGERS = (
|
||||
"httpx",
|
||||
"httpcore",
|
||||
)
|
||||
_NOISY_LIBRARY_LOGGERS = ("httpx", "httpcore", "watchfiles", "jieba")
|
||||
|
||||
# One Memory deliberately stays on ultra because its existing, user-visible
|
||||
# profile/graph flow calls digest(), which the pinned public SDK exposes only
|
||||
# for ultra clients. Reader and non-reranker behavior remain pinned for
|
||||
# compatibility with public SDK 2.0.3.
|
||||
# compatibility with public SDK 2.0.4.
|
||||
HY_MEMORY_MODE = "ultra"
|
||||
# The SDK only recognises `legacy` and `hybrid` (see
|
||||
# hy_memory/pipelines/_retrieval/config.py: `ALL_READERS = (READER_LEGACY,
|
||||
@@ -139,6 +140,18 @@ def apply_diagnostic_log_level(level: DiagnosticLogLevel) -> None:
|
||||
for name in tuple(logging.Logger.manager.loggerDict):
|
||||
if any(name == root or name.startswith(f"{root}.") for root in _COLLECTION_LOGGERS):
|
||||
logging.getLogger(name).setLevel(selected)
|
||||
# Successful HTTP exchanges, file-watch notifications and tokenizer
|
||||
# initialization are useful only while actively debugging. At INFO they
|
||||
# dominated core.log and the in-app panel without describing product work.
|
||||
noisy_level = selected if selected == logging.DEBUG else max(selected, logging.WARNING)
|
||||
for root in _NOISY_LIBRARY_LOGGERS:
|
||||
logging.getLogger(root).setLevel(noisy_level)
|
||||
for name in tuple(logging.Logger.manager.loggerDict):
|
||||
if any(
|
||||
name == root or name.startswith(f"{root}.")
|
||||
for root in _NOISY_LIBRARY_LOGGERS
|
||||
):
|
||||
logging.getLogger(name).setLevel(noisy_level)
|
||||
|
||||
os.environ["MEMORY_LOG_LEVEL"] = logging.getLevelName(selected)
|
||||
os.environ["MEMORY_PIPELINE_TRACE_ENABLED"] = "true" if level == "debug" else "false"
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
|
||||
|
||||
@@ -61,17 +61,7 @@ class HyWriteBindings:
|
||||
seed.workspace,
|
||||
)
|
||||
or record.content_kind is not EvidenceContentKind.RAW_CONVERSATION
|
||||
or (record.role, record.written_by)
|
||||
not in {
|
||||
# D98: the same change as the turn assembly. A sub agent's
|
||||
# question is a `user` message marked written by the
|
||||
# agent; excluding it here would make the source context
|
||||
# disagree with what was actually sent.
|
||||
("user", WrittenBy.PERSON),
|
||||
("user", WrittenBy.AGENT),
|
||||
("assistant", WrittenBy.AGENT),
|
||||
("tool", WrittenBy.TOOL),
|
||||
}
|
||||
or record.role not in {"user", "assistant", "tool"}
|
||||
):
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
self.all_sources: list[dict[str, Any]] = [
|
||||
|
||||
@@ -152,7 +152,7 @@ class HyStepTimer:
|
||||
if value is None:
|
||||
continue
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component=STEP_COMPONENTS[name],
|
||||
event="hy.step.completed",
|
||||
outcome="succeeded",
|
||||
@@ -168,7 +168,7 @@ class HyStepTimer:
|
||||
)
|
||||
model = measured.get("model")
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="hy.bridge",
|
||||
event="hy.steps.summary",
|
||||
outcome="succeeded",
|
||||
|
||||
@@ -105,7 +105,7 @@ async def iterate_in_worker[T](
|
||||
# was unwinding with, and a diagnostic is not worth an import.
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.async_bridge",
|
||||
event="import.source_read_finished",
|
||||
outcome="succeeded",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import sqlite3
|
||||
from contextlib import AbstractContextManager
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
@@ -20,6 +21,23 @@ CURSOR_MAX_SOURCE_FILE_BYTES = 64 * 1024 * 1024 * 1024
|
||||
GRANT_SCHEMA = source_grants_ddl("sqlite")
|
||||
|
||||
|
||||
class SourceAuthorizationState(StrEnum):
|
||||
"""Why a source cannot be read right now, or `ok` when it can.
|
||||
|
||||
The API used to answer `authorized=false` for all of these at once, which
|
||||
is how a revoked grant, a rebuilt history directory and a moved candidate
|
||||
all looked like the same bug and all got the same advice. The values carry
|
||||
no paths and no user content, because they are written to diagnostics.
|
||||
"""
|
||||
|
||||
OK = "ok"
|
||||
MISSING_GRANT = "missing_grant"
|
||||
REVOKED_GRANT = "revoked_grant"
|
||||
SOURCE_IDENTITY_CHANGED = "source_identity_changed"
|
||||
SOURCE_ROOT_UNAVAILABLE = "source_root_unavailable"
|
||||
CANDIDATE_CHANGED = "candidate_changed"
|
||||
|
||||
|
||||
class SourceGrantReceipt(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
@@ -146,6 +164,46 @@ class SourceGrantStore:
|
||||
raise SourceAccessError(SourceAccessErrorCode.SOURCE_IDENTITY_CHANGED)
|
||||
return authorized
|
||||
|
||||
def authorization_state(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
source_id: str,
|
||||
candidate: Path,
|
||||
) -> SourceAuthorizationState:
|
||||
"""Why reading this source would be refused.
|
||||
|
||||
Read-only and fails the same way `authorized_root` does: this names a
|
||||
state for the UI and for diagnostics, it does not widen access. The
|
||||
candidate is compared by path, so a grant that still points somewhere
|
||||
else reads as a moved candidate rather than as a rebuilt directory —
|
||||
the two need different fixes.
|
||||
"""
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM source_grants WHERE user_id=? AND source_id=?",
|
||||
(user_id, source_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return SourceAuthorizationState.MISSING_GRANT
|
||||
if row["revoked_at"] is not None or row["revision"] <= 0:
|
||||
return SourceAuthorizationState.REVOKED_GRANT
|
||||
granted = Path(row["requested_path"])
|
||||
if not granted.exists():
|
||||
return SourceAuthorizationState.SOURCE_ROOT_UNAVAILABLE
|
||||
try:
|
||||
if not root_identity_matches(granted, row["root_identity"]):
|
||||
return SourceAuthorizationState.SOURCE_IDENTITY_CHANGED
|
||||
except SourceAccessError:
|
||||
# Unreadable or gone between the two calls: there is no grant to
|
||||
# repair, only a directory to make available again.
|
||||
return SourceAuthorizationState.SOURCE_ROOT_UNAVAILABLE
|
||||
# Same directory, but the app now looks somewhere else. Re-authorizing
|
||||
# the old grant would not help; the candidate has to be granted.
|
||||
if granted != candidate:
|
||||
return SourceAuthorizationState.CANDIDATE_CHANGED
|
||||
return SourceAuthorizationState.OK
|
||||
|
||||
def revoke(self, *, user_id: str, source_id: str) -> SourceGrantReceipt:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
with self._connect(writing=True) as connection:
|
||||
|
||||
@@ -106,7 +106,7 @@ class PollingLiveSyncSupervisor:
|
||||
level=(
|
||||
"warn"
|
||||
if value.state in {LiveSyncState.DEGRADED, LiveSyncState.UNAVAILABLE}
|
||||
else "info"
|
||||
else "debug"
|
||||
),
|
||||
component="import.live_sync",
|
||||
event="live_sync.state_changed",
|
||||
|
||||
@@ -156,7 +156,7 @@ class FileWatchLiveSync:
|
||||
if source is not None
|
||||
):
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.live_watch",
|
||||
event="live_sync.hook_received",
|
||||
outcome="queued",
|
||||
@@ -190,7 +190,7 @@ class FileWatchLiveSync:
|
||||
try:
|
||||
await self._trigger_past(source_id, observed)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.live_watch",
|
||||
event="live_sync.hook_completed",
|
||||
outcome="written",
|
||||
|
||||
@@ -79,6 +79,8 @@ class EvidenceInput(BaseModel):
|
||||
source_ordinal: str | None = Field(default=None, min_length=1, max_length=256)
|
||||
content_kind: EvidenceContentKind = EvidenceContentKind.RAW_CONVERSATION
|
||||
role: str = Field(min_length=1, max_length=128)
|
||||
# Legacy import/storage metadata only. Protocol `role` is authoritative;
|
||||
# this value must not affect identity, admission, scheduling or ownership.
|
||||
written_by: WrittenBy
|
||||
created_at: datetime | None = None
|
||||
verbatim: str = Field(min_length=1, max_length=MAX_VERBATIM_CHARACTERS)
|
||||
@@ -149,17 +151,22 @@ class EvidenceInput(BaseModel):
|
||||
)
|
||||
)
|
||||
else:
|
||||
# `role` is the protocol-level speaker identity. `written_by` was a
|
||||
# parser guess and differed between two imports of the same host
|
||||
# message (notably sub-agent/user and unknown/user), minting two
|
||||
# Evidence rows. Keep it out of every new identity.
|
||||
#
|
||||
# No workspace and no source_ordinal in the derived key: the hook
|
||||
# lane has neither (it reports no workspace and a constant ordinal),
|
||||
# so including either gave the same message a different key per lane
|
||||
# and stored it twice. Both are record metadata here, not identity.
|
||||
identity = "\0".join(
|
||||
(
|
||||
"derived-v3",
|
||||
"derived-v4",
|
||||
self.user_id,
|
||||
self.source_id,
|
||||
self.conversation_id,
|
||||
self.written_by,
|
||||
self.role,
|
||||
self.content_hash,
|
||||
)
|
||||
)
|
||||
@@ -168,6 +175,54 @@ class EvidenceInput(BaseModel):
|
||||
# two rows. The capture time is record metadata, not identity.
|
||||
return hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
@property
|
||||
def written_by_derived_v3_idempotency_key(self) -> str:
|
||||
"""The last identity that treated the parser's author guess as canonical.
|
||||
|
||||
New builds key by protocol role. During an upgrade this key lets a
|
||||
rescan find and reuse the row already stored by derived-v3 instead of
|
||||
inserting the same host message under a second Evidence id.
|
||||
"""
|
||||
identity = "\0".join(
|
||||
(
|
||||
"derived-v3",
|
||||
self.user_id,
|
||||
self.source_id,
|
||||
self.conversation_id,
|
||||
self.written_by,
|
||||
self.content_hash,
|
||||
)
|
||||
)
|
||||
return hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
@property
|
||||
def written_by_derived_v3_idempotency_keys(self) -> tuple[str, ...]:
|
||||
"""Every legacy author guess this protocol role may have carried."""
|
||||
guesses = {
|
||||
"user": (
|
||||
WrittenBy.PERSON,
|
||||
WrittenBy.AGENT,
|
||||
WrittenBy.UNKNOWN,
|
||||
),
|
||||
"assistant": (WrittenBy.AGENT, WrittenBy.UNKNOWN, WrittenBy.PERSON),
|
||||
"tool": (WrittenBy.TOOL, WrittenBy.UNKNOWN),
|
||||
}.get(self.role, tuple(WrittenBy))
|
||||
ordered = (self.written_by, *guesses)
|
||||
keys: list[str] = []
|
||||
for written_by in dict.fromkeys(ordered):
|
||||
identity = "\0".join(
|
||||
(
|
||||
"derived-v3",
|
||||
self.user_id,
|
||||
self.source_id,
|
||||
self.conversation_id,
|
||||
written_by,
|
||||
self.content_hash,
|
||||
)
|
||||
)
|
||||
keys.append(hashlib.sha256(identity.encode()).hexdigest())
|
||||
return tuple(keys)
|
||||
|
||||
@property
|
||||
def legacy_message_idempotency_key(self) -> str | None:
|
||||
"""The pre-workspace key for a message-shaped input, when one exists.
|
||||
@@ -212,7 +267,13 @@ class EvidenceInput(BaseModel):
|
||||
"""
|
||||
return self.derived_v2_idempotency_key("derived-v2", workspace_agnostic=False)
|
||||
|
||||
def derived_v2_idempotency_key(self, label: str, *, workspace_agnostic: bool) -> str:
|
||||
def derived_v2_idempotency_key(
|
||||
self,
|
||||
label: str,
|
||||
*,
|
||||
workspace_agnostic: bool,
|
||||
written_by: WrittenBy | None = None,
|
||||
) -> str:
|
||||
identity = "\0".join(
|
||||
(
|
||||
label,
|
||||
@@ -220,7 +281,7 @@ class EvidenceInput(BaseModel):
|
||||
self.source_id,
|
||||
*([] if workspace_agnostic else [self.workspace or "\uffffnull"]),
|
||||
self.conversation_id,
|
||||
self.written_by,
|
||||
written_by or self.written_by,
|
||||
self.created_at.isoformat() if self.created_at else "",
|
||||
self.source_ordinal or "",
|
||||
# v2 hashed the cleaned body; v1 predates raw_verbatim entirely,
|
||||
@@ -230,6 +291,26 @@ class EvidenceInput(BaseModel):
|
||||
)
|
||||
return hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
@property
|
||||
def older_written_by_idempotency_keys(self) -> tuple[str, ...]:
|
||||
keys: list[str] = []
|
||||
for written_by in WrittenBy:
|
||||
keys.append(
|
||||
self.derived_v2_idempotency_key(
|
||||
"derived-v1",
|
||||
workspace_agnostic=True,
|
||||
written_by=written_by,
|
||||
)
|
||||
)
|
||||
keys.append(
|
||||
self.derived_v2_idempotency_key(
|
||||
"derived-v2",
|
||||
workspace_agnostic=False,
|
||||
written_by=written_by,
|
||||
)
|
||||
)
|
||||
return tuple(keys)
|
||||
|
||||
@property
|
||||
def evidence_id(self) -> str:
|
||||
return str(uuid5(EVIDENCE_NAMESPACE, self.idempotency_key))
|
||||
@@ -249,6 +330,7 @@ class EvidenceRecord(BaseModel):
|
||||
source_ordinal: str | None
|
||||
content_kind: EvidenceContentKind
|
||||
role: str
|
||||
# Retained while old databases and export formats still carry the column.
|
||||
written_by: WrittenBy
|
||||
created_at: datetime | None
|
||||
verbatim: str
|
||||
|
||||
@@ -199,7 +199,7 @@ class ImportRunner:
|
||||
raise ValueError("Import job source does not match adapter source")
|
||||
job, runner_token = await in_the_background(self.job_store.claim, job_id)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.parser",
|
||||
event="import.parse.started",
|
||||
outcome="started",
|
||||
@@ -310,7 +310,7 @@ class ImportRunner:
|
||||
if event.kind == "progress" and str(event.phase) != last_phase:
|
||||
phase = str(event.phase)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.parser",
|
||||
event="import.phase_changed",
|
||||
outcome="changed",
|
||||
@@ -516,7 +516,7 @@ class ImportRunner:
|
||||
async def _emit_parse_result(self, job: ImportJob, started: float) -> None:
|
||||
succeeded = job.status is ImportStatus.COMPLETED
|
||||
diagnostic_event(
|
||||
level="info" if succeeded else "warn",
|
||||
level="debug" if succeeded else "warn",
|
||||
component="import.parser",
|
||||
event="import.parse.completed",
|
||||
outcome="succeeded" if succeeded else str(job.status),
|
||||
|
||||
@@ -141,11 +141,18 @@ def parse_codex_record(
|
||||
if role == "user":
|
||||
written_by = _user_author(record, payload, session)
|
||||
# Machine inputs may quote a whole transcript, including user_query tags.
|
||||
# Session provenance is not proof of who wrote any particular future turn.
|
||||
if written_by is WrittenBy.PERSON:
|
||||
# Protocol role is authoritative; host author metadata does not change
|
||||
# how a user message is cleaned.
|
||||
if not (
|
||||
content.startswith(
|
||||
"The following is the Codex agent history whose request action you are assessing."
|
||||
)
|
||||
and ">>> TRANSCRIPT START" in content
|
||||
and ">>> TRANSCRIPT END" in content
|
||||
):
|
||||
content = strip_host_wrappers(content)
|
||||
if not content:
|
||||
return ParsedSkip(ScanIssueCode.NO_TEXT)
|
||||
if not content:
|
||||
return ParsedSkip(ScanIssueCode.NO_TEXT)
|
||||
return ParsedMessage(
|
||||
conversation_id=conversation_from_relative_path(context.relative_path),
|
||||
message_id=first_string(payload.get("id"), record.get("id")),
|
||||
|
||||
@@ -355,7 +355,7 @@ class JsonlSourceAdapter:
|
||||
source_file, checkpoint, header = await asyncio.to_thread(
|
||||
self._prepare, listed, options
|
||||
)
|
||||
_log.info(
|
||||
_log.debug(
|
||||
"import_file_read source=%s file=%d/%d file_bytes=%d planned_bytes=%d",
|
||||
self.descriptor.source_id,
|
||||
index,
|
||||
@@ -376,7 +376,7 @@ class JsonlSourceAdapter:
|
||||
else DEFAULT_IMPORT_MAX_FILE_BYTES
|
||||
)
|
||||
if source_file.size > budget:
|
||||
_log.info(
|
||||
_log.debug(
|
||||
"import_file_budgeted source=%s file=%d/%d file_bytes=%d budget=%d path=%s",
|
||||
self.descriptor.source_id,
|
||||
index,
|
||||
|
||||
@@ -7,7 +7,7 @@ import sqlite3
|
||||
import stat
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, datetime, timedelta, timezone
|
||||
from functools import partial
|
||||
|
||||
from pydantic import JsonValue
|
||||
@@ -82,6 +82,11 @@ _CURSOR_HOST_CONTEXT_START = re.compile(
|
||||
)
|
||||
_CURSOR_USER_QUERY = re.compile(r"<user_query\b")
|
||||
_CURSOR_TIMESTAMP_PREFIX = re.compile(r"<timestamp>[\s\S]*?</timestamp>\s*")
|
||||
_CURSOR_TIMESTAMP_VALUE = re.compile(
|
||||
r"<timestamp>\s*"
|
||||
r"([A-Za-z]+,\s+[A-Za-z]{3}\s+\d{1,2},\s+\d{4},\s+\d{1,2}:\d{2}\s+[AP]M)"
|
||||
r"\s+\(UTC([+-]\d{1,2})(?::(\d{2}))?\)\s*</timestamp>",
|
||||
)
|
||||
_CURSOR_IMAGE_FILES_PREFIX = re.compile(r"\[Image\]\s*<image_files>[\s\S]*?</image_files>\s*")
|
||||
_CURSOR_BARE_IMAGE_PREFIX = re.compile(r"(?:\[Image\]\s*)+")
|
||||
|
||||
@@ -123,6 +128,24 @@ def _strip_cursor_timestamp(content: str | None) -> str | None:
|
||||
return _CURSOR_TIMESTAMP_PREFIX.sub("", content, count=1) if content else None
|
||||
|
||||
|
||||
def _cursor_host_timestamp(content: str | None) -> datetime | None:
|
||||
"""Parse Cursor's user-visible timestamp envelope into UTC."""
|
||||
if not content:
|
||||
return None
|
||||
match = _CURSOR_TIMESTAMP_VALUE.search(content)
|
||||
if match is None:
|
||||
return None
|
||||
try:
|
||||
local = datetime.strptime(match[1], "%A, %b %d, %Y, %I:%M %p")
|
||||
hours = int(match[2])
|
||||
minutes = int(match[3] or "0")
|
||||
sign = -1 if hours < 0 else 1
|
||||
offset = timedelta(hours=hours, minutes=sign * minutes)
|
||||
return local.replace(tzinfo=timezone(offset)).astimezone(UTC)
|
||||
except (ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _strip_cursor_host_prefix(content: str | None) -> str | None:
|
||||
if not content:
|
||||
return content
|
||||
@@ -398,10 +421,7 @@ class CursorUnifiedSourceAdapter:
|
||||
if (
|
||||
isinstance(event, ScannedEvidence)
|
||||
and event.evidence.conversation_id in transcript_ids
|
||||
and not (
|
||||
event.evidence.written_by is WrittenBy.AGENT
|
||||
and is_host_generated_user_turn(event.evidence.verbatim)
|
||||
)
|
||||
and not is_host_generated_user_turn(event.evidence.verbatim)
|
||||
):
|
||||
yield ScanSkip(
|
||||
cursor=event.cursor,
|
||||
@@ -426,10 +446,7 @@ def _shadowed_by_transcript(event: ScanEvent, transcript_ids: set[str]) -> ScanE
|
||||
if (
|
||||
isinstance(event, ScannedEvidence)
|
||||
and event.evidence.conversation_id in transcript_ids
|
||||
and not (
|
||||
event.evidence.written_by is WrittenBy.AGENT
|
||||
and is_host_generated_user_turn(event.evidence.verbatim)
|
||||
)
|
||||
and not is_host_generated_user_turn(event.evidence.verbatim)
|
||||
):
|
||||
return ScanSkip(
|
||||
cursor=event.cursor,
|
||||
@@ -703,6 +720,7 @@ def _parse_transcript_record(
|
||||
project_name = path_parts[path_parts.index("projects") + 1] if "projects" in path_parts else ""
|
||||
session_name = context.relative_path.rsplit("/", 1)[-1].removesuffix(".jsonl")
|
||||
is_subagent = f"{project_name}\0{session_name}" in subagent_ids
|
||||
host_timestamp = _cursor_host_timestamp(content) if role == "user" else None
|
||||
created_at = _transcript_fallback_time(context)
|
||||
native_message_id = record.get("id") or message.get("id")
|
||||
message_id = (
|
||||
@@ -728,7 +746,16 @@ def _parse_transcript_record(
|
||||
created_at=created_at,
|
||||
workspace=workspace,
|
||||
record_type="cursor_transcript",
|
||||
origin_metadata=OriginMetadata(raw_meta={"time_basis": "file_mtime"}),
|
||||
origin_metadata=OriginMetadata(
|
||||
raw_meta={
|
||||
"time_basis": "file_mtime",
|
||||
**(
|
||||
{"turn_event_at": host_timestamp.isoformat()}
|
||||
if host_timestamp is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
),
|
||||
),
|
||||
tool_blocks_evidence(message.get("content"), block_types=TOOL_BLOCKS),
|
||||
)
|
||||
@@ -819,6 +846,13 @@ def _read_store_messages(
|
||||
reason_code=ScanIssueCode.UNKNOWN_RECORD,
|
||||
)
|
||||
continue
|
||||
host_timestamp = (
|
||||
_cursor_host_timestamp(content) if role == "user" else None
|
||||
)
|
||||
fallback_timestamp = datetime.fromtimestamp(
|
||||
source_file.modified_ns / 1_000_000_000,
|
||||
tz=UTC,
|
||||
) + timedelta(microseconds=rowid)
|
||||
parsed = ParsedMessage(
|
||||
conversation_id=conversation_id,
|
||||
message_id=str(record.get("id") or row["id"]),
|
||||
@@ -829,8 +863,19 @@ def _read_store_messages(
|
||||
else written_by_user
|
||||
),
|
||||
verbatim=cleaned,
|
||||
created_at=fallback_timestamp,
|
||||
workspace=_store_workspace(source_file),
|
||||
record_type="cursor_store_blob",
|
||||
origin_metadata=OriginMetadata(
|
||||
raw_meta={
|
||||
"time_basis": "file_mtime",
|
||||
**(
|
||||
{"turn_event_at": host_timestamp.isoformat()}
|
||||
if host_timestamp is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
),
|
||||
)
|
||||
yield ScannedEvidence(
|
||||
cursor=cursor,
|
||||
|
||||
@@ -57,7 +57,8 @@ _UNANSWERED_PREFIXES = (
|
||||
|
||||
|
||||
def _is_visible_user_row(role: str, written_by: str, verbatim: str) -> bool:
|
||||
if role != "user" or written_by != WrittenBy.PERSON.value:
|
||||
del written_by # Legacy storage metadata; protocol role is authoritative.
|
||||
if role != "user":
|
||||
return False
|
||||
return not verbatim.lstrip().lower().startswith(_UNANSWERED_PREFIXES)
|
||||
|
||||
@@ -468,16 +469,54 @@ class EvidenceStore:
|
||||
if anchored is not None:
|
||||
self._write_origins(connection, anchored, evidence.origin_meta, now)
|
||||
workspace, project = self._resolve_conversation_attribution(connection, evidence)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE evidence
|
||||
SET workspace = COALESCE(workspace, ?),
|
||||
project = COALESCE(project, ?),
|
||||
updated_at = ?
|
||||
WHERE evidence_id = ?
|
||||
""",
|
||||
(workspace, project, now, anchored),
|
||||
)
|
||||
existing = connection.execute(
|
||||
"SELECT role,revision FROM evidence WHERE evidence_id=?",
|
||||
(anchored,),
|
||||
).fetchone()
|
||||
if existing is None:
|
||||
raise RuntimeError("anchored evidence disappeared")
|
||||
if str(existing["role"]) != evidence.role:
|
||||
was_revised = True
|
||||
revision = int(existing["revision"]) + 1
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE evidence
|
||||
SET workspace=COALESCE(workspace,?),
|
||||
project=COALESCE(project,?),
|
||||
content_kind=?,role=?,created_at=COALESCE(?,created_at),
|
||||
verbatim=?,content_hash=?,revision=?,updated_at=?
|
||||
WHERE evidence_id=?
|
||||
""",
|
||||
(
|
||||
workspace,
|
||||
project,
|
||||
evidence.content_kind,
|
||||
evidence.role,
|
||||
evidence.created_at.isoformat() if evidence.created_at else None,
|
||||
evidence.verbatim,
|
||||
evidence.content_hash,
|
||||
revision,
|
||||
now,
|
||||
anchored,
|
||||
),
|
||||
)
|
||||
self._insert_current_version(
|
||||
connection,
|
||||
anchored,
|
||||
revision=revision,
|
||||
changed_at=now,
|
||||
)
|
||||
else:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE evidence
|
||||
SET workspace=COALESCE(workspace,?),
|
||||
project=COALESCE(project,?),
|
||||
updated_at=?
|
||||
WHERE evidence_id=?
|
||||
""",
|
||||
(workspace, project, now, anchored),
|
||||
)
|
||||
record = self._get_with_connection(connection, anchored)
|
||||
if record is None:
|
||||
raise RuntimeError("anchored evidence disappeared")
|
||||
@@ -485,7 +524,11 @@ class EvidenceStore:
|
||||
from one_memory import ingest_journal
|
||||
|
||||
ingest_journal.record(connection, anchored, cause="import")
|
||||
return PutEvidenceResult(record=record, was_created=False, was_revised=False)
|
||||
return PutEvidenceResult(
|
||||
record=record,
|
||||
was_created=False,
|
||||
was_revised=was_revised,
|
||||
)
|
||||
insert = connection.execute(
|
||||
"""
|
||||
INSERT INTO evidence (
|
||||
@@ -803,27 +846,20 @@ class EvidenceStore:
|
||||
and same_scope[0]["idempotency_key"] != evidence.idempotency_key
|
||||
):
|
||||
return str(same_scope[0]["evidence_id"])
|
||||
elif (legacy_id := claim_legacy_key(evidence.legacy_derived_idempotency_key)) is not None:
|
||||
return legacy_id
|
||||
if not evidence.message_id and not evidence.correlation_id:
|
||||
# BUG-03 moved the derived key off the capture time and onto the
|
||||
# pre-cleaning text. A replay of a row stored under the old scheme
|
||||
# has to be recognized as the same message.
|
||||
replayed = connection.execute(
|
||||
"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE idempotency_key = ? AND user_id = ? AND source_id = ?
|
||||
AND conversation_id = ?
|
||||
""",
|
||||
elif (
|
||||
legacy_id := next(
|
||||
(
|
||||
evidence.prior_derived_idempotency_key,
|
||||
evidence.user_id,
|
||||
evidence.source_id,
|
||||
evidence.conversation_id,
|
||||
anchored
|
||||
for key in (
|
||||
*evidence.written_by_derived_v3_idempotency_keys,
|
||||
*evidence.older_written_by_idempotency_keys,
|
||||
)
|
||||
if (anchored := claim_legacy_key(key)) is not None
|
||||
),
|
||||
).fetchone()
|
||||
if replayed is not None:
|
||||
return str(replayed["evidence_id"])
|
||||
None,
|
||||
)
|
||||
) is not None:
|
||||
return legacy_id
|
||||
if (
|
||||
EvidenceOrigin.HOOK in evidence.origins
|
||||
and evidence.message_id is None
|
||||
@@ -838,7 +874,7 @@ class EvidenceStore:
|
||||
ON hook.evidence_id=e.evidence_id AND hook.origin='hook'
|
||||
WHERE e.user_id=? AND e.source_id=? AND e.conversation_id=?
|
||||
AND (e.workspace IS ? OR e.workspace IS NULL OR ? IS NULL)
|
||||
AND e.role=? AND e.written_by=? AND e.content_hash=?
|
||||
AND e.role=? AND e.content_hash=?
|
||||
AND e.message_id IS NOT NULL AND hook.evidence_id IS NULL
|
||||
AND ABS((julianday(e.created_at)-julianday(?))*86400.0) <= 5.0
|
||||
ORDER BY ABS((julianday(e.created_at)-julianday(?))*86400.0), e.evidence_id
|
||||
@@ -851,7 +887,6 @@ class EvidenceStore:
|
||||
evidence.workspace,
|
||||
evidence.workspace,
|
||||
evidence.role,
|
||||
evidence.written_by,
|
||||
evidence.content_hash,
|
||||
evidence.created_at.isoformat(),
|
||||
evidence.created_at.isoformat(),
|
||||
@@ -872,7 +907,7 @@ class EvidenceStore:
|
||||
AND (e.workspace IS ? OR e.workspace IS NULL OR ? IS NULL)
|
||||
-- Unary plus keeps the coverage index from looking better than
|
||||
-- the message index, which narrows to this exact text.
|
||||
AND e.role = ? AND +e.written_by = ? AND e.content_hash = ?
|
||||
AND e.role = ? AND e.content_hash = ?
|
||||
AND e.message_id IS NULL
|
||||
""",
|
||||
(
|
||||
@@ -882,7 +917,6 @@ class EvidenceStore:
|
||||
evidence.workspace,
|
||||
evidence.workspace,
|
||||
evidence.role,
|
||||
evidence.written_by,
|
||||
evidence.content_hash,
|
||||
),
|
||||
).fetchall()
|
||||
@@ -1247,7 +1281,7 @@ class EvidenceStore:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
|
||||
WHERE user_id = 'local' AND role = 'user'
|
||||
ORDER BY imported_at, evidence_id
|
||||
LIMIT ?
|
||||
""",
|
||||
@@ -1301,7 +1335,7 @@ class EvidenceStore:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
|
||||
WHERE user_id = 'local' AND role = 'user'
|
||||
AND ({predicates})
|
||||
ORDER BY imported_at, evidence_id
|
||||
LIMIT ?
|
||||
@@ -1333,7 +1367,7 @@ class EvidenceStore:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
|
||||
WHERE user_id = 'local' AND role = 'user'
|
||||
AND workspace IS NOT NULL
|
||||
AND ({predicates})
|
||||
ORDER BY imported_at, evidence_id
|
||||
@@ -1476,7 +1510,7 @@ class EvidenceStore:
|
||||
)
|
||||
values: list[object] = [user_id, *(source_ids or ())]
|
||||
visible_user = """
|
||||
role='user' AND written_by='person' AND NOT (
|
||||
role='user' AND NOT (
|
||||
lower(ltrim(verbatim)) LIKE '<codex_internal_context%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<environment_context>%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<heartbeat>%' OR
|
||||
@@ -1485,7 +1519,7 @@ class EvidenceStore:
|
||||
)
|
||||
"""
|
||||
selected_visible_user = """
|
||||
e.role='user' AND e.written_by='person' AND NOT (
|
||||
e.role='user' AND NOT (
|
||||
lower(ltrim(e.verbatim)) LIKE '<codex_internal_context%' OR
|
||||
lower(ltrim(e.verbatim)) LIKE '<environment_context>%' OR
|
||||
lower(ltrim(e.verbatim)) LIKE '<heartbeat>%' OR
|
||||
@@ -1638,7 +1672,7 @@ class EvidenceStore:
|
||||
if seed is None:
|
||||
return ConversationTurnPage((), {}, 0, offset, limit)
|
||||
visible_user = """
|
||||
role='user' AND written_by='person' AND NOT (
|
||||
role='user' AND NOT (
|
||||
lower(ltrim(verbatim)) LIKE '<codex_internal_context%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<environment_context>%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<heartbeat>%' OR
|
||||
@@ -1731,7 +1765,7 @@ class EvidenceStore:
|
||||
AND first.conversation_id = evidence.conversation_id
|
||||
AND first.workspace IS evidence.workspace
|
||||
AND (
|
||||
(first.role = 'user' AND first.written_by = 'person')
|
||||
first.role = 'user'
|
||||
OR first.content_kind = 'memory_content'
|
||||
)
|
||||
ORDER BY julianday(COALESCE(first.created_at, first.imported_at)),
|
||||
@@ -2125,8 +2159,6 @@ class EvidenceStore:
|
||||
(
|
||||
existing["content_kind"] != evidence.content_kind,
|
||||
existing["role"] != evidence.role,
|
||||
evidence.written_by is not WrittenBy.UNKNOWN
|
||||
and existing["written_by"] != evidence.written_by,
|
||||
existing["content_hash"] != evidence.content_hash,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -111,7 +111,7 @@ class ImportSupervisor:
|
||||
)
|
||||
)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.supervisor",
|
||||
event="import.queued",
|
||||
outcome="queued",
|
||||
@@ -198,7 +198,7 @@ class ImportSupervisor:
|
||||
started = monotonic()
|
||||
failure_type: str | None = None
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="import.supervisor",
|
||||
event="import.started",
|
||||
outcome="started",
|
||||
|
||||
@@ -85,15 +85,14 @@ def _state(connection: sqlite3.Connection, evidence_id: str) -> dict[str, str |
|
||||
value = evidence.model_dump(mode="json")
|
||||
# The stored normalized content_hash is insufficient for exact-content leases.
|
||||
content = _digest(value["verbatim"])
|
||||
material = _digest(
|
||||
{key: value[key] for key in ("verbatim", "role", "written_by", "content_kind")}
|
||||
)
|
||||
material = _digest({key: value[key] for key in ("verbatim", "role", "content_kind")})
|
||||
origins = _digest(value["origin_meta"])
|
||||
identity = _digest(
|
||||
{
|
||||
key: item
|
||||
for key, item in value.items()
|
||||
if key not in {"verbatim", "origin_meta", "imported_at", "updated_at"}
|
||||
if key
|
||||
not in {"verbatim", "written_by", "origin_meta", "imported_at", "updated_at"}
|
||||
}
|
||||
)
|
||||
return dict(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import TextIO, cast
|
||||
|
||||
@@ -104,7 +105,13 @@ def capture_hook(source_id: str, payload: object, database: Path) -> bool:
|
||||
# Cursor also runs ~/.claude/settings.json hooks; its own hook already files the turn.
|
||||
if source_id != "cursor" and "cursor_version" in payload:
|
||||
return False
|
||||
if not database.is_file() or not _sync_open(database, source_id):
|
||||
if not database.is_file():
|
||||
return False
|
||||
rejection = capture_rejection(database, source_id)
|
||||
if rejection is not None:
|
||||
# Silent, and the reason is counted rather than printed: a hook that
|
||||
# spoke would put this into the host's turn as text.
|
||||
record_capture_rejection(database, source_id, rejection)
|
||||
return False
|
||||
if source_id == "pi" and _is_automated_session(first_string(payload.get("cwd"))):
|
||||
return False
|
||||
@@ -139,17 +146,79 @@ def capture_hook(source_id: str, payload: object, database: Path) -> bool:
|
||||
|
||||
|
||||
def _sync_open(database: Path, source_id: str) -> bool:
|
||||
return capture_rejection(database, source_id) is None
|
||||
|
||||
|
||||
class CaptureRejection(StrEnum):
|
||||
"""Why a hook's turn was dropped, or nothing at all.
|
||||
|
||||
Capture fails open on purpose: a hook that raises or prints would be read
|
||||
back into the host's turn, so a refusal has to be silent and exit 0. It was
|
||||
also completely invisible, which is why "持续同步开着但一条都没进" could not be
|
||||
told apart from "capture 坏了". These values name the reason without any
|
||||
payload, path, session or message — the only part that may be recorded.
|
||||
"""
|
||||
|
||||
SYNC_DISABLED = "sync_disabled"
|
||||
GRANT_MISSING = "grant_missing"
|
||||
GRANT_REVOKED = "grant_revoked"
|
||||
|
||||
|
||||
def capture_rejection(database: Path, source_id: str) -> CaptureRejection | None:
|
||||
"""None when this turn may be captured, otherwise why it may not."""
|
||||
from one_memory.memory_server.config_state import ConfigStore
|
||||
|
||||
if not ConfigStore(database.parent).sync_enabled(source_id):
|
||||
return False
|
||||
return CaptureRejection.SYNC_DISABLED
|
||||
with main_database(database).read() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT 1 FROM source_grants WHERE user_id='local' AND source_id=? "
|
||||
"AND revoked_at IS NULL",
|
||||
"SELECT revoked_at FROM source_grants WHERE user_id='local' AND source_id=?",
|
||||
(source_id,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
if row is None:
|
||||
return CaptureRejection.GRANT_MISSING
|
||||
# Indexed, not named: this connection has no row factory, and a hook is
|
||||
# not the place to add one.
|
||||
if row[0] is not None:
|
||||
return CaptureRejection.GRANT_REVOKED
|
||||
return None
|
||||
|
||||
|
||||
def record_capture_rejection(
|
||||
database: Path,
|
||||
source_id: str,
|
||||
reason: CaptureRejection,
|
||||
*,
|
||||
window_seconds: float = 600.0,
|
||||
) -> None:
|
||||
"""Note one dropped turn, at most once per window per source and reason.
|
||||
|
||||
Best effort by construction: a hook runs inside somebody's turn, so a sink
|
||||
that is slow, full or read-only must never surface here. The record is a
|
||||
count and nothing else — no payload, no path, no session or conversation
|
||||
id — because this file is meant to be handed to support as is.
|
||||
|
||||
One small file per source and reason, rather than a line per turn: capture
|
||||
runs on every prompt, and a log that grew with the conversation would be a
|
||||
privacy problem wearing a diagnosis as a disguise.
|
||||
"""
|
||||
try:
|
||||
directory = database.parent / "logs"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
marker = directory / f"capture-rejection-{source_id}-{reason.value}.json"
|
||||
now = time.time()
|
||||
payload: dict[str, object] = {"source_id": source_id, "reason": reason.value}
|
||||
if marker.exists() and now - marker.stat().st_mtime < window_seconds:
|
||||
payload = json.loads(marker.read_text(encoding="utf8"))
|
||||
previous = payload.get("count", 1)
|
||||
payload["count"] = int(previous) + 1 if isinstance(previous, (str, int, float)) else 2
|
||||
else:
|
||||
payload["count"] = 1
|
||||
payload["window_started_at"] = datetime.fromtimestamp(now, UTC).isoformat()
|
||||
payload["last_at"] = payload["window_started_at"]
|
||||
marker.write_text(json.dumps(payload, sort_keys=True), encoding="utf8")
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed: bool) -> None:
|
||||
@@ -167,7 +236,7 @@ def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed
|
||||
):
|
||||
if (
|
||||
record in chunk
|
||||
and any(item.role == "user" and item.written_by is WrittenBy.PERSON for item in chunk)
|
||||
and any(item.role == "user" for item in chunk)
|
||||
and any(item.role == "assistant" for item in chunk)
|
||||
):
|
||||
intake.submit_records(chunk, priority="live")
|
||||
|
||||
@@ -160,7 +160,9 @@ class MatterDiscovery:
|
||||
task_by_id: dict[str, MemoryTask] = {}
|
||||
for chunk in chunks:
|
||||
for record in chunk:
|
||||
task = await asyncio.to_thread(processor.tasks.task_for_evidence, record.evidence_id)
|
||||
task = await asyncio.to_thread(
|
||||
processor.tasks.task_for_evidence, record.evidence_id
|
||||
)
|
||||
if task is not None and task.status is MemoryTaskStatus.COMPLETED:
|
||||
task_by_id[task.task_id] = task
|
||||
tasks = tuple(sorted(task_by_id.values(), key=lambda task: (task.created_at, task.task_id)))
|
||||
|
||||
@@ -270,7 +270,7 @@ async def reconcile_current(
|
||||
(
|
||||
r
|
||||
for r in records
|
||||
if r.written_by == "person"
|
||||
if r.role == "user"
|
||||
and r.created_at
|
||||
and (r.source_id, r.conversation_id) in related
|
||||
),
|
||||
|
||||
@@ -51,14 +51,19 @@ def validate_claim(claim: Claim, records: Mapping[str, EvidenceRecord | SampleRe
|
||||
text = record.text if isinstance(record, SampleRecord) else record.verbatim
|
||||
if not citation.quote.strip() or citation.quote not in text:
|
||||
raise ValueError("citation quote is not in the source")
|
||||
authors.add(str(record.written_by))
|
||||
author = {
|
||||
"user": "person",
|
||||
"assistant": "agent",
|
||||
"tool": "tool",
|
||||
}.get(record.role, "unknown")
|
||||
authors.add(author)
|
||||
created = record.created_at
|
||||
citations.append(
|
||||
citation.model_copy(
|
||||
update={
|
||||
"revision": record.revision,
|
||||
"source_id": record.source_id,
|
||||
"written_by": str(record.written_by),
|
||||
"written_by": author,
|
||||
"created_at": created.isoformat() if isinstance(created, datetime) else created,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -290,7 +290,7 @@ def read_memory_library(
|
||||
COALESCE(
|
||||
MIN(
|
||||
CASE WHEN e.role = 'user'
|
||||
AND e.written_by IN ('person', 'user')
|
||||
AND e.role = 'user'
|
||||
THEN e.created_at END
|
||||
),
|
||||
MIN(e.created_at)
|
||||
|
||||
@@ -45,6 +45,7 @@ from one_memory.hy_memory_bridge import (
|
||||
HyMemoryOutput,
|
||||
HyMemoryTurn,
|
||||
builtin_retrieval_snapshot,
|
||||
coding_tool_result_max_bytes,
|
||||
decide_profile_graph_width,
|
||||
hy_memory_store_exists,
|
||||
inspect_profile_graph_width,
|
||||
@@ -52,7 +53,7 @@ from one_memory.hy_memory_bridge import (
|
||||
from one_memory.hy_runtime_config import hy_memory_version
|
||||
from one_memory.hy_source_context import HySourceContext, HySupportedMemory
|
||||
from one_memory.imports.access import opaque_workspace_id
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.imports.runner import live_sync_arrival
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
@@ -162,7 +163,7 @@ def _report_stale_known_reasons(task_id: str, reasons: Mapping[str, int]) -> Non
|
||||
"""Emit one schema-valid scalar diagnostic per stale-memory reason."""
|
||||
for reason, count in sorted(reasons.items()):
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.context",
|
||||
event="known_memory_stale_omitted",
|
||||
outcome="omitted",
|
||||
@@ -173,6 +174,21 @@ def _report_stale_known_reasons(task_id: str, reasons: Mapping[str, int]) -> Non
|
||||
|
||||
MEMORY_INGEST_POLICY_VERSION = "hy-native-v23"
|
||||
UNIFIED_HY_AGENT_ID = "one-memory"
|
||||
#: Deterministic refusals raised before the payload reaches HY, so nothing was
|
||||
#: sent even though the invocation marker is already recorded by then: the
|
||||
#: write bindings are built inside `ingest` ahead of the call, and the input
|
||||
#: window is checked while the request is frozen.
|
||||
#:
|
||||
#: They are terminal rather than unknown. `unknown` here means "HY may hold a
|
||||
#: write we cannot confirm", and that is exactly what did not happen — parking
|
||||
#: the task on it hid the reason and stopped the queue behind it (issue #44).
|
||||
#: Kept in step with `durable_queue.PRE_SEND_REFUSAL_CODES`, which the durable
|
||||
#: write path applies to the same codes.
|
||||
PRE_SEND_REFUSAL_CODES = frozenset(
|
||||
{
|
||||
"hy_memory_input_exceeds_window",
|
||||
}
|
||||
)
|
||||
_BACKFILL_BATCH_SIZE = 8
|
||||
_BACKFILL_DRAIN_BUDGET_SECONDS = 330.0
|
||||
_HISTORY_MERGE_TURNS = 8
|
||||
@@ -280,6 +296,7 @@ class MemoryTaskRunner:
|
||||
*,
|
||||
timeout: float | None = None,
|
||||
before_hy_invocation: Callable[[], Awaitable[None]] | None = None,
|
||||
input_records: tuple[EvidenceRecord, ...] | None = None,
|
||||
) -> MemoryTask:
|
||||
started = monotonic()
|
||||
diagnostic_event(
|
||||
@@ -290,13 +307,28 @@ class MemoryTaskRunner:
|
||||
task_id=task_id,
|
||||
)
|
||||
try:
|
||||
body = (
|
||||
self._run_without_content_permission(task_id)
|
||||
if before_hy_invocation is None
|
||||
else self._run_without_content_permission(
|
||||
task_id, before_hy_invocation=before_hy_invocation
|
||||
if input_records is None:
|
||||
body = (
|
||||
self._run_without_content_permission(task_id)
|
||||
if before_hy_invocation is None
|
||||
else self._run_without_content_permission(
|
||||
task_id,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
)
|
||||
)
|
||||
else:
|
||||
body = (
|
||||
self._run_without_content_permission(
|
||||
task_id,
|
||||
input_records=input_records,
|
||||
)
|
||||
if before_hy_invocation is None
|
||||
else self._run_without_content_permission(
|
||||
task_id,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
input_records=input_records,
|
||||
)
|
||||
)
|
||||
)
|
||||
# The ceiling bounds how long this task holds its harness's slot:
|
||||
# a harness is serial, so one hung task starves every turn behind
|
||||
# it. It is insurance against a single bad call, not a way to run
|
||||
@@ -367,7 +399,7 @@ class MemoryTaskRunner:
|
||||
@staticmethod
|
||||
def _log_step(task_id: str, step: int, action: str, started: float) -> None:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.task",
|
||||
event="memory_task.step_completed",
|
||||
outcome="succeeded",
|
||||
@@ -381,6 +413,7 @@ class MemoryTaskRunner:
|
||||
task_id: str,
|
||||
*,
|
||||
before_hy_invocation: Callable[[], Awaitable[None]] | None = None,
|
||||
input_records: tuple[EvidenceRecord, ...] | None = None,
|
||||
) -> MemoryTask:
|
||||
# Every store call here is synchronous, and several are writes that
|
||||
# wait on the write lock for up to five seconds. Made from the loop,
|
||||
@@ -434,7 +467,9 @@ class MemoryTaskRunner:
|
||||
step_started = monotonic()
|
||||
frozen = await in_the_background(self.tasks.inputs, task.task_id)
|
||||
chunk: tuple[EvidenceRecord, ...] | None
|
||||
if frozen:
|
||||
if input_records is not None:
|
||||
chunk = input_records
|
||||
elif frozen:
|
||||
try:
|
||||
chunk = await in_the_background(self._frozen_chunk, task)
|
||||
except HyMemoryBridgeError as error:
|
||||
@@ -467,8 +502,11 @@ class MemoryTaskRunner:
|
||||
self._source_context, task, request.source_context
|
||||
)
|
||||
request = request.model_copy(update={"source_context": context})
|
||||
uses_frozen_input = bool(frozen) or input_records is not None
|
||||
grant_token = (
|
||||
await in_the_background(self._grant_token, task, chunk[0]) if frozen else None
|
||||
await in_the_background(self._grant_token, task, chunk[0])
|
||||
if uses_frozen_input
|
||||
else None
|
||||
)
|
||||
memory_contents = tuple(
|
||||
sanitize_model_input(record.verbatim)
|
||||
@@ -511,19 +549,24 @@ class MemoryTaskRunner:
|
||||
agent_id=request.agent_id,
|
||||
),
|
||||
)
|
||||
if frozen:
|
||||
await in_the_background(self._frozen_chunk, task)
|
||||
if uses_frozen_input:
|
||||
if input_records is None:
|
||||
await in_the_background(self._frozen_chunk, task)
|
||||
if grant_token != await in_the_background(self._grant_token, task, chunk[0]):
|
||||
raise HyMemoryBridgeError("hy_memory_source_changed")
|
||||
if request.source_context is not None:
|
||||
await in_the_background(self._check_source_context, request.source_context)
|
||||
await in_the_background(
|
||||
self._check_source_context,
|
||||
request.source_context,
|
||||
skip_current=input_records is not None,
|
||||
)
|
||||
# One thread hop for the whole batch rather than a dozen writes
|
||||
# made from the loop.
|
||||
step_started = monotonic()
|
||||
await in_the_background(
|
||||
self._record_outputs,
|
||||
task=task,
|
||||
chunk=self._original_records(chunk),
|
||||
chunk=chunk,
|
||||
outputs=outputs,
|
||||
request=request,
|
||||
derivation=derivation,
|
||||
@@ -533,6 +576,13 @@ class MemoryTaskRunner:
|
||||
self._log_step(task_id, 6, "governance", step_started)
|
||||
except HyMemoryBridgeError as error:
|
||||
code = str(error)
|
||||
if code == "model_sedimentation_disabled" and not hy_invoked:
|
||||
return await in_the_background(
|
||||
self.tasks.release_unstarted,
|
||||
task_id,
|
||||
runner_token=runner_token,
|
||||
error_code=code,
|
||||
)
|
||||
model_wait_errors = {
|
||||
"hy_memory_unavailable",
|
||||
"hy_memory_rate_limit",
|
||||
@@ -547,6 +597,16 @@ class MemoryTaskRunner:
|
||||
"hy_memory_model_timeout",
|
||||
}
|
||||
safe_terminal_after_invocation = {"hy_memory_import_no_readable_output"}
|
||||
if code in PRE_SEND_REFUSAL_CODES:
|
||||
# Raised before the call, so the marker only says "about to":
|
||||
# fail with the reason rather than parking on a write HY never
|
||||
# saw. See the constant and issue #44.
|
||||
return await in_the_background(
|
||||
self.tasks.fail,
|
||||
task_id,
|
||||
runner_token=runner_token,
|
||||
error_code=code,
|
||||
)
|
||||
if hy_invoked and (
|
||||
code == "hy_memory_evidence_changed"
|
||||
or (
|
||||
@@ -744,8 +804,18 @@ class MemoryTaskRunner:
|
||||
raise HyMemoryBridgeError("hy_memory_evidence_changed") from error
|
||||
return context
|
||||
|
||||
def _check_source_context(self, context: HySourceContext) -> None:
|
||||
for original in (*context.current, *context.history, *context.supporting):
|
||||
def _check_source_context(
|
||||
self,
|
||||
context: HySourceContext,
|
||||
*,
|
||||
skip_current: bool = False,
|
||||
) -> None:
|
||||
checked = (
|
||||
(*context.history, *context.supporting)
|
||||
if skip_current
|
||||
else (*context.current, *context.history, *context.supporting)
|
||||
)
|
||||
for original in checked:
|
||||
record = self.evidence.get(original.evidence_id)
|
||||
if record is None or not matches_evidence_ref(record, evidence_ref(original)):
|
||||
raise HyMemoryBridgeError("hy_memory_evidence_changed")
|
||||
@@ -846,7 +916,7 @@ class MemoryTaskRunner:
|
||||
# HY's own attribution. An undecided memory counts as the person's:
|
||||
# HY's prompt defaults that way.
|
||||
from_agent = output.owner == "agent" or all(
|
||||
record.written_by is WrittenBy.AGENT
|
||||
record.role == "assistant"
|
||||
and record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
for record in support
|
||||
)
|
||||
@@ -1016,15 +1086,20 @@ class MemoryTaskRunner:
|
||||
task: MemoryTask,
|
||||
chunk: tuple[EvidenceRecord, ...],
|
||||
) -> HyMemoryIngestRequest:
|
||||
if not chunk:
|
||||
# A task with no inputs has nothing to name the request after and
|
||||
# nothing to attribute an answer to. Left alone this reached
|
||||
# `chunk[0]` below and came out as IndexError, which no caller can
|
||||
# read: a reconcile pass catches domain errors per task and moves
|
||||
# on, so one such task ended the whole pass and left every other
|
||||
# unknown task unreconciled. On this machine 68 of 303 unknown
|
||||
# tasks had no inputs, and the pass raised on the first of them.
|
||||
raise HyMemoryBridgeError("hy_memory_task_without_inputs")
|
||||
has_raw = any(
|
||||
record.content_kind is EvidenceContentKind.RAW_CONVERSATION for record in chunk
|
||||
)
|
||||
source = next(
|
||||
(
|
||||
record
|
||||
for record in chunk
|
||||
if record.role == "user" and record.written_by is WrittenBy.PERSON
|
||||
),
|
||||
(record for record in chunk if record.role == "user"),
|
||||
chunk[0],
|
||||
)
|
||||
request_turns = tuple(
|
||||
@@ -1033,12 +1108,9 @@ class MemoryTaskRunner:
|
||||
if (
|
||||
record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and (
|
||||
# D98: any `user` message goes, whoever the parser says
|
||||
# wrote it. A sub agent's question is ordinary
|
||||
# user-assistant data; the author field cannot tell who
|
||||
# wrote a message, and filtering on it dropped the sub
|
||||
# agent's turns. `written_by` still labels the display and
|
||||
# decides owner — it no longer decides what is sent.
|
||||
# Protocol role is authoritative. A sub-agent question is
|
||||
# an ordinary user message regardless of a host parser's
|
||||
# former author guess.
|
||||
record.role == "user"
|
||||
# The assistant's side goes too. Without it 「就按你说的那个来」
|
||||
# arrives with nothing to resolve it against, and HY's own
|
||||
@@ -1049,7 +1121,7 @@ class MemoryTaskRunner:
|
||||
or record.role == "assistant"
|
||||
# Tool calls go as they happened; whether they make the
|
||||
# turn a coding turn is HY's call, not ours.
|
||||
or (record.role == "tool" and record.written_by is WrittenBy.TOOL)
|
||||
or record.role == "tool"
|
||||
)
|
||||
)
|
||||
or (not has_raw and record.content_kind is EvidenceContentKind.MEMORY_CONTENT)
|
||||
@@ -1218,6 +1290,7 @@ class MemoryProcessingSupervisor:
|
||||
max_concurrent_tasks: int | None = None,
|
||||
source_writes_permitted: Callable[[str], bool] | None = None,
|
||||
service_levels: ServiceLevelScheduler | None = None,
|
||||
sedimentation_enabled: Callable[[], bool] | None = None,
|
||||
) -> None:
|
||||
self.tasks = tasks
|
||||
self.profiles = profiles
|
||||
@@ -1230,6 +1303,12 @@ class MemoryProcessingSupervisor:
|
||||
#: absent every source is treated as writable, which is what the
|
||||
#: product did before the switch existed.
|
||||
self.source_writes_permitted = source_writes_permitted
|
||||
#: User-level "stop turning conversations into memories" switch. Optional
|
||||
#: so tests and a Core without a config file keep their current
|
||||
#: behaviour; when absent sedimentation is on, which is what the product
|
||||
#: did before the switch existed. Read per decision rather than cached,
|
||||
#: so stopping takes effect on the next dispatch without a restart.
|
||||
self.sedimentation_enabled = sedimentation_enabled
|
||||
self.service_levels = service_levels or ServiceLevelScheduler()
|
||||
self.extraction_settings = extraction_settings or ExtractionSettingsStore(
|
||||
data_dir / "one-memory.sqlite3"
|
||||
@@ -1283,12 +1362,26 @@ class MemoryProcessingSupervisor:
|
||||
self._orphan_reclaimer: asyncio.Task[None] | None = None
|
||||
self._diagnostic_idle_reason: str | None = None
|
||||
|
||||
def _sedimentation_stopped(self) -> bool:
|
||||
"""Whether the user stopped model-driven sedimentation.
|
||||
|
||||
Stopping must not look like a broken pipeline: nothing is sent to the
|
||||
model, so the pending count stays as it is and the scheduler reports
|
||||
why it is idle. Starting again picks the same work up.
|
||||
"""
|
||||
if self.sedimentation_enabled is None:
|
||||
return False
|
||||
if self.sedimentation_enabled():
|
||||
return False
|
||||
self._record_idle("sedimentation_stopped")
|
||||
return True
|
||||
|
||||
def _record_idle(self, reason: str) -> None:
|
||||
if self._diagnostic_idle_reason == reason:
|
||||
return
|
||||
self._diagnostic_idle_reason = reason
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.idle",
|
||||
outcome="idle",
|
||||
@@ -1310,7 +1403,8 @@ class MemoryProcessingSupervisor:
|
||||
) -> tuple[MemoryTask, ...]:
|
||||
"""Process a bounded, authorized snapshot through the existing HY runner."""
|
||||
async with self._explicit_processing_lock(records):
|
||||
records = await in_the_background(self._original_records, records)
|
||||
if self._sedimentation_stopped():
|
||||
raise HyMemoryBridgeError("model_sedimentation_disabled")
|
||||
active = await in_the_background(self.profiles.active, self.secrets)
|
||||
if active is None or (profile_id is not None and profile_id != active.profile_id):
|
||||
raise HyMemoryBridgeError("model_profile_not_active")
|
||||
@@ -1325,13 +1419,6 @@ class MemoryProcessingSupervisor:
|
||||
raise HyMemoryBridgeError("hy_memory_source_disabled")
|
||||
conversations: dict[tuple[str, str, str | None], list[EvidenceRecord]] = {}
|
||||
for record in records:
|
||||
stored = await in_the_background(self.evidence.get, record.evidence_id)
|
||||
if (
|
||||
stored is None
|
||||
or stored.revision != record.revision
|
||||
or stored.content_hash != record.content_hash
|
||||
):
|
||||
raise ValueError("hy_memory_evidence_changed")
|
||||
conversations.setdefault(
|
||||
(record.source_id, record.conversation_id, record.workspace), []
|
||||
).append(record)
|
||||
@@ -1348,7 +1435,7 @@ class MemoryProcessingSupervisor:
|
||||
for conversation in conversations.values()
|
||||
for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True)
|
||||
]
|
||||
prepared = []
|
||||
prepared: list[tuple[MemoryTask, tuple[EvidenceRecord, ...]]] = []
|
||||
for chunk in chunks:
|
||||
task = await in_the_background(
|
||||
self._enqueue_chunk,
|
||||
@@ -1356,12 +1443,12 @@ class MemoryProcessingSupervisor:
|
||||
active,
|
||||
freeze_inputs=True,
|
||||
)
|
||||
prepared.append(task)
|
||||
prepared.append((task, chunk))
|
||||
runner = self.runner_factory()
|
||||
completed = []
|
||||
if on_progress:
|
||||
on_progress(tuple(prepared), 0)
|
||||
for task in prepared:
|
||||
on_progress(tuple(task for task, _chunk in prepared), 0)
|
||||
for task, chunk in prepared:
|
||||
if task.status is not MemoryTaskStatus.COMPLETED:
|
||||
# Another importer may already own this task. Never race its claim.
|
||||
task = await in_the_background(self.tasks.require, task.task_id)
|
||||
@@ -1390,23 +1477,41 @@ class MemoryProcessingSupervisor:
|
||||
work_id=task.task_id,
|
||||
scope_key=chunk[0].source_id,
|
||||
):
|
||||
task = (
|
||||
await runner.run(task.task_id, timeout=timeout)
|
||||
if before_hy_invocation is None
|
||||
else await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
if isinstance(runner, MemoryTaskRunner):
|
||||
task = (
|
||||
await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
input_records=chunk,
|
||||
)
|
||||
if before_hy_invocation is None
|
||||
else await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
input_records=chunk,
|
||||
)
|
||||
)
|
||||
else:
|
||||
task = (
|
||||
await runner.run(task.task_id, timeout=timeout)
|
||||
if before_hy_invocation is None
|
||||
else await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
)
|
||||
)
|
||||
)
|
||||
finally:
|
||||
_unmark_direct_task(self.tasks.database_path, task.task_id)
|
||||
if task.status is not MemoryTaskStatus.COMPLETED:
|
||||
raise HyMemoryBridgeError(task.last_error_code or "hy_memory_incomplete")
|
||||
await in_the_background(runner._frozen_chunk, task)
|
||||
completed.append(task)
|
||||
if on_progress:
|
||||
on_progress(tuple(prepared), len(completed))
|
||||
on_progress(
|
||||
tuple(prepared_task for prepared_task, _chunk in prepared),
|
||||
len(completed),
|
||||
)
|
||||
expected = {record.evidence_id for record in records if _is_hy_turn_record(record)}
|
||||
actual = {
|
||||
row["evidence_id"] for task in completed for row in self.tasks.inputs(task.task_id)
|
||||
@@ -1667,6 +1772,8 @@ class MemoryProcessingSupervisor:
|
||||
def enqueue_evidence(self, record: EvidenceRecord) -> MemoryTask | None:
|
||||
if self._shutting_down:
|
||||
return None
|
||||
if self._sedimentation_stopped():
|
||||
return None
|
||||
if (
|
||||
record.role not in {"user", "assistant"}
|
||||
and record.content_kind is not EvidenceContentKind.MEMORY_CONTENT
|
||||
@@ -1699,6 +1806,8 @@ class MemoryProcessingSupervisor:
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
) -> list[QueuedWork]:
|
||||
"""Synchronous compatibility entry; preparation never starts a model."""
|
||||
if self._sedimentation_stopped():
|
||||
return []
|
||||
work = [item for batch in self._prepare_backfill_batches(should_stop) for item in batch]
|
||||
return [] if should_stop() or self._shutting_down else work
|
||||
|
||||
@@ -1707,7 +1816,7 @@ class MemoryProcessingSupervisor:
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
) -> Generator[list[QueuedWork], None, None]:
|
||||
"""Read bodies as needed; yield the first task before the next conversation."""
|
||||
if should_stop() or self._shutting_down:
|
||||
if should_stop() or self._shutting_down or self._sedimentation_stopped():
|
||||
return
|
||||
active = self.profiles.active(self.secrets)
|
||||
if active is None:
|
||||
@@ -1718,7 +1827,7 @@ class MemoryProcessingSupervisor:
|
||||
visited = 0
|
||||
seed_ids = self.evidence.list_conversation_seed_ids()
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_inventory",
|
||||
outcome="observed",
|
||||
@@ -1788,7 +1897,7 @@ class MemoryProcessingSupervisor:
|
||||
started = monotonic()
|
||||
prepared_count = 0
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_started",
|
||||
outcome="started",
|
||||
@@ -1800,6 +1909,7 @@ class MemoryProcessingSupervisor:
|
||||
stop = threading.Event()
|
||||
batches = self._prepare_backfill_batches(stop.is_set)
|
||||
exhausted = False
|
||||
stopped_saturated = False
|
||||
rounds = 0
|
||||
scan_error: BaseException | None = None
|
||||
try:
|
||||
@@ -1815,20 +1925,21 @@ class MemoryProcessingSupervisor:
|
||||
>= self._backfill_round_budget_seconds
|
||||
):
|
||||
break
|
||||
while (
|
||||
if (
|
||||
cohort_size is not None
|
||||
and len(self._queue) + len(self._in_flight) >= cohort_size
|
||||
):
|
||||
if self._shutting_down:
|
||||
return
|
||||
if (
|
||||
handled_batch
|
||||
and self._backfill_clock() - round_started
|
||||
>= self._backfill_round_budget_seconds
|
||||
):
|
||||
break
|
||||
self._record_idle("queue_saturated")
|
||||
await asyncio.sleep(0.1)
|
||||
# A bounded scan stops at its ceiling instead of
|
||||
# waiting for the queue to drain. Waiting here slept
|
||||
# inside the scan lock, so whoever asked for the
|
||||
# scan — a switch toggle, over HTTP — waited with
|
||||
# it, and on a saturated backlog that is not a
|
||||
# moment but the whole backlog. The caller's
|
||||
# timeout then reported a failure for work that had
|
||||
# already been accepted.
|
||||
stopped_saturated = True
|
||||
exhausted = True
|
||||
break
|
||||
if (
|
||||
handled_batch
|
||||
and self._backfill_clock() - round_started
|
||||
@@ -1847,7 +1958,7 @@ class MemoryProcessingSupervisor:
|
||||
on_first_scheduled()
|
||||
if not exhausted and not self._shutting_down:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_yielded",
|
||||
outcome="yielded",
|
||||
@@ -1877,14 +1988,18 @@ class MemoryProcessingSupervisor:
|
||||
stop.set()
|
||||
batches.close()
|
||||
if prepared_count == 0 and not self._shutting_down:
|
||||
if scan_error is None:
|
||||
if stopped_saturated:
|
||||
# Parked behind a full queue, not out of work: the status
|
||||
# must not read "no_tasks" while hundreds wait.
|
||||
self._record_idle("queue_saturated")
|
||||
elif scan_error is None:
|
||||
self._record_idle("no_tasks")
|
||||
else:
|
||||
self._record_idle("scan_failed")
|
||||
elif prepared_count:
|
||||
self._diagnostic_idle_reason = None
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_completed",
|
||||
outcome=(
|
||||
@@ -2092,6 +2207,8 @@ class MemoryProcessingSupervisor:
|
||||
Work that falls outside the extraction window is marked deferred so
|
||||
the pending count stops pretending it is about to run.
|
||||
"""
|
||||
if self._sedimentation_stopped():
|
||||
return []
|
||||
now = datetime.now(UTC)
|
||||
window = self._extraction_window() if within_window else None
|
||||
exempt_all_apps_from_window = True
|
||||
@@ -2174,7 +2291,7 @@ class MemoryProcessingSupervisor:
|
||||
if work:
|
||||
self._diagnostic_idle_reason = None
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.tasks_queued",
|
||||
outcome="succeeded",
|
||||
@@ -2239,7 +2356,7 @@ class MemoryProcessingSupervisor:
|
||||
return self.HARNESS_SERIAL
|
||||
|
||||
def _ensure_workers(self) -> None:
|
||||
if self._shutting_down:
|
||||
if self._shutting_down or self._sedimentation_stopped():
|
||||
return
|
||||
load = self._harness_load()
|
||||
background_capacity = (
|
||||
@@ -2317,6 +2434,8 @@ class MemoryProcessingSupervisor:
|
||||
"""
|
||||
looked_for_older = False
|
||||
while not self._shutting_down:
|
||||
if self._sedimentation_stopped():
|
||||
return
|
||||
# The harness cap is recomputed from what is running, so a worker
|
||||
# that just finished frees its harness's slot immediately.
|
||||
work = self._queue.take(
|
||||
@@ -2370,7 +2489,16 @@ class MemoryProcessingSupervisor:
|
||||
scope_key=scope_key,
|
||||
),
|
||||
):
|
||||
finished = await runner.run(work.task_id, timeout=self._profile_timeout)
|
||||
|
||||
async def require_sedimentation() -> None:
|
||||
if self._sedimentation_stopped():
|
||||
raise HyMemoryBridgeError("model_sedimentation_disabled")
|
||||
|
||||
finished = await runner.run(
|
||||
work.task_id,
|
||||
timeout=self._profile_timeout,
|
||||
before_hy_invocation=require_sedimentation,
|
||||
)
|
||||
if finished.status is MemoryTaskStatus.COMPLETED:
|
||||
self._request_read_bridge_refresh(finished.profile_snapshot)
|
||||
elif (
|
||||
@@ -3049,7 +3177,7 @@ class MemoryProcessingSupervisor:
|
||||
self._read_bridges_ready.add(key)
|
||||
self._read_bridge_generations[key] = generation
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="memory.semantic",
|
||||
event="semantic_read_bridge.refreshed",
|
||||
outcome="succeeded",
|
||||
@@ -3172,9 +3300,7 @@ class MemoryProcessingSupervisor:
|
||||
if not tail:
|
||||
return False
|
||||
last = tail[-1]
|
||||
return bool(
|
||||
last.role == "user" and last.written_by is WrittenBy.PERSON and _opens_turn(last)
|
||||
)
|
||||
return bool(last.role == "user" and _opens_turn(last))
|
||||
|
||||
def _admitted_chunks(
|
||||
self,
|
||||
@@ -3519,7 +3645,6 @@ def _is_memory_note(record: EvidenceRecord) -> bool:
|
||||
meta = record.origin_meta.get(EvidenceOrigin.MANUAL_IMPORT)
|
||||
return (
|
||||
record.source_id == "one-memory-manual"
|
||||
and record.written_by is WrittenBy.PERSON
|
||||
and meta is not None
|
||||
and meta.raw_meta.get("input_mode") == "direct_memory_note"
|
||||
)
|
||||
@@ -3595,7 +3720,7 @@ _HOST_ENVELOPE = re.compile(
|
||||
#: before HY chose a path cut long answers mid-way and sent tool-heavy turns as
|
||||
#: dozens of calls. ONE_MEMORY_SPLIT_TURNS=1 restores the 12,000 / 24 split.
|
||||
_WHOLE_TURN_CHARACTERS = 60_000
|
||||
_HY_TOOL_CHARACTERS = 2_048
|
||||
_HY_TOOL_CHARACTERS = coding_tool_result_max_bytes()
|
||||
#: Hard ceiling on the UTF-8 bytes one window may hold, whatever HY counts.
|
||||
#: HY reads the payload in-process, so a turn made of enormous raw tool
|
||||
#: records is held in memory even though HY will cut each one down. Cut by
|
||||
@@ -3814,14 +3939,12 @@ def _merge_historical_chunks(
|
||||
|
||||
|
||||
def _chunk_input_ref(chunk: tuple[EvidenceRecord, ...]) -> str:
|
||||
person_records = tuple(
|
||||
item for item in chunk if item.role == "user" and item.written_by is WrittenBy.PERSON
|
||||
)
|
||||
return person_records[-1].evidence_id if person_records else chunk[-1].evidence_id
|
||||
user_records = tuple(item for item in chunk if item.role == "user")
|
||||
return user_records[-1].evidence_id if user_records else chunk[-1].evidence_id
|
||||
|
||||
|
||||
def _opens_turn(record: EvidenceRecord) -> bool:
|
||||
return record.role == "user" and record.written_by is WrittenBy.PERSON
|
||||
return record.role == "user"
|
||||
|
||||
|
||||
def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
|
||||
@@ -3838,10 +3961,8 @@ def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
|
||||
def _is_hy_turn_record(record: EvidenceRecord) -> bool:
|
||||
"""What HY receives of a turn: the input turns and the tool calls made in it."""
|
||||
if record.role == "tool":
|
||||
return (
|
||||
record.written_by is WrittenBy.TOOL
|
||||
and record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and bool(record.verbatim.strip())
|
||||
return record.content_kind is EvidenceContentKind.RAW_CONVERSATION and bool(
|
||||
record.verbatim.strip()
|
||||
)
|
||||
return _is_hy_input_turn(record)
|
||||
|
||||
@@ -3858,11 +3979,8 @@ def _is_hy_input_turn(record: EvidenceRecord) -> bool:
|
||||
# from a model.
|
||||
if record.role == "assistant":
|
||||
return True
|
||||
# D98: the author no longer decides. A sub agent's question arrives as a
|
||||
# `user` message marked written by the agent — it is ordinary
|
||||
# user-assistant data, and dropping it sent Claude Code transcripts to HY
|
||||
# with every question but the first missing. `written_by` labels the
|
||||
# display and decides owner; it cannot tell who actually wrote a message.
|
||||
# A sub-agent question is still a protocol `user` message. Host-specific
|
||||
# author guesses do not decide whether it reaches HY.
|
||||
if record.role != "user":
|
||||
return False
|
||||
if source.casefold().startswith(_HARNESS_PREFIXES):
|
||||
|
||||
@@ -30,6 +30,12 @@ class ConfigDocument(BaseModel):
|
||||
|
||||
schema_version: int = Field(default=1, ge=1, le=1)
|
||||
sources: dict[str, SourceConfig] = Field(default_factory=dict)
|
||||
#: Whether the configured model may turn imported conversations into
|
||||
#: memories. Defaulting to on keeps every existing install doing what it
|
||||
#: did before the switch existed; stopping it is the user's cost control
|
||||
#: and leaves the model configuration itself untouched, so starting again
|
||||
#: needs no re-entry of a key.
|
||||
sedimentation_enabled: StrictBool = True
|
||||
|
||||
@field_validator("sources")
|
||||
@classmethod
|
||||
@@ -63,6 +69,19 @@ class ConfigStore:
|
||||
source = self._document.sources.get(source_id)
|
||||
return bool(source and source.sync_enabled)
|
||||
|
||||
def sedimentation_enabled(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self._document.sedimentation_enabled)
|
||||
|
||||
def set_sedimentation_enabled(self, enabled: bool) -> ConfigDocument:
|
||||
if type(enabled) is not bool:
|
||||
raise ValueError("invalid_sedimentation_config")
|
||||
with self._lock:
|
||||
updated = self._document.model_copy(update={"sedimentation_enabled": enabled})
|
||||
self._write(updated)
|
||||
self._document = updated
|
||||
return updated
|
||||
|
||||
def set_sync(self, source_id: str, enabled: bool) -> ConfigDocument:
|
||||
return self.set_sync_many({source_id: enabled})
|
||||
|
||||
|
||||
@@ -23,10 +23,13 @@ import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
|
||||
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
|
||||
#: Bumped when a derivation rule changes, so old and new ids never compare equal.
|
||||
SCHEMA_VERSION: Final[str] = "ms1"
|
||||
@@ -166,7 +169,6 @@ class CanonicalMessage(BaseModel):
|
||||
message_id: str | None = Field(default=None, min_length=1, max_length=512)
|
||||
ordinal: int = Field(ge=0)
|
||||
role: str = Field(min_length=1, max_length=128)
|
||||
written_by: str = Field(min_length=1, max_length=64)
|
||||
content: str = Field(min_length=1)
|
||||
workspace: str | None = Field(default=None, max_length=1024)
|
||||
|
||||
@@ -213,8 +215,13 @@ class WriteRequest(BaseModel):
|
||||
source_id: str | None = Field(default=None, min_length=1, max_length=128)
|
||||
batch_identity: BatchIdentity
|
||||
evidence_ids: list[str] = Field(min_length=1)
|
||||
# New jobs freeze the complete input in their own payload. The default is
|
||||
# solely for reading pre-upgrade rows whose payload only carried ids.
|
||||
input_records: tuple[EvidenceRecord, ...] = ()
|
||||
idempotency_key: str = Field(min_length=1, max_length=256)
|
||||
priority: Literal["live", "normal"] = "normal"
|
||||
event_at: datetime | None = None
|
||||
time_quality: Literal["exact", "anchored", "session", "approximate"] = "approximate"
|
||||
|
||||
@field_validator("evidence_ids")
|
||||
@classmethod
|
||||
@@ -227,6 +234,16 @@ class WriteRequest(BaseModel):
|
||||
raise ValueError("evidence_ids must be unique")
|
||||
return value
|
||||
|
||||
@field_validator("input_records")
|
||||
@classmethod
|
||||
def _input_records_match_ids(
|
||||
cls, value: tuple[EvidenceRecord, ...], info: ValidationInfo
|
||||
) -> tuple[EvidenceRecord, ...]:
|
||||
evidence_ids = info.data.get("evidence_ids")
|
||||
if value and evidence_ids != [record.evidence_id for record in value]:
|
||||
raise ValueError("input_records must match evidence_ids in order")
|
||||
return value
|
||||
|
||||
@property
|
||||
def request_key(self) -> str:
|
||||
"""Stable dedup key. Same batch + same intent must not enqueue twice."""
|
||||
|
||||
@@ -30,7 +30,7 @@ from uuid import uuid4
|
||||
from one_memory.background_work import in_the_background
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.diagnostic_log import diagnostic_event
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.contracts import WriteRequest
|
||||
from one_memory.memory_server.data_safety import MIN_FREE_BYTES # noqa: F401 - re-exported
|
||||
@@ -249,6 +249,7 @@ MODEL_NOT_READY_CODES = frozenset(
|
||||
#: nothing was sent and retrying the same intent changes nothing — a terminal
|
||||
#: `failed`, not `unknown` (which would wrongly imply a possibly-applied
|
||||
#: write) and not a retry (which would fail identically).
|
||||
#:
|
||||
PRE_SEND_REFUSAL_CODES = frozenset(
|
||||
{
|
||||
"hy_memory_input_exceeds_window",
|
||||
@@ -282,11 +283,6 @@ _VALID_JOB_STATES = frozenset(
|
||||
#: A row in one of these states can never be moved back to a live one.
|
||||
_TERMINAL_WRITE_STATES = frozenset({COMPLETED, FAILED, DEAD, CANCELLED})
|
||||
|
||||
#: A job in one of these states can never run again, so its payload is dead
|
||||
#: weight. `unknown` is deliberately absent: it is non-terminal and still needs
|
||||
#: its intent for reconciliation. `running` is obviously still live.
|
||||
TERMINAL_PAYLOAD_STATES = (COMPLETED, FAILED, DEAD, CANCELLED)
|
||||
|
||||
#: Columns a current schema declares but an older `memory_write_jobs` may lack.
|
||||
#: `CREATE TABLE IF NOT EXISTS` is a no-op for an existing table, so without
|
||||
#: this an upgraded install would keep an older queue that every claim/lease
|
||||
@@ -304,9 +300,12 @@ LEGACY_MEMORY_WRITE_JOB_COLUMNS = (
|
||||
("lease_expires_at", "TEXT"),
|
||||
("hy_invoked", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("last_error", "TEXT"),
|
||||
("event_at", "TEXT"),
|
||||
("time_quality", "TEXT NOT NULL DEFAULT 'approximate'"),
|
||||
)
|
||||
|
||||
_FIFO_BLOCKING_STATES = (PENDING, RUNNING, WAITING_FOR_PERMISSION, WAITING_FOR_MODEL)
|
||||
EVENT_TIME_WINDOW = timedelta(minutes=10)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -393,6 +392,42 @@ def _matches_payload_except_priority(stored: str, payload: str) -> bool:
|
||||
return old == new
|
||||
|
||||
|
||||
def _matches_legacy_reference_replay(stored: str, payload: str) -> bool:
|
||||
"""Match an old id-only caller to the same job's frozen payload."""
|
||||
try:
|
||||
old = json.loads(stored)
|
||||
new = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if not isinstance(old, dict) or not isinstance(new, dict):
|
||||
return False
|
||||
old_inputs = old.get("input_records")
|
||||
new_inputs = new.get("input_records")
|
||||
if bool(old_inputs) == bool(new_inputs):
|
||||
return False
|
||||
old.pop("input_records", None)
|
||||
new.pop("input_records", None)
|
||||
return old == new
|
||||
|
||||
|
||||
def _matches_reaped_legacy_payload(stored: str, payload: str) -> bool:
|
||||
"""Compare a current frozen request with a pre-snapshot reaped payload."""
|
||||
try:
|
||||
old = json.loads(stored)
|
||||
current = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if (
|
||||
not isinstance(old, dict)
|
||||
or not isinstance(current, dict)
|
||||
or not isinstance(old.get(REAPED_PAYLOAD_MARKER), str)
|
||||
):
|
||||
return False
|
||||
current.pop("input_records", None)
|
||||
legacy_payload = json.dumps(current, sort_keys=True)
|
||||
return bool(old[REAPED_PAYLOAD_MARKER] == _payload_digest(legacy_payload))
|
||||
|
||||
|
||||
class DurableQueueSchemaUnavailableError(RuntimeError):
|
||||
"""The durable queue schema has not been created yet.
|
||||
|
||||
@@ -445,6 +480,7 @@ class DurableWriteQueue:
|
||||
if not initialize:
|
||||
return
|
||||
with main_database(database_path).write() as connection, connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
# First create the base table if absent. Then widen an older table
|
||||
# before replaying indexes that reference the new columns; a bare
|
||||
# CREATE INDEX would fail on a pre-priority queue.
|
||||
@@ -455,6 +491,7 @@ class DurableWriteQueue:
|
||||
for statement in memory_write_jobs_sqlite_schema()[1:]:
|
||||
connection.execute(statement)
|
||||
self._backfill_legacy_references(connection)
|
||||
self._backfill_legacy_event_times(connection)
|
||||
# A process that died while holding a lease cannot run its own
|
||||
# recovery. Opening the queue is the first safe point to recover any
|
||||
# rows it left behind, before any new worker claims them.
|
||||
@@ -479,8 +516,7 @@ class DurableWriteQueue:
|
||||
"UPDATE memory_write_jobs SET source_key=session_key WHERE source_key=''"
|
||||
)
|
||||
legacy_rows = connection.execute(
|
||||
"SELECT job_id FROM memory_write_jobs WHERE queue_seq=0 "
|
||||
"ORDER BY created_at, rowid"
|
||||
"SELECT job_id FROM memory_write_jobs WHERE queue_seq=0 ORDER BY created_at, rowid"
|
||||
).fetchall()
|
||||
next_seq = int(
|
||||
connection.execute(
|
||||
@@ -514,6 +550,49 @@ class DurableWriteQueue:
|
||||
if columns and columns != ["state", "priority", "created_at"]:
|
||||
connection.execute("DROP INDEX idx_write_jobs_claim")
|
||||
|
||||
@staticmethod
|
||||
def _backfill_legacy_event_times(connection: sqlite3.Connection) -> None:
|
||||
if (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='evidence'"
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET event_at=created_at WHERE event_at IS NULL"
|
||||
)
|
||||
return
|
||||
evidence_columns = {
|
||||
str(row[1]) for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
|
||||
}
|
||||
if not {"created_at", "imported_at", "role"} <= evidence_columns:
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET event_at=created_at WHERE event_at IS NULL"
|
||||
)
|
||||
return
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE memory_write_jobs
|
||||
SET event_at = COALESCE(
|
||||
(
|
||||
SELECT MAX(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
AND e.role = 'user'
|
||||
),
|
||||
(
|
||||
SELECT MIN(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
),
|
||||
created_at
|
||||
)
|
||||
WHERE event_at IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _backfill_legacy_references(connection: sqlite3.Connection) -> None:
|
||||
"""Populate the reference table after its schema exists."""
|
||||
@@ -610,8 +689,20 @@ class DurableWriteQueue:
|
||||
replacing the earlier intent.
|
||||
"""
|
||||
self._refuse_if_below_reserve()
|
||||
payload = json.dumps(request.model_dump(mode="json"), sort_keys=True)
|
||||
request = self._freeze_request_inputs(request)
|
||||
payload = json.dumps(
|
||||
request.model_dump(
|
||||
mode="json",
|
||||
exclude={"event_at", "time_quality"},
|
||||
),
|
||||
sort_keys=True,
|
||||
)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
resolved_event, resolved_quality = self._event_time_for_request(request)
|
||||
event = request.event_at or resolved_event or datetime.fromisoformat(now)
|
||||
if event.tzinfo is None:
|
||||
event = event.replace(tzinfo=UTC)
|
||||
event_at = event.astimezone(UTC).isoformat()
|
||||
key = request.request_key
|
||||
with (
|
||||
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
|
||||
@@ -624,16 +715,22 @@ class DurableWriteQueue:
|
||||
(key,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
# A replay of work that already finished is a hit, not a
|
||||
# conflict. Terminal rows store a reaped digest instead of the
|
||||
# original payload, and the caller's key is derived from the
|
||||
# evidence's revision, which can move on between replays while
|
||||
# the payload's evidence ids stay the same. Refusing that pair
|
||||
# turned every startup replay of one settled job into an
|
||||
# exception: 236 of them in a single afternoon, all one key.
|
||||
already_settled = (
|
||||
str(existing["state"]) == COMPLETED and int(existing["hy_invoked"]) == 1
|
||||
)
|
||||
stored_payload = str(existing["payload_json"])
|
||||
priority_replay = _matches_payload_except_priority(stored_payload, payload)
|
||||
legacy_reaped_match = bool(
|
||||
already_settled
|
||||
and _matches_reaped_legacy_payload(stored_payload, payload)
|
||||
)
|
||||
if (
|
||||
not _matches_stored_payload(stored_payload, payload)
|
||||
and not priority_replay
|
||||
and not _matches_legacy_reference_replay(stored_payload, payload)
|
||||
and not legacy_reaped_match
|
||||
):
|
||||
raise ValueError(f"idempotency key {key} already exists with different content")
|
||||
if already_settled:
|
||||
return DurableWriteRecord(
|
||||
job_id=str(existing["job_id"]),
|
||||
@@ -644,14 +741,6 @@ class DurableWriteQueue:
|
||||
updated_at=str(existing["updated_at"]),
|
||||
detail="already completed; replay treated as a hit",
|
||||
)
|
||||
priority_replay = _matches_payload_except_priority(
|
||||
str(existing["payload_json"]), payload
|
||||
)
|
||||
if (
|
||||
not _matches_stored_payload(str(existing["payload_json"]), payload)
|
||||
and not priority_replay
|
||||
):
|
||||
raise ValueError(f"idempotency key {key} already exists with different content")
|
||||
if priority_replay and request.priority == "live":
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET priority='live', updated_at=? "
|
||||
@@ -711,11 +800,13 @@ class DurableWriteQueue:
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_write_jobs ("
|
||||
" job_id, queue_seq, session_key, source_key, batch_id, idempotency_key, payload_json,"
|
||||
" job_id, queue_seq, session_key, source_key, batch_id,"
|
||||
" idempotency_key, payload_json,"
|
||||
" evidence_ids_json,"
|
||||
" state, priority, attempts, max_attempts, lease_owner,"
|
||||
" lease_expires_at, hy_invoked, last_error, created_at, updated_at"
|
||||
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
" lease_expires_at, hy_invoked, last_error, event_at, time_quality,"
|
||||
" created_at, updated_at"
|
||||
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
job_id,
|
||||
queue_seq,
|
||||
@@ -733,6 +824,8 @@ class DurableWriteQueue:
|
||||
None,
|
||||
0,
|
||||
None,
|
||||
event_at,
|
||||
request.time_quality if request.event_at is not None else resolved_quality,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
@@ -741,6 +834,18 @@ class DurableWriteQueue:
|
||||
"INSERT INTO memory_write_job_evidence (job_id, evidence_id) VALUES (?,?)",
|
||||
((job_id, str(evidence_id)) for evidence_id in request.evidence_ids),
|
||||
)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.write_queue",
|
||||
event="memory_write_job.queued",
|
||||
outcome="queued",
|
||||
job_id=job_id,
|
||||
details={
|
||||
"source_id": request.source_id or request.internal_session_id,
|
||||
"mode": request.priority,
|
||||
"count": len(request.evidence_ids),
|
||||
},
|
||||
)
|
||||
return DurableWriteRecord(
|
||||
job_id=job_id,
|
||||
request_id=key,
|
||||
@@ -750,6 +855,45 @@ class DurableWriteQueue:
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def _freeze_request_inputs(self, request: WriteRequest) -> WriteRequest:
|
||||
if request.input_records or self.evidence is None:
|
||||
return request
|
||||
records: list[EvidenceRecord] = []
|
||||
for evidence_id in request.evidence_ids:
|
||||
record = self.evidence.get(evidence_id)
|
||||
if record is None:
|
||||
return request
|
||||
records.append(record)
|
||||
return request.model_copy(update={"input_records": tuple(records)})
|
||||
|
||||
def _event_time_for_request(self, request: WriteRequest) -> tuple[datetime | None, str]:
|
||||
records = list(request.input_records)
|
||||
if not records and self.evidence is not None:
|
||||
records = [
|
||||
record
|
||||
for evidence_id in request.evidence_ids
|
||||
if (record := self.evidence.get(evidence_id)) is not None
|
||||
]
|
||||
if not records:
|
||||
return None, "approximate"
|
||||
opener = next(
|
||||
(record for record in records if record.role == "user"),
|
||||
records[0],
|
||||
)
|
||||
if opener.created_at is None:
|
||||
return opener.imported_at, "approximate"
|
||||
origin = opener.origin_meta.get(EvidenceOrigin.FILE)
|
||||
basis = origin.raw_meta.get("time_basis") if origin is not None else None
|
||||
turn_event_at = origin.raw_meta.get("turn_event_at") if origin is not None else None
|
||||
if isinstance(turn_event_at, str):
|
||||
try:
|
||||
return datetime.fromisoformat(turn_event_at), "exact"
|
||||
except ValueError:
|
||||
pass
|
||||
if basis == "file_mtime":
|
||||
return opener.created_at, "session"
|
||||
return opener.created_at, "exact"
|
||||
|
||||
def repair_pre_send_identity_failures(self, *, limit: int = 128) -> int:
|
||||
"""Refresh stale Evidence journals and requeue only provably unsent jobs."""
|
||||
if self.evidence is None:
|
||||
@@ -859,11 +1003,55 @@ class DurableWriteQueue:
|
||||
raise ReconcileSchemaUnavailableError(
|
||||
"evidence table is unavailable; run migrations before reconcile"
|
||||
)
|
||||
where = (
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence ref "
|
||||
"WHERE ref.evidence_id=evidence.evidence_id)"
|
||||
has_versions = (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM sqlite_master "
|
||||
"WHERE type='table' AND name='evidence_versions'"
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
evidence_columns = {
|
||||
str(row[1])
|
||||
for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
|
||||
}
|
||||
if {"revision", "content_hash", "updated_at"} <= evidence_columns:
|
||||
revision_time = (
|
||||
"COALESCE((SELECT version.changed_at FROM evidence_versions version "
|
||||
"WHERE version.evidence_id=evidence.evidence_id "
|
||||
"AND version.revision=evidence.revision),"
|
||||
"evidence.updated_at,evidence.imported_at)"
|
||||
if has_versions
|
||||
else "COALESCE(evidence.updated_at,evidence.imported_at)"
|
||||
)
|
||||
where = (
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_job_evidence linked "
|
||||
"JOIN memory_write_jobs job ON job.job_id=linked.job_id, "
|
||||
"json_each(job.payload_json,'$.input_records') frozen "
|
||||
"WHERE linked.evidence_id=evidence.evidence_id "
|
||||
"AND json_extract(frozen.value,'$.evidence_id')=evidence.evidence_id "
|
||||
"AND json_extract(frozen.value,'$.revision')=evidence.revision "
|
||||
"AND json_extract(frozen.value,'$.content_hash')=evidence.content_hash"
|
||||
") "
|
||||
# Pre-snapshot builds only recorded an Evidence id. Treat
|
||||
# that as coverage only when its Job was created after this
|
||||
# exact revision.
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_job_evidence legacy "
|
||||
"JOIN memory_write_jobs legacy_job ON legacy_job.job_id=legacy.job_id "
|
||||
"WHERE legacy.evidence_id=evidence.evidence_id "
|
||||
"AND COALESCE(json_array_length("
|
||||
"json_extract(legacy_job.payload_json,'$.input_records')),0)=0 "
|
||||
f"AND legacy_job.created_at>=({revision_time}))"
|
||||
)
|
||||
else:
|
||||
# A genuinely old/minimal schema has no revision identity.
|
||||
where = (
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence legacy "
|
||||
"WHERE legacy.evidence_id=evidence.evidence_id)"
|
||||
)
|
||||
params: tuple[Any, ...] = ()
|
||||
cursor_clause = ""
|
||||
if after_cursor is not None:
|
||||
@@ -1095,6 +1283,7 @@ class DurableWriteQueue:
|
||||
with main_database(self.database_path).write() as connection, connection:
|
||||
updated = connection.execute(
|
||||
"UPDATE memory_write_jobs SET state=?,lease_owner=NULL,"
|
||||
"attempts=CASE WHEN attempts>0 THEN attempts-1 ELSE 0 END,"
|
||||
"lease_expires_at=NULL,updated_at=? WHERE " + where,
|
||||
(PENDING, now, *params),
|
||||
)
|
||||
@@ -1162,45 +1351,10 @@ class DurableWriteQueue:
|
||||
return state
|
||||
|
||||
def clear_terminal_payloads(self, *, limit: int = 512) -> int:
|
||||
"""Replace terminal rows' request bodies with a content digest.
|
||||
|
||||
A completed/failed/dead/cancelled row can never run again, so the full
|
||||
serialized `WriteRequest` is unbounded dead weight. A digest keeps the
|
||||
idempotency contract intact: a replay of the same intent still resolves
|
||||
to the same row, while the same key with different content still
|
||||
conflicts. `unknown` is left alone because it is non-terminal and its
|
||||
intent is needed for reconciliation.
|
||||
|
||||
Bounded by `limit` so a large backlog drains over several calls instead
|
||||
of one long write on the single writer.
|
||||
"""
|
||||
"""Deprecated compatibility hook; durable inputs are immutable audit data."""
|
||||
if type(limit) is not int or limit < 1:
|
||||
raise ValueError("limit must be positive")
|
||||
placeholders = ",".join("?" for _ in TERMINAL_PAYLOAD_STATES)
|
||||
with (
|
||||
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
|
||||
connection,
|
||||
):
|
||||
rows = connection.execute(
|
||||
"SELECT job_id, payload_json FROM memory_write_jobs "
|
||||
f"WHERE state IN ({placeholders}) "
|
||||
"AND instr(payload_json, ?) = 0 "
|
||||
"LIMIT ?",
|
||||
(*TERMINAL_PAYLOAD_STATES, REAPED_PAYLOAD_MARKER, limit),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
for row in rows:
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET payload_json=? "
|
||||
f"WHERE job_id=? AND state IN ({placeholders})",
|
||||
(
|
||||
_reaped_payload(_payload_digest(str(row["payload_json"]))),
|
||||
str(row["job_id"]),
|
||||
*TERMINAL_PAYLOAD_STATES,
|
||||
),
|
||||
)
|
||||
return len(rows)
|
||||
return 0
|
||||
|
||||
def readmit_parked(self, *, states: tuple[str, ...] = PARKED_STATES) -> int:
|
||||
"""Move parked jobs back to `pending` for a fresh claim.
|
||||
@@ -1402,7 +1556,13 @@ class DurableWriteQueue:
|
||||
cross back to pending, so this cannot duplicate a possibly-sent write.
|
||||
"""
|
||||
now = datetime.now(UTC).isoformat()
|
||||
payload = json.dumps(request.model_dump(mode="json"), sort_keys=True)
|
||||
payload = json.dumps(
|
||||
request.model_dump(
|
||||
mode="json",
|
||||
exclude={"event_at", "time_quality"},
|
||||
),
|
||||
sort_keys=True,
|
||||
)
|
||||
with (
|
||||
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
|
||||
connection,
|
||||
@@ -1741,9 +1901,12 @@ class DurableWriteRegistry:
|
||||
def evidence_for_request(store: EvidenceStore, request: WriteRequest) -> tuple[EvidenceRecord, ...]:
|
||||
"""The Evidence a stored write points at, in the request's own order.
|
||||
|
||||
A write may only point at Evidence that still exists; a missing id is a
|
||||
rejection rather than a silent partial send.
|
||||
Current jobs are self-contained: their immutable records are the model
|
||||
input even if a Harness later rewinds or edits its live transcript. The
|
||||
store lookup remains only for payloads written by pre-upgrade builds.
|
||||
"""
|
||||
if request.input_records:
|
||||
return request.input_records
|
||||
records = []
|
||||
for evidence_id in request.evidence_ids:
|
||||
record = store.get(evidence_id)
|
||||
@@ -1850,7 +2013,14 @@ async def execute_job(
|
||||
# Never persist the raw message: it may be a provider response or the
|
||||
# user's own text, and `last_error` is shown over HTTP as `detail`.
|
||||
code = safe_error_code(error)
|
||||
if code == "model_content_source_disabled" and not hy_invoked:
|
||||
if (
|
||||
code
|
||||
in {
|
||||
"model_content_source_disabled",
|
||||
"model_sedimentation_disabled",
|
||||
}
|
||||
and not hy_invoked
|
||||
):
|
||||
await in_the_background(queue.release_unstarted, job_id, owner=owner)
|
||||
return
|
||||
# The task layer classifies these before any bridge is opened, so HY
|
||||
@@ -1861,7 +2031,11 @@ async def execute_job(
|
||||
# - transient model/provider trouble -> waiting_for_model (parked).
|
||||
# Everything else may have crossed into HY, and HY has no idempotency
|
||||
# key, so it is `unknown` rather than a blind retry.
|
||||
if hy_invoked:
|
||||
if code in PRE_SEND_REFUSAL_CODES:
|
||||
# The frozen input window is checked before the payload reaches
|
||||
# HY. This one refusal therefore has a proven pre-send boundary.
|
||||
state, invoked_marker = FAILED, False
|
||||
elif hy_invoked:
|
||||
# Error names do not prove execution phase: permission and
|
||||
# evidence rechecks also run after HY returns. Once the durable
|
||||
# marker is set, every failure is possibly-sent and must stay out
|
||||
@@ -1873,10 +2047,6 @@ async def execute_job(
|
||||
state, invoked_marker = WAITING_FOR_PERMISSION, False
|
||||
elif code in MODEL_NOT_READY_CODES:
|
||||
state, invoked_marker = WAITING_FOR_MODEL, False
|
||||
elif code in PRE_SEND_REFUSAL_CODES:
|
||||
# Provably before the bridge: the frozen input is stale or too
|
||||
# large. Nothing was sent, so this is terminal rather than unknown.
|
||||
state, invoked_marker = FAILED, False
|
||||
elif code == "model_content_transport_request_failed":
|
||||
# The generation or embedding HTTP exchange can fail after HY has
|
||||
# already persisted a node. Treating that as a pre-send failure
|
||||
@@ -1965,7 +2135,6 @@ class DurableWriteDrain:
|
||||
#: sustained backlog must not keep finished rows' request bodies in the
|
||||
#: business database indefinitely. Its own gate so neither interval can
|
||||
#: starve the other.
|
||||
self._last_terminal_reap: datetime | None = None
|
||||
#: `waiting_for_model` re-admission is also time-based recoverability:
|
||||
#: a model outage clears on its own, and a queue that never goes idle
|
||||
#: must still re-admit the write once the wait has elapsed.
|
||||
@@ -2016,7 +2185,7 @@ class DurableWriteDrain:
|
||||
now: datetime | None = None,
|
||||
budget_ms: int | None = None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Claim at most one FIFO head per enabled Harness."""
|
||||
"""Claim event-time heads, with one running job per enabled Harness."""
|
||||
if type(limit) is not int or limit < 1:
|
||||
raise ValueError("limit must be positive")
|
||||
now = now or datetime.now(UTC)
|
||||
@@ -2058,9 +2227,7 @@ class DurableWriteDrain:
|
||||
(PENDING,),
|
||||
).fetchall()
|
||||
enabled_sources = {
|
||||
str(row[0])
|
||||
for row in source_rows
|
||||
if self.source_writes_permitted(str(row[0]))
|
||||
str(row[0]) for row in source_rows if self.source_writes_permitted(str(row[0]))
|
||||
}
|
||||
if not enabled_sources:
|
||||
return ()
|
||||
@@ -2079,38 +2246,142 @@ class DurableWriteDrain:
|
||||
placeholders = ",".join("?" for _ in enabled_sources)
|
||||
source_clause = f"AND candidate.source_key IN ({placeholders}) "
|
||||
source_params = tuple(sorted(enabled_sources))
|
||||
window = connection.execute(
|
||||
"SELECT window_anchor,window_end FROM memory_write_schedule_state WHERE singleton=1"
|
||||
).fetchone()
|
||||
if window is not None:
|
||||
anchor_text, end_text = str(window[0]), str(window[1])
|
||||
active_source_clause = ""
|
||||
active_source_params: tuple[Any, ...] = ()
|
||||
if enabled_sources is not None:
|
||||
placeholders = ",".join("?" for _ in enabled_sources)
|
||||
active_source_clause = f"AND active.source_key IN ({placeholders}) "
|
||||
active_source_params = tuple(sorted(enabled_sources))
|
||||
# The contract retires a window when every *eligible*
|
||||
# pending/running job in it has settled or parked. A pending row
|
||||
# that scheduling already holds back (an unexpired lease, a
|
||||
# busy Harness, or an earlier event in its Session) is not eligible, so it
|
||||
# must not keep the anchor alive: those blockers usually sit
|
||||
# outside this ten-minute range and can never be claimed from
|
||||
# inside it, which pinned the anchor and starved the whole
|
||||
# queue. Reuse the claim's own predicate so "the window is
|
||||
# busy" and "the window can still be worked" agree.
|
||||
active = connection.execute(
|
||||
"SELECT 1 FROM memory_write_jobs AS active "
|
||||
"WHERE active.event_at>=? AND active.event_at<? AND ("
|
||||
"active.state=? OR (active.state=? "
|
||||
"AND (active.lease_expires_at IS NULL "
|
||||
"OR active.lease_expires_at < ?) " + active_source_clause + "AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS held "
|
||||
"WHERE held.source_key=active.source_key AND held.state=? "
|
||||
"AND held.lease_owner IS NOT NULL "
|
||||
"AND held.lease_expires_at>=?) "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS earlier "
|
||||
"WHERE earlier.session_key=active.session_key "
|
||||
f"AND earlier.state IN ({blocking}) "
|
||||
"AND (earlier.event_at<active.event_at OR "
|
||||
"(earlier.event_at=active.event_at "
|
||||
"AND earlier.queue_seq<active.queue_seq))) "
|
||||
")) LIMIT 1",
|
||||
(
|
||||
anchor_text,
|
||||
end_text,
|
||||
RUNNING,
|
||||
PENDING,
|
||||
now.isoformat(),
|
||||
*active_source_params,
|
||||
RUNNING,
|
||||
now.isoformat(),
|
||||
*_FIFO_BLOCKING_STATES,
|
||||
),
|
||||
).fetchone()
|
||||
if active is None:
|
||||
connection.execute("DELETE FROM memory_write_schedule_state WHERE singleton=1")
|
||||
window = None
|
||||
if window is None:
|
||||
anchor = connection.execute(
|
||||
"SELECT candidate.event_at "
|
||||
"FROM memory_write_jobs AS candidate "
|
||||
"WHERE candidate.state=? "
|
||||
"AND (candidate.lease_expires_at IS NULL "
|
||||
"OR candidate.lease_expires_at < ?) " + source_clause + "AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS held "
|
||||
"WHERE held.source_key=candidate.source_key AND held.state=? "
|
||||
"AND held.lease_owner IS NOT NULL "
|
||||
"AND held.lease_expires_at>=?) "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS earlier "
|
||||
"WHERE earlier.session_key=candidate.session_key "
|
||||
f"AND earlier.state IN ({blocking}) "
|
||||
"AND (earlier.event_at<candidate.event_at OR "
|
||||
"(earlier.event_at=candidate.event_at "
|
||||
"AND earlier.queue_seq<candidate.queue_seq))) "
|
||||
"ORDER BY candidate.event_at,candidate.queue_seq LIMIT 1",
|
||||
(
|
||||
PENDING,
|
||||
now.isoformat(),
|
||||
*source_params,
|
||||
RUNNING,
|
||||
now.isoformat(),
|
||||
*_FIFO_BLOCKING_STATES,
|
||||
),
|
||||
).fetchone()
|
||||
if anchor is None:
|
||||
return ()
|
||||
anchor_at = datetime.fromisoformat(str(anchor[0]))
|
||||
if anchor_at.tzinfo is None:
|
||||
anchor_at = anchor_at.replace(tzinfo=UTC)
|
||||
anchor_text = anchor_at.astimezone(UTC).isoformat()
|
||||
end_text = (anchor_at + EVENT_TIME_WINDOW).astimezone(UTC).isoformat()
|
||||
connection.execute(
|
||||
"INSERT INTO memory_write_schedule_state "
|
||||
"(singleton,window_anchor,window_end,updated_at) "
|
||||
"VALUES (1,?,?,?)",
|
||||
(anchor_text, end_text, now.isoformat()),
|
||||
)
|
||||
rows = connection.execute(
|
||||
"SELECT candidate.job_id,candidate.source_key "
|
||||
"FROM memory_write_jobs AS candidate "
|
||||
"WHERE candidate.state=? "
|
||||
"AND (candidate.lease_expires_at IS NULL OR candidate.lease_expires_at < ?) "
|
||||
+ source_clause +
|
||||
+ source_clause
|
||||
+ "AND candidate.event_at>=? AND candidate.event_at<? "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS held "
|
||||
"WHERE held.source_key=candidate.source_key AND held.state=? "
|
||||
"AND held.lease_owner IS NOT NULL AND held.lease_expires_at>=?) "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS earlier "
|
||||
"WHERE earlier.source_key=candidate.source_key "
|
||||
"WHERE earlier.session_key=candidate.session_key "
|
||||
f"AND earlier.state IN ({blocking}) "
|
||||
"AND earlier.queue_seq<candidate.queue_seq) "
|
||||
"ORDER BY candidate.queue_seq",
|
||||
"AND (earlier.event_at<candidate.event_at OR "
|
||||
"(earlier.event_at=candidate.event_at "
|
||||
"AND earlier.queue_seq<candidate.queue_seq))) "
|
||||
"ORDER BY candidate.event_at,candidate.queue_seq",
|
||||
(
|
||||
PENDING,
|
||||
now.isoformat(),
|
||||
*source_params,
|
||||
anchor_text,
|
||||
end_text,
|
||||
RUNNING,
|
||||
now.isoformat(),
|
||||
*_FIFO_BLOCKING_STATES,
|
||||
),
|
||||
).fetchall()
|
||||
selected: list[str] = []
|
||||
selected_sources: set[str] = set()
|
||||
for row in rows:
|
||||
source_key = str(row["source_key"])
|
||||
if source_key in selected_sources:
|
||||
continue
|
||||
if self.source_claim_enabled is not None and not self.source_claim_enabled(
|
||||
str(row["source_key"])
|
||||
source_key
|
||||
):
|
||||
continue
|
||||
selected.append(str(row["job_id"]))
|
||||
selected_sources.add(source_key)
|
||||
if len(selected) == min(limit, self.max_concurrent_sources):
|
||||
break
|
||||
if not selected:
|
||||
@@ -2339,19 +2610,6 @@ class DurableWriteDrain:
|
||||
self.queue.reclaim_expired,
|
||||
operation_name="reclaim_expired",
|
||||
)
|
||||
# Terminal-payload reaping is a bounded retention write: a
|
||||
# finished row's full request body must not sit in the business
|
||||
# database forever just because the queue never went idle. Its
|
||||
# own interval keeps it off every poll.
|
||||
if (
|
||||
self._last_terminal_reap is None
|
||||
or tick_started - self._last_terminal_reap >= self._reap_interval
|
||||
):
|
||||
self._last_terminal_reap = tick_started
|
||||
await self._run_idle_maintenance(
|
||||
self.queue.clear_terminal_payloads,
|
||||
operation_name="clear_terminal_payloads",
|
||||
)
|
||||
# A model outage clears on its own, so `waiting_for_model` jobs
|
||||
# are re-admitted on a timer. That must not depend on idleness: a
|
||||
# queue busy for longer than the wait window would otherwise keep
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The result decides whether the `unknown` job state can ever be retired, so it
|
||||
is probed against the installed SDK rather than written down from memory.
|
||||
|
||||
What was verified against `hy-memory==2.0.3`:
|
||||
What was verified against `hy-memory==2.0.4`:
|
||||
|
||||
- `HyMemoryClient.add(..., request_id=...)` accepts a caller-supplied id and
|
||||
echoes it back. It is described as a request/receipt handle, and the SDK
|
||||
|
||||
@@ -151,6 +151,8 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
lease_expires_at TEXT,
|
||||
hy_invoked INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
event_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
time_quality TEXT NOT NULL DEFAULT 'approximate',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
@@ -164,6 +166,8 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
ON memory_write_jobs(source_key, state, queue_seq);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_jobs_fifo
|
||||
ON memory_write_jobs(queue_seq);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_jobs_event_claim
|
||||
ON memory_write_jobs(state, event_at, queue_seq);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_jobs_source_recent
|
||||
ON memory_write_jobs(source_key, updated_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS memory_write_job_evidence (
|
||||
@@ -172,7 +176,13 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
PRIMARY KEY (job_id, evidence_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_job_evidence_evidence
|
||||
ON memory_write_job_evidence(evidence_id)
|
||||
ON memory_write_job_evidence(evidence_id);
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor TEXT NOT NULL,
|
||||
window_end TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
return """
|
||||
CREATE TABLE IF NOT EXISTS memory_write_jobs (
|
||||
@@ -192,6 +202,8 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
lease_expires_at DATETIME(6) NULL,
|
||||
hy_invoked TINYINT(1) NOT NULL DEFAULT 0,
|
||||
last_error TEXT NULL,
|
||||
event_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
|
||||
time_quality VARCHAR(32) NOT NULL DEFAULT 'approximate',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
|
||||
ON UPDATE CURRENT_TIMESTAMP(6),
|
||||
@@ -201,6 +213,7 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
KEY idx_write_jobs_source_claim (state, source_key, priority, created_at),
|
||||
KEY idx_write_jobs_source_fifo (source_key, state, queue_seq),
|
||||
KEY idx_write_jobs_fifo (queue_seq),
|
||||
KEY idx_write_jobs_event_claim (state, event_at, queue_seq),
|
||||
KEY idx_write_jobs_source_recent (source_key, updated_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
@@ -209,6 +222,13 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
evidence_id VARCHAR(128) NOT NULL,
|
||||
PRIMARY KEY (job_id, evidence_id),
|
||||
KEY idx_write_job_evidence_evidence (evidence_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INT PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor DATETIME(6) NOT NULL,
|
||||
window_end DATETIME(6) NOT NULL,
|
||||
updated_at DATETIME(6) NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
|
||||
@@ -1231,6 +1251,68 @@ CREATE TABLE IF NOT EXISTS model_request_budget_events (
|
||||
"""
|
||||
|
||||
|
||||
def model_request_audit_ddl(dialect: str = "mysql") -> str:
|
||||
"""Content-free accounting for physical precipitation model requests."""
|
||||
if dialect == "sqlite":
|
||||
return """
|
||||
CREATE TABLE IF NOT EXISTS model_request_audit (
|
||||
audit_id TEXT PRIMARY KEY,
|
||||
request_id TEXT NOT NULL,
|
||||
profile_id TEXT NOT NULL,
|
||||
profile_revision INTEGER NOT NULL,
|
||||
provider_kind TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
request_kind TEXT NOT NULL,
|
||||
prompt_stage TEXT NOT NULL DEFAULT 'other',
|
||||
started_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
outcome TEXT NOT NULL CHECK(outcome IN ('in_flight','succeeded','failed')),
|
||||
http_status INTEGER,
|
||||
error_type TEXT,
|
||||
provider_request_id TEXT,
|
||||
prompt_tokens INTEGER,
|
||||
completion_tokens INTEGER,
|
||||
total_tokens INTEGER,
|
||||
cached_prompt_tokens INTEGER,
|
||||
reasoning_tokens INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS model_request_audit_request
|
||||
ON model_request_audit(request_id, started_at);
|
||||
CREATE INDEX IF NOT EXISTS model_request_audit_started
|
||||
ON model_request_audit(started_at);
|
||||
"""
|
||||
return """
|
||||
CREATE TABLE IF NOT EXISTS model_request_audit (
|
||||
audit_id VARCHAR(128) PRIMARY KEY,
|
||||
request_id VARCHAR(512) NOT NULL,
|
||||
profile_id VARCHAR(128) NOT NULL,
|
||||
profile_revision INT NOT NULL,
|
||||
provider_kind VARCHAR(64) NOT NULL,
|
||||
model VARCHAR(512) NOT NULL,
|
||||
request_kind VARCHAR(64) NOT NULL,
|
||||
prompt_stage VARCHAR(128) NOT NULL DEFAULT 'other',
|
||||
started_at DATETIME(6) NOT NULL,
|
||||
completed_at DATETIME(6) NULL,
|
||||
outcome VARCHAR(32) NOT NULL CHECK(outcome IN ('in_flight','succeeded','failed')),
|
||||
http_status INT NULL,
|
||||
error_type VARCHAR(128) NULL,
|
||||
provider_request_id VARCHAR(512) NULL,
|
||||
prompt_tokens BIGINT NULL,
|
||||
completion_tokens BIGINT NULL,
|
||||
total_tokens BIGINT NULL,
|
||||
cached_prompt_tokens BIGINT NULL,
|
||||
reasoning_tokens BIGINT NULL,
|
||||
KEY model_request_audit_request (request_id, started_at),
|
||||
KEY model_request_audit_started (started_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
|
||||
|
||||
async def _052_model_request_audit(cursor: Any) -> None:
|
||||
for statement in _split_statements(model_request_audit_ddl(_dialect(cursor))):
|
||||
await cursor.execute(statement)
|
||||
|
||||
|
||||
def model_probe_operations_ddl(dialect: str = "mysql") -> str:
|
||||
"""Probe operations; `state` is CHECKed to a known set.
|
||||
|
||||
@@ -1414,6 +1496,97 @@ async def _051_drop_hy_node_permissions(cursor: Any) -> None:
|
||||
await cursor.execute("DROP TABLE IF EXISTS hy_node_permissions")
|
||||
|
||||
|
||||
async def _053_memory_write_event_time(cursor: Any) -> None:
|
||||
"""Add event-time scheduling without re-keying any existing write."""
|
||||
|
||||
dialect = _dialect(cursor)
|
||||
columns = await cursor.columns("memory_write_jobs")
|
||||
added_event_at = "event_at" not in columns
|
||||
if added_event_at:
|
||||
await cursor.execute(
|
||||
"ALTER TABLE memory_write_jobs ADD COLUMN event_at "
|
||||
+ ("TEXT" if dialect == "sqlite" else "DATETIME(6) NULL")
|
||||
)
|
||||
if "time_quality" not in columns:
|
||||
await cursor.execute(
|
||||
"ALTER TABLE memory_write_jobs ADD COLUMN time_quality "
|
||||
+ (
|
||||
"TEXT NOT NULL DEFAULT 'approximate'"
|
||||
if dialect == "sqlite"
|
||||
else "VARCHAR(32) NOT NULL DEFAULT 'approximate'"
|
||||
)
|
||||
)
|
||||
await cursor.execute(
|
||||
"""
|
||||
UPDATE memory_write_jobs
|
||||
SET event_at = COALESCE(
|
||||
(
|
||||
SELECT MAX(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
AND e.role = 'user'
|
||||
),
|
||||
(
|
||||
SELECT MIN(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
),
|
||||
created_at
|
||||
)
|
||||
WHERE event_at IS NULL
|
||||
"""
|
||||
)
|
||||
if dialect == "sqlite":
|
||||
await cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_write_jobs_event_claim "
|
||||
"ON memory_write_jobs(state, event_at, queue_seq)"
|
||||
)
|
||||
await cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor TEXT NOT NULL,
|
||||
window_end TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
else:
|
||||
if added_event_at:
|
||||
await cursor.execute(
|
||||
"CREATE INDEX idx_write_jobs_event_claim "
|
||||
"ON memory_write_jobs(state, event_at, queue_seq)"
|
||||
)
|
||||
await cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INT PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor DATETIME(6) NOT NULL,
|
||||
window_end DATETIME(6) NOT NULL,
|
||||
updated_at DATETIME(6) NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
async def _054_legacy_possibly_sent_tasks(cursor: Any) -> None:
|
||||
"""Preserve the old sent marker before removing its retired permission table."""
|
||||
columns = await cursor.columns("memory_task_continuous_permissions")
|
||||
if "task_id" not in columns:
|
||||
return
|
||||
await cursor.execute(
|
||||
"""
|
||||
UPDATE memory_tasks
|
||||
SET hy_invocation_count=CASE
|
||||
WHEN hy_invocation_count<1 THEN 1 ELSE hy_invocation_count END
|
||||
WHERE task_id IN (SELECT task_id FROM memory_task_continuous_permissions)
|
||||
"""
|
||||
)
|
||||
await cursor.execute("DROP TABLE memory_task_continuous_permissions")
|
||||
|
||||
|
||||
def model_profiles_ddl(dialect: str = "mysql") -> str:
|
||||
"""Model profiles and their version history.
|
||||
|
||||
@@ -2680,7 +2853,11 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
2,
|
||||
"memory_write_jobs",
|
||||
_002_write_jobs,
|
||||
("memory_write_jobs", "memory_write_job_evidence"),
|
||||
(
|
||||
"memory_write_jobs",
|
||||
"memory_write_job_evidence",
|
||||
"memory_write_schedule_state",
|
||||
),
|
||||
),
|
||||
Migration(3, "source_file_cursors", _003_scanner_cursors, ("source_file_cursors",)),
|
||||
Migration(4, "change_outbox", _004_change_outbox, ("change_outbox",)),
|
||||
@@ -2849,4 +3026,21 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
_050_model_probe_operations_reserved_at,
|
||||
),
|
||||
Migration(51, "drop_hy_node_permissions", _051_drop_hy_node_permissions),
|
||||
Migration(
|
||||
52,
|
||||
"model_request_audit",
|
||||
_052_model_request_audit,
|
||||
("model_request_audit",),
|
||||
),
|
||||
Migration(
|
||||
53,
|
||||
"memory_write_event_time",
|
||||
_053_memory_write_event_time,
|
||||
("memory_write_schedule_state",),
|
||||
),
|
||||
Migration(
|
||||
54,
|
||||
"legacy_possibly_sent_tasks",
|
||||
_054_legacy_possibly_sent_tasks,
|
||||
),
|
||||
)
|
||||
@@ -107,6 +107,8 @@ def build_request(
|
||||
first_round: int,
|
||||
last_round: int,
|
||||
evidence_ids: list[str],
|
||||
input_records: tuple[EvidenceRecord, ...] = (),
|
||||
source_id: str | None = None,
|
||||
priority: Literal["live", "normal"] = "normal",
|
||||
idempotency_key: str | None = None,
|
||||
) -> WriteRequest:
|
||||
@@ -125,6 +127,8 @@ def build_request(
|
||||
internal_session_id=internal_session_id,
|
||||
batch_identity=batch,
|
||||
evidence_ids=evidence_ids,
|
||||
input_records=input_records,
|
||||
source_id=source_id,
|
||||
idempotency_key=idempotency_key or batch.batch_id,
|
||||
priority=priority,
|
||||
)
|
||||
@@ -271,6 +275,8 @@ def build_request_from_evidence(
|
||||
first_round=first_round,
|
||||
last_round=last_round,
|
||||
evidence_ids=list(ordered),
|
||||
input_records=tuple(ordered.values()),
|
||||
source_id=seed.source_id,
|
||||
priority=priority,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
@@ -18,10 +18,11 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from hashlib import sha256
|
||||
from typing import Literal
|
||||
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.memory_server.contracts import (
|
||||
BatchIdentity,
|
||||
WriteRequest,
|
||||
@@ -124,6 +125,12 @@ class UnifiedWriteIntake:
|
||||
priority: WritePriority,
|
||||
) -> WriteRequest:
|
||||
session_id = internal_session_id_for(record.conversation_id, record.source_id)
|
||||
event_at, time_quality = _job_time((record,))
|
||||
request_key = (
|
||||
f"entry:{record.evidence_id}"
|
||||
if record.revision == 1
|
||||
else (f"entry:v2:{record.evidence_id}:{record.revision}:{record.content_hash}")
|
||||
)
|
||||
return WriteRequest(
|
||||
internal_session_id=session_id,
|
||||
source_id=record.source_id,
|
||||
@@ -133,8 +140,11 @@ class UnifiedWriteIntake:
|
||||
last_round=last_round,
|
||||
),
|
||||
evidence_ids=[record.evidence_id],
|
||||
idempotency_key=f"entry:{record.evidence_id}",
|
||||
input_records=(record,),
|
||||
idempotency_key=request_key,
|
||||
priority=priority,
|
||||
event_at=event_at,
|
||||
time_quality=time_quality,
|
||||
)
|
||||
|
||||
def submit_records(
|
||||
@@ -161,6 +171,7 @@ class UnifiedWriteIntake:
|
||||
)
|
||||
request_id = f"turn:v1:{sha256(material.encode()).hexdigest()}"
|
||||
session_id = internal_session_id_for(first.conversation_id, first.source_id)
|
||||
event_at, time_quality = _job_time(records)
|
||||
durable = self.queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id=session_id,
|
||||
@@ -171,8 +182,11 @@ class UnifiedWriteIntake:
|
||||
last_round=1,
|
||||
),
|
||||
evidence_ids=[record.evidence_id for record in records],
|
||||
input_records=records,
|
||||
idempotency_key=request_id,
|
||||
priority=priority,
|
||||
event_at=event_at,
|
||||
time_quality=time_quality,
|
||||
)
|
||||
)
|
||||
return UnifiedWriteReceipt(
|
||||
@@ -189,3 +203,29 @@ class SourceWriteDisabledError(RuntimeError):
|
||||
def __init__(self, evidence_id: str) -> None:
|
||||
super().__init__("model_content_source_disabled")
|
||||
self.evidence_id = evidence_id
|
||||
|
||||
|
||||
def _job_time(
|
||||
records: tuple[EvidenceRecord, ...],
|
||||
) -> tuple[datetime, Literal["exact", "anchored", "session", "approximate"]]:
|
||||
"""Use the turn opener's wall clock, with an explicit fallback quality."""
|
||||
|
||||
opener = next(
|
||||
(record for record in records if record.role == "user"),
|
||||
records[0],
|
||||
)
|
||||
if opener.created_at is None:
|
||||
return opener.imported_at, "approximate"
|
||||
file_origin = opener.origin_meta.get(EvidenceOrigin.FILE)
|
||||
basis = file_origin.raw_meta.get("time_basis") if file_origin is not None else None
|
||||
turn_event_at = file_origin.raw_meta.get("turn_event_at") if file_origin is not None else None
|
||||
if isinstance(turn_event_at, str):
|
||||
try:
|
||||
return datetime.fromisoformat(turn_event_at), "exact"
|
||||
except ValueError:
|
||||
pass
|
||||
if basis == "file_mtime":
|
||||
return opener.created_at, "session"
|
||||
if EvidenceOrigin.HOOK in opener.origins:
|
||||
return opener.created_at, "exact"
|
||||
return opener.created_at, "exact"
|
||||
@@ -17,7 +17,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.database import take_the_write_lock_first
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceRecord, WrittenBy
|
||||
from one_memory.memory_server.migrations import memory_tasks_ddl
|
||||
from one_memory.model_profiles import ModelProfile, ModelProfileSnapshot
|
||||
|
||||
@@ -174,29 +174,60 @@ def evidence_ref(record: EvidenceRecord) -> TaskEvidenceRef:
|
||||
"segment_end": int(marker["end"]),
|
||||
"segment_hash": str(marker["hash"]),
|
||||
}
|
||||
identity = record.model_dump(mode="json", exclude={"origin_meta", "imported_at", "updated_at"})
|
||||
return {
|
||||
"evidence_id": record.evidence_id,
|
||||
"revision": record.revision,
|
||||
"content_hash": record.content_hash,
|
||||
"identity_hash": sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest(),
|
||||
"identity_hash": _evidence_identity_hash(record, include_written_by=False),
|
||||
}
|
||||
|
||||
|
||||
def _evidence_identity_hash(record: EvidenceRecord, *, include_written_by: bool) -> str:
|
||||
excluded = {"origin_meta", "imported_at", "updated_at"}
|
||||
if not include_written_by:
|
||||
excluded.add("written_by")
|
||||
identity = record.model_dump(mode="json", exclude=excluded)
|
||||
return sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def _identity_hash_matches(record: EvidenceRecord, expected: object) -> bool:
|
||||
candidates = {_evidence_identity_hash(record, include_written_by=False)}
|
||||
# Tasks created by pre-role-only builds keep their original proof. The
|
||||
# legacy metadata could also have changed in place without a revision, so
|
||||
# accept the old hash under every historical author guess.
|
||||
for written_by in WrittenBy:
|
||||
candidates.add(
|
||||
_evidence_identity_hash(
|
||||
record.model_copy(update={"written_by": written_by}),
|
||||
include_written_by=True,
|
||||
)
|
||||
)
|
||||
return expected in candidates
|
||||
|
||||
|
||||
def matches_evidence_ref(record: EvidenceRecord, ref: TaskEvidenceRef) -> bool:
|
||||
if "segment_hash" in ref:
|
||||
base = evidence_ref(record)
|
||||
if any(
|
||||
base.get(key) != ref.get(key)
|
||||
for key in ("evidence_id", "revision", "content_hash", "identity_hash")
|
||||
for key in ("evidence_id", "revision", "content_hash")
|
||||
):
|
||||
return False
|
||||
if not _identity_hash_matches(record, ref.get("identity_hash")):
|
||||
return False
|
||||
start, end = ref.get("segment_start"), ref.get("segment_end")
|
||||
if not isinstance(start, int) or not isinstance(end, int) or not (0 <= start < end):
|
||||
return False
|
||||
return sha256(record.verbatim[start:end].encode()).hexdigest() == ref["segment_hash"]
|
||||
current = evidence_ref(record)
|
||||
return all(current.get(key) == value for key, value in ref.items())
|
||||
return all(
|
||||
(
|
||||
_identity_hash_matches(record, value)
|
||||
if key == "identity_hash"
|
||||
else current.get(key) == value
|
||||
)
|
||||
for key, value in ref.items()
|
||||
)
|
||||
|
||||
|
||||
def segment_evidence_record(
|
||||
@@ -977,6 +1008,44 @@ class MemoryTaskStore:
|
||||
error_code=error_code,
|
||||
)
|
||||
|
||||
def release_unstarted(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
runner_token: str,
|
||||
error_code: str,
|
||||
) -> MemoryTask:
|
||||
"""Return a claimed task to pending when a user pause wins before HY.
|
||||
|
||||
Claim increments ``attempts``. A pause is not an execution failure, so
|
||||
the attempt is returned as well; repeatedly pausing and resuming can
|
||||
never exhaust an otherwise healthy task.
|
||||
"""
|
||||
now = datetime.now(UTC).isoformat()
|
||||
with self._connect(writing=True) as connection:
|
||||
changed = connection.execute(
|
||||
"""
|
||||
UPDATE memory_tasks
|
||||
SET status=?, runner_token=NULL,
|
||||
attempts=CASE WHEN attempts>0 THEN attempts-1 ELSE 0 END,
|
||||
last_error_code=?, duration_seconds=NULL, updated_at=?
|
||||
WHERE task_id=? AND status=? AND runner_token=?
|
||||
""",
|
||||
(
|
||||
MemoryTaskStatus.PENDING,
|
||||
error_code,
|
||||
now,
|
||||
task_id,
|
||||
MemoryTaskStatus.RUNNING,
|
||||
runner_token,
|
||||
),
|
||||
)
|
||||
if changed.rowcount != 1:
|
||||
raise MemoryTaskOwnershipError(
|
||||
"Memory task runner no longer owns this task"
|
||||
)
|
||||
return self.require(task_id)
|
||||
|
||||
def park_for_permission(self, task_id: str, *, error_code: str) -> bool:
|
||||
"""Park unstarted work until a later permission snapshot admits it."""
|
||||
now = datetime.now(UTC).isoformat()
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
"""Persistent, content-free accounting for precipitation model requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import TypedDict
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.memory_server.migrations import model_request_audit_ddl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelRequestAuditContext:
|
||||
request_id: str
|
||||
|
||||
|
||||
_audit_context: ContextVar[ModelRequestAuditContext | None] = ContextVar(
|
||||
"one_memory_model_request_audit_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def audit_precipitation_request(request_id: str) -> Iterator[None]:
|
||||
"""Enable physical-request accounting for one memory precipitation."""
|
||||
|
||||
token = _audit_context.set(ModelRequestAuditContext(request_id=request_id))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_audit_context.reset(token)
|
||||
|
||||
|
||||
def _current_prompt_stage() -> str:
|
||||
try:
|
||||
resolver = getattr( # noqa: B009 - optional SDK version boundary
|
||||
import_module("hy_memory.utils.model_call_context"),
|
||||
"current_model_call_stage",
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
return "other"
|
||||
return str(resolver())
|
||||
|
||||
|
||||
class ModelRequestAuditStore:
|
||||
def __init__(self, database: Path) -> None:
|
||||
self.database = database
|
||||
with main_database(database).write() as connection, connection:
|
||||
for statement in model_request_audit_ddl("sqlite").split(";"):
|
||||
if statement.strip():
|
||||
connection.execute(statement)
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
request_id: str,
|
||||
profile_id: str,
|
||||
profile_revision: int,
|
||||
provider_kind: str,
|
||||
model: str,
|
||||
request_kind: str,
|
||||
prompt_stage: str,
|
||||
) -> str:
|
||||
audit_id = str(uuid4())
|
||||
with main_database(self.database).write() as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO model_request_audit (
|
||||
audit_id,request_id,profile_id,profile_revision,provider_kind,
|
||||
model,request_kind,prompt_stage,started_at,outcome
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
audit_id,
|
||||
request_id,
|
||||
profile_id,
|
||||
profile_revision,
|
||||
provider_kind,
|
||||
model,
|
||||
request_kind,
|
||||
prompt_stage,
|
||||
datetime.now(UTC).isoformat(),
|
||||
"in_flight",
|
||||
),
|
||||
)
|
||||
return audit_id
|
||||
|
||||
def finish(
|
||||
self,
|
||||
audit_id: str,
|
||||
*,
|
||||
outcome: str,
|
||||
http_status: int | None,
|
||||
error_type: str | None = None,
|
||||
provider_request_id: str | None = None,
|
||||
prompt_tokens: int | None = None,
|
||||
completion_tokens: int | None = None,
|
||||
total_tokens: int | None = None,
|
||||
cached_prompt_tokens: int | None = None,
|
||||
reasoning_tokens: int | None = None,
|
||||
) -> None:
|
||||
with main_database(self.database).write() as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE model_request_audit
|
||||
SET completed_at=?,outcome=?,http_status=?,error_type=?,
|
||||
provider_request_id=?,prompt_tokens=?,completion_tokens=?,
|
||||
total_tokens=?,cached_prompt_tokens=?,reasoning_tokens=?
|
||||
WHERE audit_id=?
|
||||
""",
|
||||
(
|
||||
datetime.now(UTC).isoformat(),
|
||||
outcome,
|
||||
http_status,
|
||||
error_type,
|
||||
provider_request_id,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens,
|
||||
cached_prompt_tokens,
|
||||
reasoning_tokens,
|
||||
audit_id,
|
||||
),
|
||||
)
|
||||
|
||||
def report(
|
||||
self,
|
||||
*,
|
||||
since: str | None = None,
|
||||
request_id: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> dict[str, object]:
|
||||
clauses: list[str] = []
|
||||
params: list[object] = []
|
||||
if since is not None:
|
||||
clauses.append("started_at>=?")
|
||||
params.append(since)
|
||||
if request_id is not None:
|
||||
clauses.append("request_id=?")
|
||||
params.append(request_id)
|
||||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
with main_database(self.database).read() as connection:
|
||||
summary = connection.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS request_count,
|
||||
SUM(CASE WHEN outcome='succeeded' THEN 1 ELSE 0 END) AS succeeded,
|
||||
SUM(CASE WHEN outcome='failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN outcome='in_flight' THEN 1 ELSE 0 END) AS in_flight,
|
||||
SUM(prompt_tokens) AS prompt_tokens,
|
||||
SUM(completion_tokens) AS completion_tokens,
|
||||
SUM(total_tokens) AS total_tokens,
|
||||
SUM(cached_prompt_tokens) AS cached_prompt_tokens,
|
||||
SUM(reasoning_tokens) AS reasoning_tokens,
|
||||
AVG(prompt_tokens) AS avg_prompt_tokens,
|
||||
AVG(completion_tokens) AS avg_completion_tokens
|
||||
FROM model_request_audit {where}
|
||||
""",
|
||||
tuple(params),
|
||||
).fetchone()
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT audit_id,request_id,profile_id,profile_revision,provider_kind,
|
||||
model,request_kind,prompt_stage,started_at,completed_at,outcome,http_status,
|
||||
error_type,provider_request_id,prompt_tokens,completion_tokens,
|
||||
total_tokens,cached_prompt_tokens,reasoning_tokens
|
||||
FROM model_request_audit {where}
|
||||
ORDER BY started_at DESC,audit_id DESC LIMIT ?
|
||||
""",
|
||||
(*params, limit),
|
||||
).fetchall()
|
||||
minute_rows = connection.execute(
|
||||
f"""
|
||||
SELECT substr(started_at,1,16) || ':00Z' AS minute,
|
||||
prompt_stage,
|
||||
COUNT(*) AS request_count,
|
||||
SUM(prompt_tokens) AS prompt_tokens,
|
||||
SUM(completion_tokens) AS completion_tokens,
|
||||
SUM(total_tokens) AS total_tokens,
|
||||
AVG(prompt_tokens) AS avg_prompt_tokens,
|
||||
AVG(completion_tokens) AS avg_completion_tokens
|
||||
FROM model_request_audit {where}
|
||||
GROUP BY substr(started_at,1,16),prompt_stage
|
||||
ORDER BY minute,prompt_stage
|
||||
""",
|
||||
tuple(params),
|
||||
).fetchall()
|
||||
keys = (
|
||||
"audit_id",
|
||||
"request_id",
|
||||
"profile_id",
|
||||
"profile_revision",
|
||||
"provider_kind",
|
||||
"model",
|
||||
"request_kind",
|
||||
"prompt_stage",
|
||||
"started_at",
|
||||
"completed_at",
|
||||
"outcome",
|
||||
"http_status",
|
||||
"error_type",
|
||||
"provider_request_id",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"cached_prompt_tokens",
|
||||
"reasoning_tokens",
|
||||
)
|
||||
summary_keys = (
|
||||
"request_count",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"in_flight",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"cached_prompt_tokens",
|
||||
"reasoning_tokens",
|
||||
"avg_prompt_tokens",
|
||||
"avg_completion_tokens",
|
||||
)
|
||||
return {
|
||||
"summary": {
|
||||
key: (
|
||||
round(float(value), 2)
|
||||
if key.startswith("avg_") and value is not None
|
||||
else int(value)
|
||||
if value is not None
|
||||
else 0
|
||||
)
|
||||
for key, value in zip(summary_keys, summary, strict=True)
|
||||
},
|
||||
"requests": [
|
||||
dict(zip(keys, tuple(row), strict=True))
|
||||
for row in rows
|
||||
],
|
||||
"by_minute": [
|
||||
{
|
||||
"minute": str(row[0]),
|
||||
"prompt_stage": str(row[1]),
|
||||
"request_count": int(row[2]),
|
||||
"prompt_tokens": int(row[3] or 0),
|
||||
"completion_tokens": int(row[4] or 0),
|
||||
"total_tokens": int(row[5] or 0),
|
||||
"avg_prompt_tokens": round(float(row[6] or 0), 2),
|
||||
"avg_completion_tokens": round(float(row[7] or 0), 2),
|
||||
}
|
||||
for row in minute_rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _non_negative_int(value: object) -> int | None:
|
||||
return value if type(value) is int and value >= 0 else None
|
||||
|
||||
|
||||
class _ResponseUsage(TypedDict):
|
||||
provider_request_id: str | None
|
||||
prompt_tokens: int | None
|
||||
completion_tokens: int | None
|
||||
total_tokens: int | None
|
||||
cached_prompt_tokens: int | None
|
||||
reasoning_tokens: int | None
|
||||
|
||||
|
||||
def _empty_usage() -> _ResponseUsage:
|
||||
return {
|
||||
"provider_request_id": None,
|
||||
"prompt_tokens": None,
|
||||
"completion_tokens": None,
|
||||
"total_tokens": None,
|
||||
"cached_prompt_tokens": None,
|
||||
"reasoning_tokens": None,
|
||||
}
|
||||
|
||||
|
||||
def _response_usage(content: bytes) -> _ResponseUsage:
|
||||
try:
|
||||
body = json.loads(content)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
return _empty_usage()
|
||||
if not isinstance(body, dict):
|
||||
return _empty_usage()
|
||||
usage = body.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
usage = {}
|
||||
prompt_details = usage.get("prompt_tokens_details")
|
||||
completion_details = usage.get("completion_tokens_details")
|
||||
cached = (
|
||||
prompt_details.get("cached_tokens")
|
||||
if isinstance(prompt_details, dict)
|
||||
else usage.get("prompt_cache_hit_tokens")
|
||||
)
|
||||
reasoning = (
|
||||
completion_details.get("reasoning_tokens")
|
||||
if isinstance(completion_details, dict)
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"provider_request_id": body.get("id") if isinstance(body.get("id"), str) else None,
|
||||
"prompt_tokens": _non_negative_int(usage.get("prompt_tokens")),
|
||||
"completion_tokens": _non_negative_int(usage.get("completion_tokens")),
|
||||
"total_tokens": _non_negative_int(usage.get("total_tokens")),
|
||||
"cached_prompt_tokens": _non_negative_int(cached),
|
||||
"reasoning_tokens": _non_negative_int(reasoning),
|
||||
}
|
||||
|
||||
|
||||
def _request_kind(request: httpx.Request) -> str:
|
||||
path = request.url.path.rstrip("/")
|
||||
if path.endswith("/chat/completions"):
|
||||
return "chat_completion"
|
||||
if path.endswith("/embeddings"):
|
||||
return "embedding"
|
||||
return "model_http"
|
||||
|
||||
|
||||
def _request_model(content: bytes, fallback: str) -> str:
|
||||
try:
|
||||
body = json.loads(content)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
return fallback
|
||||
model = body.get("model") if isinstance(body, dict) else None
|
||||
return model if isinstance(model, str) and model else fallback
|
||||
|
||||
|
||||
class AuditedModelTransport(httpx.AsyncBaseTransport):
|
||||
"""Record one row for each physical precipitation HTTP attempt."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: ModelRequestAuditStore,
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_revision: int,
|
||||
provider_kind: str,
|
||||
model: str,
|
||||
inner: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.profile_id = profile_id
|
||||
self.profile_revision = profile_revision
|
||||
self.provider_kind = provider_kind
|
||||
self.model = model
|
||||
self.inner = inner or httpx.AsyncHTTPTransport(retries=0, trust_env=False)
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
context = _audit_context.get()
|
||||
if context is None:
|
||||
return await self.inner.handle_async_request(request)
|
||||
content = await request.aread()
|
||||
audit_id = await asyncio.to_thread(
|
||||
self.store.start,
|
||||
request_id=context.request_id,
|
||||
profile_id=self.profile_id,
|
||||
profile_revision=self.profile_revision,
|
||||
provider_kind=self.provider_kind,
|
||||
model=_request_model(content, self.model),
|
||||
request_kind=_request_kind(request),
|
||||
prompt_stage=_current_prompt_stage(),
|
||||
)
|
||||
try:
|
||||
response = await self.inner.handle_async_request(request)
|
||||
except BaseException as error:
|
||||
await asyncio.to_thread(
|
||||
self.store.finish,
|
||||
audit_id,
|
||||
outcome="failed",
|
||||
http_status=None,
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise
|
||||
content = await response.aread()
|
||||
usage = _response_usage(content)
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
self.store.finish,
|
||||
audit_id,
|
||||
outcome="succeeded" if response.is_success else "failed",
|
||||
http_status=response.status_code,
|
||||
error_type=None if response.is_success else f"http_{response.status_code}",
|
||||
**usage,
|
||||
)
|
||||
except Exception:
|
||||
# The durable in_flight row is more truthful than hiding the model
|
||||
# response after it has already incurred cost.
|
||||
logger.exception("model_request_audit_finish_failed audit_id=%s", audit_id)
|
||||
return response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.inner.aclose()
|
||||
@@ -87,7 +87,7 @@ def build_acceptance_pack(
|
||||
"""
|
||||
SELECT DISTINCT e.evidence_id
|
||||
FROM evidence e
|
||||
WHERE e.role = 'user' AND e.written_by = 'person'
|
||||
WHERE e.role = 'user'
|
||||
AND LENGTH(e.verbatim) <= 300
|
||||
AND (
|
||||
e.verbatim LIKE '%我是一名%' OR e.verbatim LIKE '%我是一个%'
|
||||
@@ -118,7 +118,7 @@ def build_acceptance_pack(
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE role = 'user' AND written_by != 'person'
|
||||
WHERE role != 'user'
|
||||
"""
|
||||
):
|
||||
strata["attribution_edge"].add(row["evidence_id"])
|
||||
@@ -138,11 +138,7 @@ def build_acceptance_pack(
|
||||
quota,
|
||||
)
|
||||
if len(selected) < sample_size:
|
||||
eligible = (
|
||||
evidence_id
|
||||
for evidence_id, row in evidence.items()
|
||||
if row["role"] == "user" and row["written_by"] == "person"
|
||||
)
|
||||
eligible = (evidence_id for evidence_id, row in evidence.items() if row["role"] == "user")
|
||||
_extend_unique(selected, _stable_order(eligible, seed=f"{seed}:fill"), sample_size)
|
||||
selected = selected[:sample_size]
|
||||
if len(selected) < sample_size:
|
||||
@@ -163,7 +159,11 @@ def build_acceptance_pack(
|
||||
row_id=f"{index:03d}",
|
||||
evidence_id=evidence_id,
|
||||
source_id=source["source_id"],
|
||||
written_by=source["written_by"],
|
||||
written_by=(
|
||||
"person"
|
||||
if source["role"] == "user"
|
||||
else ("agent" if source["role"] == "assistant" else source["role"])
|
||||
),
|
||||
character_count=len(source["verbatim"]),
|
||||
content_hash=source["content_hash"],
|
||||
strata=tuple(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
|
||||
from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
from one_memory.memory_processing import _is_hy_input_turn
|
||||
@@ -15,10 +15,7 @@ def identity_content_ready(record: EvidenceRecord) -> bool:
|
||||
# cleaning requires a source map and remains explicitly uncovered for now.
|
||||
return (
|
||||
record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and (
|
||||
(record.role == "user" and record.written_by is WrittenBy.PERSON)
|
||||
or (record.role == "assistant" and record.written_by is WrittenBy.AGENT)
|
||||
)
|
||||
and record.role in {"user", "assistant"}
|
||||
and _is_hy_input_turn(record)
|
||||
and not is_host_generated_user_turn(record.verbatim)
|
||||
and strip_host_wrappers(record.verbatim) == record.verbatim.strip()
|
||||
|
||||
@@ -10,7 +10,6 @@ from one_memory.imports.models import (
|
||||
MAX_VERBATIM_CHARACTERS,
|
||||
EvidenceContentKind,
|
||||
EvidenceRecord,
|
||||
WrittenBy,
|
||||
)
|
||||
from one_memory.imports.sources.parsing import (
|
||||
is_host_generated_user_turn,
|
||||
@@ -164,10 +163,10 @@ def prepare_mapped_content(record: EvidenceRecord) -> PreparedMappedContent:
|
||||
The returned text is transient; only metadata is suitable for an outer manifest.
|
||||
That manifest must independently bind Evidence identity/revision/hash and grants.
|
||||
"""
|
||||
if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or not (
|
||||
(record.role == "user" and record.written_by is WrittenBy.PERSON)
|
||||
or (record.role == "assistant" and record.written_by is WrittenBy.AGENT)
|
||||
):
|
||||
if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or record.role not in {
|
||||
"user",
|
||||
"assistant",
|
||||
}:
|
||||
raise MappedContentError("mapped_content_ineligible")
|
||||
original = record.verbatim
|
||||
if len(original) > MAX_VERBATIM_CHARACTERS:
|
||||
|
||||
@@ -11,7 +11,7 @@ from datetime import UTC
|
||||
|
||||
from one_memory import ingest_journal as journal
|
||||
from one_memory.context_raw import _check, _connection
|
||||
from one_memory.imports.models import EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.retrieval.index_store import RetrievalIndexStore
|
||||
from one_memory.retrieval.models import EvidenceSpan, RawEvidence
|
||||
@@ -166,7 +166,7 @@ def materialize_raw_candidates(
|
||||
source_id=record.source_id,
|
||||
conversation_id=record.conversation_id,
|
||||
role=record.role,
|
||||
written_by="person" if record.written_by is WrittenBy.PERSON else "agent",
|
||||
written_by="person" if record.role == "user" else "agent",
|
||||
event_time=event_time,
|
||||
spans=tuple(spans),
|
||||
matched_by=("semantic",),
|
||||
|
||||
@@ -264,7 +264,7 @@ class ServiceLevelScheduler:
|
||||
self._quiet_admission_logged_at[key] = now
|
||||
if last is not None:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="service_level.scheduler",
|
||||
event="service_level.summary",
|
||||
outcome="succeeded",
|
||||
@@ -276,7 +276,7 @@ class ServiceLevelScheduler:
|
||||
)
|
||||
return
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
level="debug",
|
||||
component="service_level.scheduler",
|
||||
event="service_level.admitted",
|
||||
outcome="succeeded",
|
||||
|
||||
@@ -56,10 +56,10 @@ def build_source_import_receipt(
|
||||
"SELECT count(*) AS messages, "
|
||||
"count(DISTINCT conversation_id || char(0) || COALESCE(workspace, '')) "
|
||||
"AS conversations, "
|
||||
"coalesce(sum(written_by='person'),0) AS person_messages, "
|
||||
"coalesce(sum(written_by='agent'),0) AS agent_messages, "
|
||||
"coalesce(sum(written_by='tool'),0) AS tool_messages, "
|
||||
"coalesce(sum(written_by NOT IN ('person','agent','tool')),0) "
|
||||
"coalesce(sum(role='user'),0) AS person_messages, "
|
||||
"coalesce(sum(role='assistant'),0) AS agent_messages, "
|
||||
"coalesce(sum(role='tool'),0) AS tool_messages, "
|
||||
"coalesce(sum(role NOT IN ('user','assistant','tool')),0) "
|
||||
"AS unknown_author_messages, "
|
||||
"coalesce(sum(created_at IS NULL),0) AS unknown_time_messages "
|
||||
"FROM evidence WHERE user_id=? AND source_id=?",
|
||||
@@ -68,7 +68,9 @@ def build_source_import_receipt(
|
||||
# Bound text reads before materializing snippets. These are samples, not a
|
||||
# recency ranking; sorting every large original body is unnecessary here.
|
||||
examples = db.execute(
|
||||
"SELECT evidence_id, role, written_by, created_at, "
|
||||
"SELECT evidence_id, role, CASE role "
|
||||
"WHEN 'user' THEN 'person' WHEN 'assistant' THEN 'agent' "
|
||||
"WHEN 'tool' THEN 'tool' ELSE 'unknown' END AS written_by, created_at, "
|
||||
"substr(verbatim,1,600) AS excerpt, length(verbatim)>600 AS truncated "
|
||||
"FROM evidence WHERE evidence_id IN ("
|
||||
"SELECT evidence_id FROM evidence WHERE user_id=? AND source_id=? LIMIT 3)",
|
||||
@@ -80,10 +82,13 @@ def build_source_import_receipt(
|
||||
# Qualify before LIMIT: agent/system rows must not consume the three samples.
|
||||
query_filter = " AND evidence_id=?" if query_evidence_id is not None else ""
|
||||
queries = db.execute(
|
||||
f"""SELECT evidence_id, revision AS evidence_revision, role, written_by, created_at,
|
||||
CASE WHEN user_id=? AND source_id=? AND role='user' AND written_by='person'
|
||||
f"""SELECT evidence_id, revision AS evidence_revision, role,
|
||||
CASE role WHEN 'user' THEN 'person' WHEN 'assistant' THEN 'agent'
|
||||
WHEN 'tool' THEN 'tool' ELSE 'unknown' END AS written_by,
|
||||
created_at,
|
||||
CASE WHEN user_id=? AND source_id=? AND role='user'
|
||||
AND length(verbatim)<=65536
|
||||
THEN display_query_projection(verbatim,role,written_by)
|
||||
THEN display_query_projection(verbatim,role,'person')
|
||||
ELSE NULL END AS projection
|
||||
FROM evidence WHERE user_id=? AND source_id=?{query_filter}
|
||||
AND projection IS NOT NULL
|
||||
|
||||
@@ -250,6 +250,7 @@
|
||||
| `test_model_profile_identity.py` | `contract` | Stable model editing and configured-vs-available credentials; no Keychain/network. |
|
||||
| `test_model_profile_queue_handover.py` | `contract` | Queued work when the model changes or its configuration is deleted (D75). |
|
||||
| `test_model_profiles.py` | `contract` | 覆盖 profile uses one model and keeps key out of database、api key updates reuse one stable secret reference、startup cleanup removes only secrets not referenced by current profiles 等 15 个场景。 |
|
||||
| `test_model_request_audit.py` | `contract` | 覆盖 failure is recorded without error message、missing usage stays unknown in detail and zero in summary、records one physical request with provider usage and no content 等 5 个场景。 |
|
||||
| `test_model_request_budget.py` | `contract` | 覆盖 persists failed request reservations without content、parallel leases cannot exceed request limit、unknown or excessive output and body are denied 等 6 个场景。 |
|
||||
| `test_mysql_lifecycle.py` | `contract` | A1-07: the decisions Runtime makes around the bundled MySQL. |
|
||||
| `test_mysql_platform.py` | `contract` | A1-07: the machine-facing half of managing the bundled MySQL. |
|
||||
@@ -327,6 +328,7 @@
|
||||
| `test_secret_store_error_codes.py` | `integration` | 3925a kernel item 3: keychain failure classification and backoff that heals. |
|
||||
| `test_secure_storage.py` | `integration` | 覆盖 isolated keychain service namespace、custom keychain service is refused outside isolated tests、packaged keychain budget covers loaded process startup 等 20 个场景。 |
|
||||
| `test_sedimentation_concurrency.py` | `contract` | Sedimentation runs several harnesses at once, but one harness at a time. |
|
||||
| `test_sedimentation_switch.py` | `contract` | Stopping model-driven sedimentation without losing the configuration. |
|
||||
| `test_service_level_pressure_acceptance.py` | `contract` | 覆盖 workbuddy recall remains available during 100k import 场景。 |
|
||||
| `test_service_levels.py` | `contract` | 覆盖 background never consumes the realtime reservation、realtime is next for the same source after background yields、import history gets one slice after nine realtime admissions 等 8 个场景。 |
|
||||
| `test_session_inbox.py` | `contract` | A session inbox is not a shared mailbox and a pull is not adoption. |
|
||||
@@ -336,6 +338,7 @@
|
||||
| `test_snapshot_pool.py` | `contract` | A whole-directory snapshot must not hold a thread everything else is waiting for. |
|
||||
| `test_soak_probe_effectiveness.py` | `integration` | The soak probe must observe the load it reports on. |
|
||||
| `test_source_access.py` | `contract` | 覆盖 source root requires explicit user grant、sensitive home directory is rejected without requesting os access、authorized file cannot escape root through a symlink 等 24 个场景。 |
|
||||
| `test_source_authorization_state.py` | `unit` | Why a source cannot be read, named separately from whether it can. |
|
||||
| `test_source_candidate_api.py` | `integration` | Candidate location changes must not implicitly grant or import anything. |
|
||||
| `test_source_candidate_connections.py` | `contract` | 覆盖 empty candidate revisions preserve default fingerprint、persisted candidate requires exact revision、connection resolves candidate in transaction and prior replay never regrants 等 7 个场景。 |
|
||||
| `test_source_candidates.py` | `contract` | 覆盖 preview is metadata only and cancel changes nothing、select persists only candidate and restarts with revision、compare and swap and aba cannot reuse old preview 等 9 个场景。 |
|
||||
@@ -385,4 +388,4 @@
|
||||
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
|
||||
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
|
||||
|
||||
共 380 个测试文件。
|
||||
共 383 个测试文件。
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
|
||||
EXPECTED_TEST_FILE_COUNT = 380
|
||||
EXPECTED_TEST_FILES_SHA256 = "9f075d18403328882c25325d29ebe77d0e3365d675ba19d56134a6a0fb9b64b4"
|
||||
EXPECTED_TEST_FILE_COUNT = 383
|
||||
EXPECTED_TEST_FILES_SHA256 = "3c1a3d3febaf478e1ba005fbcd22ad23d76fc5925b32b8d8740e4e65c554b922"
|
||||
|
||||
# Pure transformations, validation models, parity checks and state machines.
|
||||
# This list is intentionally small: uncertain files default to contract.
|
||||
@@ -35,6 +35,7 @@ UNIT_FILES = frozenset(
|
||||
"test_memory_server_job_state.py",
|
||||
"test_scan_roots.py",
|
||||
"test_scanner_classifier.py",
|
||||
"test_source_authorization_state.py",
|
||||
"test_scanner_sessionizer.py",
|
||||
"test_source_envelope_positions.py",
|
||||
"test_source_lifecycle.py",
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import asyncio
|
||||
import gc
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.background_work import (
|
||||
BULK_WORKERS,
|
||||
bulk_pool,
|
||||
in_interactive_read,
|
||||
in_read_pool,
|
||||
in_the_background,
|
||||
shutdown_bulk_pool,
|
||||
)
|
||||
@@ -110,3 +115,73 @@ def test_interactive_read_does_not_wait_behind_saturated_default_pool() -> None:
|
||||
shutdown_bulk_pool()
|
||||
|
||||
assert waited < 0.25
|
||||
|
||||
|
||||
def test_an_abandoned_read_does_not_report_an_unretrieved_failure() -> None:
|
||||
"""A read that outlives its caller must not leave its failure lying about.
|
||||
|
||||
`in_read_pool` shields the worker, so giving up on an answer does not yank
|
||||
a query out from under SQLite. The caller cancels; the read runs on. When
|
||||
such an abandoned read then failed, nothing awaited its future any more and
|
||||
asyncio reported an exception nobody retrieved -- which the loop handler
|
||||
files as `core.loop.task_exception`, a red error sitting next to a lane
|
||||
that had already degraded on its own budget and already said so.
|
||||
"""
|
||||
|
||||
def open_pool() -> ThreadPoolExecutor:
|
||||
return ThreadPoolExecutor(max_workers=1, thread_name_prefix="test-read-pool")
|
||||
|
||||
async def scenario() -> list[BaseException]:
|
||||
loop = asyncio.get_running_loop()
|
||||
reported: list[BaseException] = []
|
||||
previous = loop.get_exception_handler()
|
||||
loop.set_exception_handler(lambda _loop, context: reported.append(context.get("exception")))
|
||||
gate = threading.Event()
|
||||
|
||||
def work() -> None:
|
||||
gate.wait(5.0)
|
||||
raise TimeoutError("raw_context_timeout")
|
||||
|
||||
pool = open_pool()
|
||||
try:
|
||||
read = asyncio.create_task(in_read_pool(pool, work))
|
||||
await asyncio.sleep(0.05)
|
||||
read.cancel() # the lane ran out of budget; the read is still going
|
||||
await asyncio.sleep(0.10) # the cancellation settles first
|
||||
gate.set() # only now does the abandoned read fail
|
||||
await asyncio.gather(read, return_exceptions=True)
|
||||
# Let the shielded worker finish, then let asyncio notice anything
|
||||
# that nobody ever retrieved. The reference has to go: asyncio only
|
||||
# reports an unretrieved outcome when the future is collected.
|
||||
del read
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0.02)
|
||||
gc.collect()
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
loop.set_exception_handler(previous)
|
||||
return [error for error in reported if isinstance(error, TimeoutError)]
|
||||
|
||||
assert asyncio.run(scenario()) == []
|
||||
|
||||
|
||||
def test_an_awaited_read_still_raises_its_failure() -> None:
|
||||
"""Marking the outcome seen must not swallow it for a caller who waited."""
|
||||
|
||||
def open_pool() -> ThreadPoolExecutor:
|
||||
return ThreadPoolExecutor(max_workers=1, thread_name_prefix="test-read-pool")
|
||||
|
||||
async def scenario() -> str:
|
||||
def work() -> None:
|
||||
raise TimeoutError("raw_context_timeout")
|
||||
|
||||
pool = open_pool()
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
await in_read_pool(pool, work)
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
return "raised"
|
||||
|
||||
assert asyncio.run(scenario()) == "raised"
|
||||
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -13,6 +14,7 @@ from one_memory.diagnostic_log import (
|
||||
configure_diagnostics,
|
||||
current_trace_id,
|
||||
database_error_code,
|
||||
diagnostic_event,
|
||||
diagnostic_exception,
|
||||
diagnostics_enabled,
|
||||
diagnostics_requested,
|
||||
@@ -23,6 +25,32 @@ from one_memory.diagnostic_log import (
|
||||
)
|
||||
|
||||
|
||||
def test_debug_events_are_collected_only_at_debug_level(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.INFO)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
component="core.poll",
|
||||
event="poll.completed",
|
||||
outcome="succeeded",
|
||||
)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.task",
|
||||
event="memory_task.finished",
|
||||
outcome="completed",
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["event"] for event in events] == ["memory_task.finished"]
|
||||
|
||||
|
||||
def test_high_frequency_success_events_are_summarized_but_warnings_are_not(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
@@ -97,6 +125,53 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) -> None:
|
||||
"""Rejection protects the log; it must not become the caller's outage.
|
||||
|
||||
Two of our own nodes shipped with unlisted `details` keys. Both sat on a
|
||||
request path, so the rejection surfaced as a 500 to the person using the
|
||||
app — including the sedimentation switch, whose every request failed
|
||||
because logging the toggle raised before the endpoint could answer.
|
||||
"""
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"prompt": "private"},
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
# Rejected, so nothing is written — but the call returned.
|
||||
assert not receipt.path.exists() or receipt.path.read_text() == ""
|
||||
|
||||
|
||||
def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
|
||||
"""`enabled` is a bounded boolean and has to be a legal detail key.
|
||||
|
||||
Omitting it from the allow-list is what made the switch's own endpoint
|
||||
answer 500 while still saving the new value.
|
||||
"""
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"enabled": False},
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["event"] for event in events] == ["memory_sedimentation.toggled"]
|
||||
assert events[0]["details"]["enabled"] is False
|
||||
|
||||
|
||||
def test_exception_diagnostic_keeps_code_location_not_message():
|
||||
secret = "private-provider-response-918273"
|
||||
log = DiagnosticLog(Path.cwd())
|
||||
|
||||
@@ -16,16 +16,15 @@ import pytest
|
||||
|
||||
from one_memory.diagnostic_log import _DETAIL_KEYS
|
||||
|
||||
SOURCES = (
|
||||
"api.py",
|
||||
"imports/async_bridge.py",
|
||||
"imports/live_sync.py",
|
||||
"imports/runner.py",
|
||||
"memory_processing.py",
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
|
||||
|
||||
#: Every module, not a hand-picked list. The two payloads that caused real
|
||||
#: outages were in files nobody thought to add here, so the only shape of this
|
||||
#: check that can actually hold the line is "every module, always".
|
||||
SOURCES = tuple(
|
||||
sorted(str(path.relative_to(ROOT)) for path in ROOT.rglob("*.py") if path.name != "__init__.py")
|
||||
)
|
||||
|
||||
|
||||
def _literal_calls(path: Path):
|
||||
tree = ast.parse(path.read_text())
|
||||
|
||||
@@ -162,8 +162,6 @@ def test_import_request_trace_reaches_the_background_job_without_original_text(t
|
||||
]
|
||||
names = [event["event"] for event in events]
|
||||
assert "import.request.started" in names
|
||||
assert "import.queued" in names
|
||||
assert "import.started" in names
|
||||
assert "import.finished" in names
|
||||
finished = next(event for event in events if event["event"] == "import.finished")
|
||||
assert {
|
||||
@@ -175,18 +173,17 @@ def test_import_request_trace_reaches_the_background_job_without_original_text(t
|
||||
assert secret not in written
|
||||
|
||||
|
||||
def test_import_run_logs_parse_stages_and_core_resources_without_content(tmp_path):
|
||||
def test_default_info_omits_parser_and_resource_internals(tmp_path):
|
||||
written, job_id = _run_codex_import(tmp_path)
|
||||
events = list(map(json.loads, written.splitlines()))
|
||||
job_events = [event for event in events if event.get("job_id") == job_id]
|
||||
names = [event["event"] for event in job_events]
|
||||
assert names.index("import.parse.started") < names.index("import.parse.completed")
|
||||
assert "import.phase_changed" in names
|
||||
completed = next(event for event in job_events if event["event"] == "import.parse.completed")
|
||||
assert completed["outcome"] == "succeeded"
|
||||
assert {"stored_count", "skipped_count", "error_count"} <= completed["details"].keys()
|
||||
assert "import.parse.started" not in names
|
||||
assert "import.parse.completed" not in names
|
||||
assert "import.phase_changed" not in names
|
||||
assert "import.finished" in names
|
||||
everything = [event["event"] for event in events]
|
||||
assert "core.resource_snapshot" in everything
|
||||
assert "core.resource_snapshot" not in everything
|
||||
assert everything.index("core.ready") < everything.index("core.stopped")
|
||||
assert IMPORT_SECRET not in written
|
||||
|
||||
@@ -230,4 +227,5 @@ def test_a_failing_import_finished_log_does_not_escape_the_supervisor_task(tmp_p
|
||||
|
||||
monkeypatch.setattr(supervisor, "diagnostic_event", rejecting)
|
||||
written, _job_id = _run_codex_import(tmp_path)
|
||||
assert "import.started" in written
|
||||
assert "core.stopped" in written
|
||||
assert "import.finished" not in written
|
||||
@@ -33,6 +33,42 @@ def test_trace_lookup_is_disabled_until_detailed_logging_is_enabled(tmp_path):
|
||||
assert response.json() == {"schema_version": 1, "state": "disabled", "events": []}
|
||||
|
||||
|
||||
def test_model_usage_route_returns_content_free_totals(tmp_path):
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path,
|
||||
source_home=tmp_path / "home",
|
||||
api_token="trace-token",
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
with TestClient(app, headers={"Authorization": "Bearer trace-token"}) as client:
|
||||
response = client.get("/api/v1/diagnostics/model-usage")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"request_count": 0,
|
||||
"succeeded": 0,
|
||||
"failed": 0,
|
||||
"in_flight": 0,
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"cached_prompt_tokens": 0,
|
||||
"reasoning_tokens": 0,
|
||||
"avg_prompt_tokens": 0,
|
||||
"avg_completion_tokens": 0,
|
||||
},
|
||||
"requests": [],
|
||||
"by_minute": [],
|
||||
}
|
||||
|
||||
|
||||
def test_trace_lookup_returns_only_safe_structured_steps(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
set_detailed_logging(tmp_path, True)
|
||||
|
||||
@@ -2,6 +2,7 @@ import json
|
||||
import sqlite3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
@@ -14,6 +15,7 @@ from one_memory.imports.models import (
|
||||
WrittenBy,
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_tasks import evidence_ref, matches_evidence_ref
|
||||
from one_memory.workspace_resolution import KnownWorkspace
|
||||
|
||||
|
||||
@@ -25,6 +27,7 @@ def evidence(
|
||||
verbatim: str = "Remember the source, not the guess.",
|
||||
origin: EvidenceOrigin = EvidenceOrigin.FILE,
|
||||
written_by: WrittenBy = WrittenBy.PERSON,
|
||||
role: str = "user",
|
||||
source_ordinal: str | None = None,
|
||||
) -> EvidenceInput:
|
||||
return EvidenceInput(
|
||||
@@ -35,7 +38,7 @@ def evidence(
|
||||
source_ordinal=source_ordinal
|
||||
if source_ordinal is not None
|
||||
else ("line:1" if message_id is None and correlation_id is None else None),
|
||||
role="user",
|
||||
role=role,
|
||||
written_by=written_by,
|
||||
created_at=datetime(2026, 8, 31, 12, 0, tzinfo=UTC),
|
||||
verbatim=verbatim,
|
||||
@@ -87,7 +90,7 @@ def test_rescanning_an_unchanged_file_does_not_write_the_message_again(tmp_path)
|
||||
"""A rescan is idempotent because the identity is derived from the content.
|
||||
|
||||
`EvidenceInput.idempotency_key` is a hash of the source, conversation,
|
||||
author, ordinal and content hash, and `evidence_id` is a uuid5 over that
|
||||
protocol role and content hash, and `evidence_id` is a uuid5 over that
|
||||
key, so replaying an unchanged file reproduces the same primary key and
|
||||
`ON CONFLICT` absorbs it. Verified: removing the anchoring logic still
|
||||
leaves this green, so this pins the contract, not one implementation.
|
||||
@@ -102,6 +105,87 @@ def test_rescanning_an_unchanged_file_does_not_write_the_message_again(tmp_path)
|
||||
assert store.count() == 1
|
||||
|
||||
|
||||
def test_derived_identity_uses_role_not_host_author_guess(tmp_path) -> None:
|
||||
store = EvidenceStore(tmp_path / "one-memory.sqlite3")
|
||||
first = store.put(evidence(message_id=None, written_by=WrittenBy.UNKNOWN))
|
||||
replay = store.put(evidence(message_id=None, written_by=WrittenBy.PERSON))
|
||||
assistant = store.put(evidence(message_id=None, role="assistant", written_by=WrittenBy.AGENT))
|
||||
|
||||
assert replay.record.evidence_id == first.record.evidence_id
|
||||
assert replay.was_created is False
|
||||
assert assistant.record.evidence_id != first.record.evidence_id
|
||||
assert store.count() == 2
|
||||
|
||||
|
||||
def test_task_proof_ignores_legacy_written_by_metadata(tmp_path) -> None:
|
||||
store = EvidenceStore(tmp_path / "one-memory.sqlite3")
|
||||
record = store.put(evidence()).record
|
||||
proof = evidence_ref(record)
|
||||
|
||||
changed_author = record.model_copy(update={"written_by": WrittenBy.AGENT})
|
||||
legacy_identity = record.model_dump(
|
||||
mode="json",
|
||||
exclude={"origin_meta", "imported_at", "updated_at"},
|
||||
)
|
||||
legacy_proof = {
|
||||
**proof,
|
||||
"identity_hash": sha256(
|
||||
json.dumps(legacy_identity, sort_keys=True).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
|
||||
assert matches_evidence_ref(changed_author, proof) is True
|
||||
assert matches_evidence_ref(changed_author, legacy_proof) is True
|
||||
|
||||
|
||||
def test_rescan_reuses_a_row_stored_under_written_by_derived_v3(tmp_path) -> None:
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(database)
|
||||
item = evidence(message_id=None, written_by=WrittenBy.UNKNOWN)
|
||||
original = store.put(item).record
|
||||
with sqlite3.connect(database) as connection, connection:
|
||||
connection.execute(
|
||||
"UPDATE evidence SET idempotency_key=? WHERE evidence_id=?",
|
||||
(item.written_by_derived_v3_idempotency_key, original.evidence_id),
|
||||
)
|
||||
|
||||
replay = store.put(evidence(message_id=None, written_by=WrittenBy.PERSON))
|
||||
|
||||
assert replay.was_created is False
|
||||
assert replay.record.evidence_id == original.evidence_id
|
||||
assert store.count() == 1
|
||||
|
||||
|
||||
def test_legacy_identity_role_change_revises_the_existing_evidence(tmp_path) -> None:
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(database)
|
||||
old = evidence(
|
||||
message_id=None,
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
)
|
||||
original = store.put(old).record
|
||||
with sqlite3.connect(database) as connection, connection:
|
||||
connection.execute(
|
||||
"UPDATE evidence SET idempotency_key=? WHERE evidence_id=?",
|
||||
(old.written_by_derived_v3_idempotency_key, original.evidence_id),
|
||||
)
|
||||
|
||||
replay = store.put(
|
||||
evidence(
|
||||
message_id=None,
|
||||
role="user",
|
||||
written_by=WrittenBy.AGENT,
|
||||
)
|
||||
)
|
||||
|
||||
assert replay.record.evidence_id == original.evidence_id
|
||||
assert replay.was_revised is True
|
||||
assert replay.record.revision == original.revision + 1
|
||||
assert replay.record.role == "user"
|
||||
assert store.count() == 1
|
||||
|
||||
|
||||
def test_changed_text_at_one_position_is_a_new_message_not_a_silent_edit(tmp_path) -> None:
|
||||
"""An edited line becomes new evidence, never an overwrite of the old one.
|
||||
|
||||
@@ -146,6 +230,7 @@ def test_personal_candidate_prefilter_excludes_unrelated_user_turns(tmp_path) ->
|
||||
evidence(
|
||||
message_id="assistant",
|
||||
verbatim="我偏好简洁文案。",
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -310,6 +310,7 @@ def test_ollama_profile_ignores_ambient_remote_storage_and_models(
|
||||
monkeypatch.setenv("MEMORY_VECTOR_HOST", "remote.example.com")
|
||||
monkeypatch.setenv("MEMORY_COLLECTION_NAME", "hostile-collection")
|
||||
monkeypatch.setenv("MEMORY_EMBEDDING_DIMS", "99")
|
||||
monkeypatch.delenv("MEMORY_CODING_TOOL_RESULT_MAX_BYTES", raising=False)
|
||||
profile = snapshot(
|
||||
ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
@@ -333,6 +334,20 @@ def test_ollama_profile_ignores_ambient_remote_storage_and_models(
|
||||
assert config.vector_store.embedding_dims == 768
|
||||
assert config.graph_store.provider == "kuzu"
|
||||
assert str(tmp_path) in config.graph_store.db_path
|
||||
assert config.coding.tool_result_max_bytes == 512
|
||||
|
||||
|
||||
def test_memhub_coding_tool_result_ceiling_is_configurable(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("MEMORY_CODING_TOOL_RESULT_MAX_BYTES", "1024")
|
||||
profile = snapshot(
|
||||
ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
has_api_key=False,
|
||||
)
|
||||
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert config.coding.tool_result_max_bytes == 1024
|
||||
|
||||
|
||||
def _kuzu_graph_with_embedding_width(db_path: Path, width: int) -> None:
|
||||
@@ -1326,7 +1341,7 @@ def test_hy_runtime_matches_public_sdk_without_reranker_or_service_backends(
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert HY_MEMORY_DISTRIBUTION == "hy-memory"
|
||||
assert HY_MEMORY_VERSION == "2.0.3"
|
||||
assert HY_MEMORY_VERSION == "2.0.4"
|
||||
assert HY_MEMORY_MODE == "ultra"
|
||||
assert HY_MEMORY_READER == "hybrid"
|
||||
assert os.environ["MEMORY_MODE"] == "ultra"
|
||||
|
||||
@@ -5,6 +5,7 @@ Checking that contract automatically prevents a silently flipped rerank flag
|
||||
from going unnoticed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
@@ -16,6 +17,7 @@ from one_memory.hy_runtime_config import (
|
||||
HY_MEMORY_MODE,
|
||||
HY_MEMORY_READER,
|
||||
HY_MEMORY_VERSION,
|
||||
apply_diagnostic_log_level,
|
||||
hy_memory_version,
|
||||
)
|
||||
|
||||
@@ -69,3 +71,19 @@ def test_the_pin_matches_the_lock_file_and_its_wheel():
|
||||
)
|
||||
assert f"hy_memory-{HY_MEMORY_VERSION}-" in lock
|
||||
assert f'specifier = "=={HY_MEMORY_VERSION}"' in lock
|
||||
|
||||
|
||||
def test_info_level_suppresses_noisy_third_party_success_logs():
|
||||
names = ("", "one_memory", "httpx", "httpcore", "watchfiles", "jieba")
|
||||
previous = {name: logging.getLogger(name).level for name in names}
|
||||
try:
|
||||
apply_diagnostic_log_level("info")
|
||||
assert logging.getLogger("one_memory").level == logging.INFO
|
||||
for name in ("httpx", "httpcore", "watchfiles", "jieba"):
|
||||
assert logging.getLogger(name).level == logging.WARNING
|
||||
apply_diagnostic_log_level("debug")
|
||||
for name in ("one_memory", "httpx", "httpcore", "watchfiles", "jieba"):
|
||||
assert logging.getLogger(name).level == logging.DEBUG
|
||||
finally:
|
||||
for name, level in previous.items():
|
||||
logging.getLogger(name).setLevel(level)
|
||||
@@ -69,7 +69,7 @@ def test_no_idempotency_key_means_unknown_must_remain() -> None:
|
||||
|
||||
|
||||
def test_measured_sdk_verdict_is_recorded() -> None:
|
||||
"""Pin the measured answer for `hy-memory==2.0.3`.
|
||||
"""Pin the measured answer for `hy-memory==2.0.4`.
|
||||
|
||||
The finding: `add()` takes a `request_id` but nothing de-duplicates on it,
|
||||
while `MemoryInspector.get_pipeline_logs(request_id=...)` can look the
|
||||
|
||||
@@ -59,6 +59,35 @@ def test_bound_context_rejects_a_same_named_conversation_in_another_workspace(tm
|
||||
HyWriteBindings(HySourceContext(current=(current,), history=(other_workspace,)))
|
||||
|
||||
|
||||
def test_bound_context_accepts_a_user_turn_the_host_left_unattributed(tmp_path):
|
||||
"""A missing author is not a scope violation.
|
||||
|
||||
WorkBuddy records real user turns with `written_by=unknown` — 1078 of them
|
||||
across 1002 conversations on this machine. Refusing the write over that
|
||||
failed the whole conversation: the task ended `model_content_outcome_unknown`
|
||||
after two attempts, so a memory that may well have been written could never
|
||||
be confirmed, and the turn was never sedimented again.
|
||||
"""
|
||||
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
|
||||
|
||||
store = EvidenceStore(tmp_path / "evidence.db")
|
||||
current = store.put(
|
||||
EvidenceInput(
|
||||
source_id="workbuddy",
|
||||
conversation_id="atlas",
|
||||
message_id="now",
|
||||
role="user",
|
||||
written_by=WrittenBy.UNKNOWN,
|
||||
verbatim="帮我看下这个报错。",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
|
||||
bound = HyWriteBindings(HySourceContext(current=(current,)))
|
||||
|
||||
assert bound.sources[0]["role"] == "user"
|
||||
|
||||
|
||||
def test_bound_context_rejects_a_duplicated_evidence_id(tmp_path):
|
||||
"""The same Evidence must not be cited twice in one HY context.
|
||||
|
||||
@@ -219,18 +248,8 @@ def test_profile_rendering_keeps_per_field_evidence_without_dropping_native_prof
|
||||
assert bindings.profile_support("The user's name is 陈.", {"name": "陈"}) == ()
|
||||
|
||||
|
||||
def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
|
||||
"""The author must match the role: only user/person, assistant/agent, tool/tool.
|
||||
|
||||
The objective requires 作者 to be truthful. The other scope tests only ever
|
||||
build well-formed pairs, so this pins the pair check itself: an assistant
|
||||
turn attributed to a person, or a tool turn attributed to one, must be
|
||||
refused with `context_scope` rather than silently trusted.
|
||||
|
||||
`user` + AGENT is deliberately accepted since D98: the host parser marks a
|
||||
sub agent's question that way, and that turn is sent like any other. What
|
||||
is refused is the pair that cannot occur, not the one we now allow.
|
||||
"""
|
||||
def test_bound_context_uses_protocol_role_not_author_guess(tmp_path):
|
||||
"""Host author guesses never reject a valid user/assistant/tool message."""
|
||||
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
|
||||
|
||||
store = EvidenceStore(tmp_path / "evidence.db")
|
||||
@@ -239,8 +258,7 @@ def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
|
||||
|
||||
assistant = source(store, "reply", "assistant", "好的,继续推进。")
|
||||
wrong_author = assistant.model_copy(update={"written_by": WrittenBy.PERSON})
|
||||
with pytest.raises(ValueError, match="context_scope"):
|
||||
HyWriteBindings(HySourceContext(current=(wrong_author,)))
|
||||
HyWriteBindings(HySourceContext(current=(wrong_author,)))
|
||||
|
||||
# `source()` maps non-user roles to AGENT, so a tool turn needs its own build.
|
||||
tool = store.put(
|
||||
@@ -255,7 +273,10 @@ def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
assert HyWriteBindings(HySourceContext(current=(tool,))) # valid tool/tool pair
|
||||
assert HyWriteBindings(HySourceContext(current=(tool,)))
|
||||
wrong_tool = tool.model_copy(update={"written_by": WrittenBy.PERSON})
|
||||
HyWriteBindings(HySourceContext(current=(wrong_tool,)))
|
||||
|
||||
unsupported = assistant.model_copy(update={"role": "system"})
|
||||
with pytest.raises(ValueError, match="context_scope"):
|
||||
HyWriteBindings(HySourceContext(current=(wrong_tool,)))
|
||||
HyWriteBindings(HySourceContext(current=(unsupported,)))
|
||||
@@ -111,7 +111,7 @@ def test_import_metadata_changes_invalidate_without_authorizing_content(setup, c
|
||||
@pytest.mark.parametrize("cause", ["import", "maintenance"])
|
||||
@pytest.mark.parametrize(
|
||||
"column,value",
|
||||
[("verbatim", "changed raw body"), ("written_by", "agent"), ("role", "assistant")],
|
||||
[("verbatim", "changed raw body"), ("role", "assistant")],
|
||||
)
|
||||
def test_material_change_eligibility_depends_on_trusted_cause(setup, cause, column, value):
|
||||
db, store, _ = setup
|
||||
@@ -126,6 +126,17 @@ def test_material_change_eligibility_depends_on_trusted_cause(setup, cause, colu
|
||||
assert new.kind == ("revise" if cause == "import" else "metadata")
|
||||
|
||||
|
||||
def test_legacy_written_by_change_is_not_a_material_event(setup):
|
||||
db, store, _ = setup
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
key = put(store, db)
|
||||
journal.initialize(db)
|
||||
old = journal.current(db, key)
|
||||
db.execute("UPDATE evidence SET written_by='agent' WHERE evidence_id=?", (key,))
|
||||
|
||||
assert journal.record(db, key, cause="import") == old
|
||||
|
||||
|
||||
def test_complete_origin_metadata_changes_and_observation_noise(setup):
|
||||
db, store, _ = setup
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
@@ -1255,6 +1256,9 @@ def test_triggering_a_sync_leaves_the_loop_free_to_answer(tmp_path) -> None:
|
||||
|
||||
def test_live_sync_diagnostics_only_emit_state_changes(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
supervisor = PollingLiveSyncSupervisor(
|
||||
@@ -1267,5 +1271,7 @@ def test_live_sync_diagnostics_only_emit_state_changes(tmp_path, monkeypatch):
|
||||
supervisor._set_health("codex", unavailable.model_copy(update={"source_id": "codex"}))
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["event"] for event in events] == ["live_sync.state_changed"]
|
||||
assert events[0]["level"] == "warn"
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
@@ -87,6 +88,9 @@ def _segment_at_the_legacy_window(monkeypatch):
|
||||
|
||||
def test_stale_known_reason_diagnostics_never_fail_dispatch(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
_report_stale_known_reasons(
|
||||
@@ -95,6 +99,7 @@ def test_stale_known_reason_diagnostics_never_fail_dispatch(tmp_path, monkeypatc
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
|
||||
records = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [(row["details"]["reason"], row["details"]["count"]) for row in records] == [
|
||||
@@ -306,16 +311,23 @@ def test_memory_task_logs_six_step_durations_without_content(tmp_path) -> None:
|
||||
content=marker,
|
||||
)
|
||||
task = _task(tasks, profile, record)
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
asyncio.run(_runner(tmp_path, stores, FakeBridge()).run(task.task_id))
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
written = receipt.path.read_text()
|
||||
events = [json.loads(line) for line in written.splitlines()]
|
||||
started = next(event for event in events if event["event"] == "memory_task.started")
|
||||
assert started["level"] == "info"
|
||||
steps = [event for event in events if event["event"] == "memory_task.step_completed"]
|
||||
assert [event["details"]["phase"] for event in steps] == ["1", "2", "3", "4", "5", "6"]
|
||||
finished = next(event for event in events if event["event"] == "memory_task.finished")
|
||||
assert finished["level"] == "info"
|
||||
assert finished["details"] == {
|
||||
"attempt": 1,
|
||||
"pending_count": 0,
|
||||
@@ -1390,6 +1402,9 @@ def test_backfill_scan_logs_inventory_and_queue_without_content(tmp_path) -> Non
|
||||
conversation_id=f"conversation-{index}",
|
||||
content_kind=EvidenceContentKind.MEMORY_CONTENT,
|
||||
)
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path / "logs")
|
||||
|
||||
async def exercise() -> None:
|
||||
@@ -1409,6 +1424,7 @@ def test_backfill_scan_logs_inventory_and_queue_without_content(tmp_path) -> Non
|
||||
asyncio.run(exercise())
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
|
||||
written = receipt.path.read_text()
|
||||
events = [json.loads(line) for line in written.splitlines()]
|
||||
@@ -1764,7 +1780,7 @@ def test_process_records_bounds_a_hung_model_call_by_the_profile_timeout(
|
||||
with sqlite3.connect(tmp_path / "one-memory.sqlite3") as db:
|
||||
db.executescript(
|
||||
"""
|
||||
CREATE TABLE source_grants (
|
||||
CREATE TABLE IF NOT EXISTS source_grants (
|
||||
user_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL,
|
||||
@@ -2022,6 +2038,126 @@ def test_oversized_single_turn_segments_are_each_processed(tmp_path) -> None:
|
||||
assert len(governance.list(user_id="local")) == 2
|
||||
|
||||
|
||||
def test_direct_processing_uses_the_supplied_frozen_revision(tmp_path) -> None:
|
||||
stores = _setup(tmp_path)
|
||||
evidence, profiles, tasks, governance, secrets, _profile = stores
|
||||
old_user = _put(
|
||||
evidence,
|
||||
message_id="rewound-user",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="old branch",
|
||||
)
|
||||
reply = _put(
|
||||
evidence,
|
||||
message_id="rewound-reply",
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
content="old reply",
|
||||
)
|
||||
revised = _put(
|
||||
evidence,
|
||||
message_id="rewound-user",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="new branch",
|
||||
)
|
||||
assert revised.evidence_id == old_user.evidence_id
|
||||
assert revised.revision == old_user.revision + 1
|
||||
with sqlite3.connect(tasks.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE source_grants (
|
||||
user_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL,
|
||||
requested_path TEXT NOT NULL,
|
||||
root_identity TEXT NOT NULL,
|
||||
candidate_kind TEXT NOT NULL,
|
||||
granted_at TEXT NOT NULL,
|
||||
revoked_at TEXT,
|
||||
revision INTEGER NOT NULL,
|
||||
PRIMARY KEY (user_id, source_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO source_grants (
|
||||
user_id,source_id,root_path,requested_path,root_identity,
|
||||
candidate_kind,granted_at,revoked_at,revision
|
||||
) VALUES (?,?,?,?,?,?,?,NULL,1)
|
||||
""",
|
||||
(
|
||||
"local",
|
||||
"cursor",
|
||||
"/authorized",
|
||||
"/authorized",
|
||||
"test-root",
|
||||
"default",
|
||||
datetime.now(UTC).isoformat(),
|
||||
),
|
||||
)
|
||||
bridge = FakeBridge(outputs=())
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=tasks,
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=evidence,
|
||||
governance=governance,
|
||||
data_dir=tmp_path,
|
||||
runner_factory=lambda: _runner(tmp_path, stores, bridge),
|
||||
)
|
||||
|
||||
completed = asyncio.run(supervisor.process_records((old_user, reply)))
|
||||
|
||||
assert len(completed) == 1
|
||||
assert [turn.content for turn in bridge.requests[0].turns] == [
|
||||
"old branch",
|
||||
"old reply",
|
||||
]
|
||||
|
||||
|
||||
def test_frozen_current_still_revalidates_history_after_the_model_call(tmp_path) -> None:
|
||||
from one_memory.hy_source_context import HySourceContext
|
||||
|
||||
stores = _setup(tmp_path)
|
||||
evidence, _profiles, tasks, _governance, _secrets, _profile = stores
|
||||
current = _put(
|
||||
evidence,
|
||||
message_id="frozen-current",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="old current",
|
||||
)
|
||||
history = _put(
|
||||
evidence,
|
||||
message_id="frozen-history",
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
content="stable history",
|
||||
)
|
||||
_put(
|
||||
evidence,
|
||||
message_id="frozen-current",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="new current",
|
||||
)
|
||||
runner = _runner(tmp_path, stores, FakeBridge(outputs=()))
|
||||
context = HySourceContext(current=(current,), history=(history,))
|
||||
|
||||
runner._check_source_context(context, skip_current=True)
|
||||
with sqlite3.connect(tasks.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"DELETE FROM evidence WHERE evidence_id=?",
|
||||
(history.evidence_id,),
|
||||
)
|
||||
|
||||
with pytest.raises(HyMemoryBridgeError, match="hy_memory_evidence_changed"):
|
||||
runner._check_source_context(context, skip_current=True)
|
||||
|
||||
|
||||
def test_oversized_assistant_segments_keep_the_question_as_context(tmp_path) -> None:
|
||||
evidence = EvidenceStore(tmp_path / "one-memory.sqlite3")
|
||||
question = _put(
|
||||
@@ -2378,6 +2514,105 @@ def test_unknown_task_is_reconciled_from_exact_hy_output_without_model_retry(tmp
|
||||
assert bridge.closed is True
|
||||
|
||||
|
||||
def test_an_unknown_task_without_inputs_does_not_end_the_reconcile_pass(tmp_path) -> None:
|
||||
"""One task with nothing to send must not strand the other unknown tasks.
|
||||
|
||||
An unknown task whose inputs are gone yields an empty chunk, and building a
|
||||
request from that reached `chunk[0]` and came out as IndexError. A
|
||||
reconcile pass skips a task that raises a domain error, so IndexError did
|
||||
not skip anything -- it ended the pass. On this machine 68 of 303 unknown
|
||||
tasks had no inputs, and while the retry backed off to 128 and gave up, the
|
||||
235 that could have been reconciled stayed unknown with them.
|
||||
"""
|
||||
stores = _setup(tmp_path)
|
||||
evidence, profiles, tasks, governance, secrets, profile = stores
|
||||
stranded = _put(
|
||||
evidence,
|
||||
message_id="reconcile-without-inputs",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="这条任务的输入已经不在了。",
|
||||
)
|
||||
record = _put(
|
||||
evidence,
|
||||
message_id="reconcile-with-inputs",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="用户偏好每周日上午攀岩。",
|
||||
)
|
||||
empty = tasks.create(
|
||||
request_id="reconcile-empty-request",
|
||||
user_id=stranded.user_id,
|
||||
kind=MemoryTaskKind.EXTRACT,
|
||||
input_ref=stranded.evidence_id,
|
||||
profile=profile,
|
||||
input_records=(stranded,),
|
||||
)
|
||||
task = tasks.create(
|
||||
request_id="reconcile-kept-request",
|
||||
user_id=record.user_id,
|
||||
kind=MemoryTaskKind.EXTRACT,
|
||||
input_ref=record.evidence_id,
|
||||
profile=profile,
|
||||
input_records=(record,),
|
||||
)
|
||||
with sqlite3.connect(tasks.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE source_grants (
|
||||
user_id TEXT NOT NULL, source_id TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL, candidate_kind TEXT NOT NULL,
|
||||
granted_at TEXT NOT NULL, revoked_at TEXT,
|
||||
PRIMARY KEY (user_id, source_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO source_grants VALUES (?,?,?,?,?,NULL)",
|
||||
(
|
||||
record.user_id,
|
||||
record.source_id,
|
||||
"/authorized",
|
||||
"default",
|
||||
datetime.now(UTC).isoformat(),
|
||||
),
|
||||
)
|
||||
for task_id in (empty.task_id, task.task_id):
|
||||
connection.execute(
|
||||
"UPDATE memory_tasks SET status='unknown', last_error_code=? WHERE task_id=?",
|
||||
("model_content_outcome_unknown", task_id),
|
||||
)
|
||||
# What the field data actually looks like: the task is there, its
|
||||
# inputs are not.
|
||||
connection.execute("DELETE FROM memory_task_inputs WHERE task_id=?", (empty.task_id,))
|
||||
bridge = FakeBridge(
|
||||
outputs=(
|
||||
HyMemoryOutput(
|
||||
memory_id="hy-recovered",
|
||||
content="用户偏好每周日上午攀岩。",
|
||||
layer="L2_FACT",
|
||||
owner="user",
|
||||
evidence_ids=(record.evidence_id,),
|
||||
),
|
||||
)
|
||||
)
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=tasks,
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=evidence,
|
||||
governance=governance,
|
||||
data_dir=tmp_path,
|
||||
runner_factory=lambda: _runner(tmp_path, stores, FakeBridge()),
|
||||
bridge_factory=lambda _profile, _secret, _data_dir: bridge,
|
||||
)
|
||||
|
||||
# The task with inputs is reconciled; the one without is passed over.
|
||||
assert asyncio.run(supervisor.reconcile_unknown_outputs()) == 1
|
||||
assert tasks.require(task.task_id).status is MemoryTaskStatus.COMPLETED
|
||||
assert [memory.memory_id for memory in governance.list(user_id="local")] == ["hy-recovered"]
|
||||
|
||||
|
||||
def test_two_workspaces_with_one_conversation_id_prepare_apart(tmp_path) -> None:
|
||||
"""Cursor can reuse a conversation id across projects.
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ def test_sync_config_defaults_off_and_survives_restart(tmp_path):
|
||||
saved = json.loads((tmp_path / "config.json").read_text())
|
||||
assert saved == {
|
||||
"schema_version": 1,
|
||||
"sedimentation_enabled": True,
|
||||
"sources": {"cursor": {"sync_enabled": True}},
|
||||
}
|
||||
assert ConfigStore(tmp_path).sync_enabled("cursor")
|
||||
@@ -46,6 +47,7 @@ def test_legacy_model_content_permission_is_ignored_on_upgrade(tmp_path):
|
||||
store.set_sync("codex", True)
|
||||
assert json.loads((tmp_path / "config.json").read_text()) == {
|
||||
"schema_version": 1,
|
||||
"sedimentation_enabled": True,
|
||||
"sources": {
|
||||
"codex": {"sync_enabled": True},
|
||||
"cursor": {"sync_enabled": True},
|
||||
@@ -57,3 +59,35 @@ def test_invalid_config_fails_loudly_instead_of_silently_disabling_sources(tmp_p
|
||||
(tmp_path / "config.json").write_text('{"schema_version":1,"sources":{"cursor":{}}')
|
||||
with pytest.raises(ConfigFileError, match="config_invalid"):
|
||||
ConfigStore(tmp_path)
|
||||
|
||||
|
||||
def test_sedimentation_defaults_on_and_survives_restart(tmp_path):
|
||||
assert ConfigStore(tmp_path).sedimentation_enabled()
|
||||
|
||||
store = ConfigStore(tmp_path)
|
||||
store.set_sedimentation_enabled(False)
|
||||
assert not store.sedimentation_enabled()
|
||||
|
||||
# The model configuration is a separate store, so stopping must not touch
|
||||
# it: starting again has to be a switch, not a re-entry of a key.
|
||||
reopened = ConfigStore(tmp_path)
|
||||
assert not reopened.sedimentation_enabled()
|
||||
reopened.set_sedimentation_enabled(True)
|
||||
assert ConfigStore(tmp_path).sedimentation_enabled()
|
||||
|
||||
|
||||
def test_stopping_sedimentation_keeps_source_sync_config(tmp_path):
|
||||
store = ConfigStore(tmp_path)
|
||||
store.set_sync("cursor", True)
|
||||
store.set_sedimentation_enabled(False)
|
||||
|
||||
reopened = ConfigStore(tmp_path)
|
||||
assert not reopened.sedimentation_enabled()
|
||||
assert reopened.sync_enabled("cursor")
|
||||
|
||||
|
||||
def test_sedimentation_rejects_a_non_boolean(tmp_path):
|
||||
store = ConfigStore(tmp_path)
|
||||
with pytest.raises(ValueError, match="invalid_sedimentation_config"):
|
||||
store.set_sedimentation_enabled("yes") # type: ignore[arg-type]
|
||||
assert store.sedimentation_enabled()
|
||||
@@ -125,8 +125,7 @@ def test_write_job_state_covers_unknown_and_terminal() -> None:
|
||||
} == {state for state in WriteJobState if not state.is_terminal}
|
||||
|
||||
|
||||
def test_canonical_message_requires_author_and_ordinal() -> None:
|
||||
"""A message without a known author or position cannot be ordered later."""
|
||||
def test_canonical_message_requires_a_nonnegative_ordinal() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
CanonicalMessage(
|
||||
harness_id="cursor",
|
||||
@@ -134,6 +133,5 @@ def test_canonical_message_requires_author_and_ordinal() -> None:
|
||||
message_id=None,
|
||||
ordinal=-1,
|
||||
role="user",
|
||||
written_by="person",
|
||||
content="hello",
|
||||
)
|
||||
@@ -10,10 +10,12 @@ import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridgeError
|
||||
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
|
||||
from one_memory.memory_server.durable_queue import (
|
||||
CANCELLED,
|
||||
@@ -37,14 +39,18 @@ def _request(
|
||||
evidence: tuple[str, ...] = ("e1",),
|
||||
key: str = "idem-1",
|
||||
source: str | None = None,
|
||||
session: str | None = None,
|
||||
event_at: datetime | None = None,
|
||||
) -> WriteRequest:
|
||||
batch = BatchIdentity(internal_session_id="sess-1", first_round=1, last_round=5)
|
||||
session_id = session or f"sess-{source or '1'}"
|
||||
batch = BatchIdentity(internal_session_id=session_id, first_round=1, last_round=5)
|
||||
return WriteRequest(
|
||||
internal_session_id="sess-1",
|
||||
internal_session_id=session_id,
|
||||
batch_identity=batch,
|
||||
evidence_ids=list(evidence),
|
||||
idempotency_key=key,
|
||||
source_id=source,
|
||||
event_at=event_at,
|
||||
)
|
||||
|
||||
|
||||
@@ -54,11 +60,19 @@ def _rows(path):
|
||||
return [dict(row) for row in db.execute("SELECT * FROM memory_write_jobs")]
|
||||
|
||||
|
||||
def test_acceptance_is_written_to_the_business_database(tmp_path) -> None:
|
||||
def test_acceptance_is_written_to_the_business_database(tmp_path, monkeypatch) -> None:
|
||||
from one_memory.memory_server import durable_queue
|
||||
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
durable_queue,
|
||||
"diagnostic_event",
|
||||
lambda **event: events.append(event),
|
||||
)
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
|
||||
record = queue.submit(_request())
|
||||
record = queue.submit(_request(source="cursor"))
|
||||
|
||||
rows = _rows(path)
|
||||
assert len(rows) == 1
|
||||
@@ -67,15 +81,24 @@ def test_acceptance_is_written_to_the_business_database(tmp_path) -> None:
|
||||
assert rows[0]["state"] == "pending"
|
||||
assert rows[0]["hy_invoked"] == 0
|
||||
assert record.enqueued is True
|
||||
assert events == [
|
||||
{
|
||||
"level": "info",
|
||||
"component": "memory.write_queue",
|
||||
"event": "memory_write_job.queued",
|
||||
"outcome": "queued",
|
||||
"job_id": record.job_id,
|
||||
"details": {
|
||||
"source_id": "cursor",
|
||||
"mode": "normal",
|
||||
"count": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_the_stored_payload_is_references_only(tmp_path) -> None:
|
||||
"""The job payload must point at Evidence, never carry its text.
|
||||
|
||||
A queue row is durable and much longer-lived than the API call that made
|
||||
it, so storing verbatim content here would be a second copy of the user's
|
||||
private text. The payload is the request's identifiers only.
|
||||
"""
|
||||
def test_an_isolated_legacy_queue_keeps_the_empty_snapshot_compatibility_shape(tmp_path) -> None:
|
||||
"""Queues without an Evidence authority exist only for legacy/unit compatibility."""
|
||||
import json
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
@@ -87,10 +110,12 @@ def test_the_stored_payload_is_references_only(tmp_path) -> None:
|
||||
"internal_session_id",
|
||||
"batch_identity",
|
||||
"evidence_ids",
|
||||
"input_records",
|
||||
"idempotency_key",
|
||||
"priority",
|
||||
"source_id",
|
||||
}
|
||||
assert payload["input_records"] == []
|
||||
assert payload["evidence_ids"] == ["ev-1", "ev-2"]
|
||||
|
||||
|
||||
@@ -127,6 +152,7 @@ def test_submit_accepts_a_write_whose_evidence_exists(tmp_path) -> None:
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
occurred_at = datetime(2026, 10, 17, 23, 45, tzinfo=UTC)
|
||||
created = store.put(
|
||||
EvidenceInput(
|
||||
source_id="test",
|
||||
@@ -134,6 +160,7 @@ def test_submit_accepts_a_write_whose_evidence_exists(tmp_path) -> None:
|
||||
message_id="m1",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=occurred_at,
|
||||
verbatim="present evidence",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
@@ -144,6 +171,8 @@ def test_submit_accepts_a_write_whose_evidence_exists(tmp_path) -> None:
|
||||
|
||||
assert record.enqueued is True
|
||||
assert len(_rows(path)) == 1
|
||||
assert datetime.fromisoformat(_rows(path)[0]["event_at"]) == occurred_at
|
||||
assert _rows(path)[0]["time_quality"] == "exact"
|
||||
|
||||
|
||||
def test_a_replay_is_idempotent_even_after_its_evidence_is_removed(tmp_path) -> None:
|
||||
@@ -187,20 +216,17 @@ def test_a_replay_is_idempotent_even_after_its_evidence_is_removed(tmp_path) ->
|
||||
assert replay.enqueued is False
|
||||
assert len(_rows(path)) == 1
|
||||
|
||||
|
||||
def test_a_settled_job_replays_as_a_hit_not_a_conflict(tmp_path) -> None:
|
||||
"""Replaying a completed key must return the job, never raise.
|
||||
|
||||
Real trigger: 236 startup replays of one key, all
|
||||
`ValueError: idempotency key ... already exists with different content`.
|
||||
The stored payload had been reaped to a digest sentinel, and the caller's
|
||||
key is derived from the evidence's revision, which can move on between
|
||||
replays while the payload's evidence ids stay put. Refusing that pair
|
||||
turned a normal startup replay into an exception every time.
|
||||
"""
|
||||
"""A historical reaped payload matches only its exact pre-snapshot intent."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
first = queue.submit(_request(evidence=("e1",), key="idem-settled"))
|
||||
request = _request(evidence=("e1",), key="idem-settled")
|
||||
first = queue.submit(request)
|
||||
legacy = request.model_dump(
|
||||
mode="json",
|
||||
exclude={"event_at", "time_quality", "input_records"},
|
||||
)
|
||||
digest = sha256(json.dumps(legacy, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
# Settle it the way the drain does: terminal state, HY invoked, and the
|
||||
# payload reaped down to its digest so no body is kept.
|
||||
@@ -209,19 +235,40 @@ def test_a_settled_job_replays_as_a_hit_not_a_conflict(tmp_path) -> None:
|
||||
"UPDATE memory_write_jobs SET state=?, hy_invoked=1, payload_json=? WHERE job_id=?",
|
||||
(
|
||||
COMPLETED,
|
||||
json.dumps({"_one_memory_reaped_payload": "0" * 64}),
|
||||
json.dumps({"_one_memory_reaped_payload": digest}),
|
||||
first.job_id,
|
||||
),
|
||||
)
|
||||
|
||||
# A payload that cannot match the reaped digest, under the same key.
|
||||
replay = queue.submit(_request(evidence=("e1",), key="idem-settled"))
|
||||
replay = queue.submit(request)
|
||||
|
||||
assert replay.job_id == first.job_id
|
||||
assert replay.state == COMPLETED
|
||||
assert replay.enqueued is False
|
||||
assert len(_rows(path)) == 1
|
||||
|
||||
with pytest.raises(ValueError, match="different content"):
|
||||
queue.submit(
|
||||
_request(
|
||||
evidence=("e1",),
|
||||
key="idem-settled",
|
||||
source="cursor",
|
||||
session="different-session",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_a_real_completed_job_still_rejects_the_same_key_with_different_input(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
first = queue.submit(_request(evidence=("e1",), key="completed-conflict"))
|
||||
queue.settle(first.job_id, state=COMPLETED, hy_invoked=True)
|
||||
|
||||
with pytest.raises(ValueError, match="different content"):
|
||||
queue.submit(_request(evidence=("e2",), key="completed-conflict"))
|
||||
|
||||
|
||||
def test_a_killed_worker_leaves_a_possibly_sent_job_unknown_after_restart(tmp_path) -> None:
|
||||
"""A real process death after HY was invoked must not become retryable.
|
||||
@@ -462,6 +509,8 @@ def test_an_older_queue_backfills_evidence_refs_from_the_intact_payload(tmp_path
|
||||
queue = DurableWriteQueue(path)
|
||||
|
||||
assert _rows(path)[0]["evidence_ids_json"] == '["legacy-evidence"]'
|
||||
assert _rows(path)[0]["event_at"] == "2026-09-01T00:00:00+00:00"
|
||||
assert _rows(path)[0]["time_quality"] == "approximate"
|
||||
assert queue.unsubmitted_evidence_ids() == ()
|
||||
|
||||
|
||||
@@ -1113,10 +1162,13 @@ def test_unresolved_unknown_is_automatically_retried_only_once(tmp_path) -> None
|
||||
first_retry = queue.get(record.request_id)
|
||||
assert first_retry.state == PENDING
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0] == 1
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0]
|
||||
== 1
|
||||
)
|
||||
|
||||
drain = DurableWriteDrain(queue, execute=_never_execute, owner="retry-owner")
|
||||
assert drain.claim(limit=1) == (record.job_id,)
|
||||
@@ -1130,10 +1182,13 @@ def test_unresolved_unknown_is_automatically_retried_only_once(tmp_path) -> None
|
||||
)
|
||||
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0] == 2
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0]
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
queue.retry_first_unknown_after_reconcile(
|
||||
older_than=datetime.now(UTC) + timedelta(minutes=1)
|
||||
@@ -2510,21 +2565,16 @@ def test_settle_with_an_owner_does_not_touch_an_ownerless_row(tmp_path, caplog)
|
||||
assert not any("durable_write_settle_missed_row" in message for message in messages), messages
|
||||
|
||||
|
||||
def test_terminal_payloads_are_reaped_but_replay_still_detects_conflicts(tmp_path) -> None:
|
||||
"""Payload cleanup must not weaken idempotency.
|
||||
|
||||
Once a job is terminal the serialized request is dead weight, so it is
|
||||
replaced by a digest. Replaying the same intent must still resolve to the
|
||||
same row, and the same key with different content must still be refused.
|
||||
"""
|
||||
def test_terminal_payloads_are_retained_and_replay_still_detects_conflicts(tmp_path) -> None:
|
||||
"""Terminal input is immutable audit data and idempotency remains strict."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(evidence=("e1",)))
|
||||
queue.settle(record.job_id, state="completed")
|
||||
|
||||
assert queue.clear_terminal_payloads() == 1
|
||||
assert queue.clear_terminal_payloads() == 0
|
||||
stored = _rows(path)[0]["payload_json"]
|
||||
assert "e1" not in stored and "_one_memory_reaped_payload" in stored
|
||||
assert "e1" in stored and "_one_memory_reaped_payload" not in stored
|
||||
assert _rows(path)[0]["evidence_ids_json"] == '["e1"]'
|
||||
|
||||
# Same intent still resolves to the existing row, not a second write.
|
||||
@@ -2538,15 +2588,15 @@ def test_terminal_payloads_are_reaped_but_replay_still_detects_conflicts(tmp_pat
|
||||
queue.submit(_request(evidence=("e2",)))
|
||||
|
||||
|
||||
def test_terminal_payload_reaping_is_bounded_and_idempotent(tmp_path) -> None:
|
||||
def test_terminal_payload_cleanup_is_a_compatibility_noop(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
for index in range(3):
|
||||
record = queue.submit(_request(evidence=("e1",), key=f"idem-{index}"))
|
||||
queue.settle(record.job_id, state="dead")
|
||||
|
||||
assert queue.clear_terminal_payloads(limit=2) == 2
|
||||
assert queue.clear_terminal_payloads(limit=2) == 1
|
||||
assert queue.clear_terminal_payloads(limit=2) == 0
|
||||
assert queue.clear_terminal_payloads(limit=2) == 0
|
||||
assert queue.clear_terminal_payloads(limit=2) == 0
|
||||
|
||||
|
||||
@@ -2565,8 +2615,8 @@ def test_unknown_and_pending_payloads_are_never_reaped(tmp_path) -> None:
|
||||
assert "e2" in payloads[unknown.request_id]
|
||||
|
||||
|
||||
def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_path) -> None:
|
||||
"""Payload cleanup must not make a completed Evidence look unqueued.
|
||||
def test_retained_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_path) -> None:
|
||||
"""A completed Evidence remains queued while its full input is retained.
|
||||
|
||||
The reconcile primitive scans the durable queue by Evidence id. If it
|
||||
derived that set from `payload_json`, reaping a terminal row would erase
|
||||
@@ -2600,7 +2650,7 @@ def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(evidence=(evidence.evidence_id,), key="idem-reaped"))
|
||||
queue.settle(record.job_id, state="completed")
|
||||
assert queue.clear_terminal_payloads() == 1
|
||||
assert queue.clear_terminal_payloads() == 0
|
||||
assert queue.unsubmitted_evidence_ids() == ()
|
||||
|
||||
orphan = store.put(
|
||||
@@ -2618,19 +2668,17 @@ def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_
|
||||
assert queue.unsubmitted_evidence_ids() == (orphan.evidence_id,)
|
||||
|
||||
|
||||
def test_payload_for_a_reaped_terminal_row_returns_none(tmp_path) -> None:
|
||||
"""A stale claim of a reaped row must not resurrect a dead intent."""
|
||||
def test_payload_for_a_terminal_row_keeps_the_immutable_input(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request())
|
||||
queue.settle(record.job_id, state="completed")
|
||||
queue.clear_terminal_payloads()
|
||||
|
||||
assert queue.payload_for(record.job_id) is None
|
||||
assert queue.payload_for(record.job_id) == _request()
|
||||
|
||||
|
||||
def test_drain_reaps_terminal_payloads_when_idle(tmp_path) -> None:
|
||||
"""The reaper must actually run in the worker, not only in tests."""
|
||||
def test_drain_retains_terminal_payloads_when_idle(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request())
|
||||
@@ -2655,19 +2703,11 @@ def test_drain_reaps_terminal_payloads_when_idle(tmp_path) -> None:
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert executed == []
|
||||
assert "_one_memory_reaped_payload" in _rows(path)[0]["payload_json"]
|
||||
assert "_one_memory_reaped_payload" not in _rows(path)[0]["payload_json"]
|
||||
|
||||
|
||||
def test_the_drain_reaps_terminal_payloads_even_while_never_idle(tmp_path) -> None:
|
||||
"""Terminal payloads must be reaped even if the queue never goes idle.
|
||||
|
||||
Regression: `clear_terminal_payloads` ran only on the `executed == 0` idle
|
||||
branch, so a sustained backlog kept every finished row's full serialized
|
||||
request body (which holds the written evidence references) in the business
|
||||
database indefinitely — a disk/privacy-retention invariant starved exactly
|
||||
when the queue is busiest. Reaping is a bounded maintenance write and must
|
||||
be gated by its own interval, not by idleness.
|
||||
"""
|
||||
def test_the_drain_retains_terminal_payloads_while_never_idle(tmp_path) -> None:
|
||||
"""A sustained backlog must not mutate an earlier job's input snapshot."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
finished = queue.submit(_request(key="finished"))
|
||||
@@ -2695,9 +2735,8 @@ def test_the_drain_reaps_terminal_payloads_even_while_never_idle(tmp_path) -> No
|
||||
|
||||
payload = _rows(path)
|
||||
by_key = {row["idempotency_key"]: row["payload_json"] for row in payload}
|
||||
assert "_one_memory_reaped_payload" in by_key["finished"], (
|
||||
"a completed row's payload must be reaped even under a standing backlog"
|
||||
)
|
||||
assert "_one_memory_reaped_payload" not in by_key["finished"]
|
||||
assert '"e1"' in by_key["finished"]
|
||||
|
||||
|
||||
def test_the_drain_readmits_model_waiters_even_while_never_idle(tmp_path) -> None:
|
||||
@@ -3146,8 +3185,8 @@ def test_claim_budget_ab_against_a_held_writer(tmp_path) -> None:
|
||||
def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> None:
|
||||
"""An empty queue must not issue a write per idle poll.
|
||||
|
||||
`clear_terminal_payloads` and `readmit_model_waiters` both write. On an
|
||||
empty queue the loop wakes every `idle_pause`; running them each time would
|
||||
`readmit_model_waiters` writes. On an empty queue the loop wakes every
|
||||
`idle_pause`; running it each time would
|
||||
contend with imports for the single writer for no benefit. They share a
|
||||
coarse interval gate, so a 20-poll window performs at most a couple of
|
||||
writes.
|
||||
@@ -3156,22 +3195,14 @@ def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> No
|
||||
queue = DurableWriteQueue(path)
|
||||
drain = DurableWriteDrain(queue, execute=_never_execute)
|
||||
|
||||
reaps = 0
|
||||
readmits = 0
|
||||
real_reap = queue.clear_terminal_payloads
|
||||
real_readmit = queue.readmit_model_waiters
|
||||
|
||||
def counted_reap(*args, **kwargs):
|
||||
nonlocal reaps
|
||||
reaps += 1
|
||||
return real_reap(*args, **kwargs)
|
||||
|
||||
def counted_readmit(*args, **kwargs):
|
||||
nonlocal readmits
|
||||
readmits += 1
|
||||
return real_readmit(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(queue, "clear_terminal_payloads", counted_reap)
|
||||
monkeypatch.setattr(queue, "readmit_model_waiters", counted_readmit)
|
||||
monkeypatch.setattr(drain, "claim", lambda **_kw: ())
|
||||
|
||||
@@ -3185,7 +3216,6 @@ def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> No
|
||||
|
||||
# Well over 20 polls happened; maintenance ran at most once (the interval
|
||||
# gate), not once per poll.
|
||||
assert reaps <= 1
|
||||
assert readmits <= 1
|
||||
|
||||
|
||||
@@ -3316,7 +3346,7 @@ def test_idle_maintenance_database_failure_is_sampled_without_database_error(
|
||||
def locked(*_args, **_kwargs):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
monkeypatch.setattr(drain.queue, "clear_terminal_payloads", locked)
|
||||
monkeypatch.setattr(drain.queue, "readmit_model_waiters", locked)
|
||||
monkeypatch.setattr(drain, "claim", lambda **_kw: ())
|
||||
|
||||
async def scenario() -> None:
|
||||
@@ -3332,8 +3362,8 @@ def test_idle_maintenance_database_failure_is_sampled_without_database_error(
|
||||
def programming_error(*_args, **_kwargs):
|
||||
raise RuntimeError("maintenance bug")
|
||||
|
||||
monkeypatch.setattr(drain.queue, "clear_terminal_payloads", programming_error)
|
||||
drain._last_terminal_reap = None # noqa: SLF001 - force the next maintenance tick
|
||||
monkeypatch.setattr(drain.queue, "readmit_model_waiters", programming_error)
|
||||
drain._last_model_readmit = None # noqa: SLF001 - force the next maintenance tick
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
@@ -4187,6 +4217,42 @@ def test_disabled_source_is_not_claimed_until_reenabled(tmp_path) -> None:
|
||||
assert drain.claim(limit=1) == (record.job_id,)
|
||||
|
||||
|
||||
def test_a_source_admitted_only_by_grant_is_claimed(tmp_path) -> None:
|
||||
"""The claim gate must ask the question the write gate asks.
|
||||
|
||||
`one-memory-manual` is not a connected app: it never appears in the
|
||||
configuration document, so a per-app toggle answers False for it while the
|
||||
write gate admits it by its grant. Wired as the drain's claim gate, that
|
||||
second answer skipped the job on every tick — a saved manual note stayed
|
||||
pending with no task behind it and never reached the model. Admission is
|
||||
one decision, and it has one predicate.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="manual-note", source="one-memory-manual"))
|
||||
# What `sync_enabled` answers for this source: no entry, therefore not on.
|
||||
per_app_toggle: dict[str, bool] = {}
|
||||
granted = {"one-memory-manual"}
|
||||
|
||||
drained = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
source_writes_permitted=lambda source_id: source_id in granted,
|
||||
source_claim_enabled=lambda source_id: source_id in granted,
|
||||
)
|
||||
assert drained.claim(limit=1) == (record.job_id,)
|
||||
|
||||
# The wiring that shipped: admitted for writes, refused at claim.
|
||||
queue.settle(record.job_id, state=PENDING, owner=None)
|
||||
stranded = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
source_writes_permitted=lambda source_id: source_id in granted,
|
||||
source_claim_enabled=lambda source_id: per_app_toggle.get(source_id, False),
|
||||
)
|
||||
assert stranded.claim(limit=1) == ()
|
||||
|
||||
|
||||
def test_pause_before_hy_releases_an_already_claimed_job(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
@@ -4210,9 +4276,130 @@ def test_pause_before_hy_releases_an_already_claimed_job(tmp_path) -> None:
|
||||
assert current.state == PENDING
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
"SELECT attempts,hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0] == 0
|
||||
).fetchone() == (0, 0)
|
||||
|
||||
|
||||
def test_event_time_window_claims_cross_harness_jobs_within_ten_minutes(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
first = queue.submit(_request(key="window-a", source="cursor", event_at=anchor))
|
||||
inside = queue.submit(
|
||||
_request(
|
||||
key="window-b",
|
||||
source="codex",
|
||||
event_at=anchor + timedelta(minutes=9, seconds=59),
|
||||
)
|
||||
)
|
||||
boundary = queue.submit(
|
||||
_request(
|
||||
key="window-c",
|
||||
source="claude-code",
|
||||
event_at=anchor + timedelta(minutes=10),
|
||||
)
|
||||
)
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="window-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (first.job_id, inside.job_id)
|
||||
queue.settle(
|
||||
first.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="window-owner",
|
||||
)
|
||||
queue.settle(
|
||||
inside.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="window-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (boundary.job_id,)
|
||||
|
||||
|
||||
def test_event_time_window_anchor_does_not_slide_while_a_job_is_running(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
first = queue.submit(_request(key="fixed-a", source="cursor", event_at=anchor))
|
||||
inside = queue.submit(
|
||||
_request(
|
||||
key="fixed-b",
|
||||
source="codex",
|
||||
event_at=anchor + timedelta(minutes=9),
|
||||
)
|
||||
)
|
||||
chained = queue.submit(
|
||||
_request(
|
||||
key="fixed-c",
|
||||
source="claude-code",
|
||||
event_at=anchor + timedelta(minutes=18),
|
||||
)
|
||||
)
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="fixed-window-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=2) == (first.job_id, inside.job_id)
|
||||
queue.settle(
|
||||
first.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="fixed-window-owner",
|
||||
)
|
||||
assert drain.claim(limit=1) == ()
|
||||
assert queue.get(chained.request_id).state == PENDING
|
||||
queue.settle(
|
||||
inside.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="fixed-window-owner",
|
||||
)
|
||||
assert drain.claim(limit=1) == (chained.job_id,)
|
||||
|
||||
|
||||
def test_sedimentation_pause_releases_claim_without_spending_an_attempt(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="sedimentation-pause", source="cursor"))
|
||||
drain = DurableWriteDrain(queue, execute=_never_execute, owner="pause-owner")
|
||||
assert drain.claim(limit=1) == (record.job_id,)
|
||||
|
||||
async def refuse_before_hy(_source_id, _mark_invoked):
|
||||
raise RuntimeError("model_sedimentation_disabled")
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
record.job_id,
|
||||
queue=queue,
|
||||
supervisor=_Recorder(invoke_hy=True),
|
||||
evidence=_EvidenceStore({"e1"}),
|
||||
owner="pause-owner",
|
||||
source_writes_permitted=lambda _source_id: True,
|
||||
admit_hy_invocation=refuse_before_hy,
|
||||
)
|
||||
)
|
||||
|
||||
current = queue.get(record.request_id)
|
||||
assert current is not None and current.state == PENDING
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT attempts,hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone() == (0, 0)
|
||||
|
||||
|
||||
def test_pause_after_first_hy_call_does_not_interrupt_the_current_job(tmp_path) -> None:
|
||||
@@ -4621,3 +4808,170 @@ def test_readmit_paths_never_requeue_a_possibly_sent_row(tmp_path) -> None:
|
||||
assert rows["parked"]["state"] == WAITING_FOR_PERMISSION
|
||||
assert rows["model"]["state"] == WAITING_FOR_MODEL
|
||||
assert DurableWriteDrain(queue, execute=_never_execute).claim(limit=5) == ()
|
||||
|
||||
|
||||
def test_a_refused_source_context_is_nonblocking_unknown(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""An ambiguous call-boundary failure is retained but never blocks FIFO."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="refused-scope", source="opencode"))
|
||||
claimer = DurableWriteDrain(queue, execute=_never_execute, owner="refused-owner")
|
||||
assert claimer.claim(limit=1) == (record.job_id,)
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
record.job_id,
|
||||
queue=queue,
|
||||
supervisor=_Recorder(
|
||||
error=HyMemoryBridgeError("hy_memory_write_verification_context_scope"),
|
||||
invoke_hy=True,
|
||||
),
|
||||
evidence=_EvidenceStore({"e1"}),
|
||||
owner="refused-owner",
|
||||
)
|
||||
)
|
||||
|
||||
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
|
||||
state, invoked, last_error = db.execute(
|
||||
"SELECT state, hy_invoked, last_error FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()
|
||||
|
||||
assert state == UNKNOWN
|
||||
assert invoked == 1
|
||||
assert last_error == "hy_memory_write_verification_context_scope"
|
||||
assert DurableWriteDrain(queue, execute=_never_execute).claim(limit=1) == ()
|
||||
|
||||
|
||||
def test_a_failure_after_the_call_still_parks_as_unknown(tmp_path) -> None:
|
||||
"""The fix must not widen: a post-call refusal is still possibly-sent."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="after-call", source="opencode"))
|
||||
claimer = DurableWriteDrain(queue, execute=_never_execute, owner="after-call-owner")
|
||||
assert claimer.claim(limit=1) == (record.job_id,)
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
record.job_id,
|
||||
queue=queue,
|
||||
supervisor=_Recorder(
|
||||
error=HyMemoryBridgeError("hy_memory_write_verification_authority_invalid"),
|
||||
invoke_hy=True,
|
||||
),
|
||||
evidence=_EvidenceStore({"e1"}),
|
||||
owner="after-call-owner",
|
||||
)
|
||||
)
|
||||
|
||||
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
|
||||
state, invoked = db.execute(
|
||||
"SELECT state, hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()
|
||||
|
||||
assert state == UNKNOWN
|
||||
assert invoked == 1
|
||||
|
||||
|
||||
def test_event_time_not_enqueue_order_drives_the_window(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""A later-enqueued old turn runs before an earlier-enqueued new turn."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
|
||||
enqueued_first = queue.submit(
|
||||
_request(
|
||||
key="newer-event",
|
||||
source="workbuddy",
|
||||
session="newer-session",
|
||||
event_at=anchor + timedelta(hours=1),
|
||||
)
|
||||
)
|
||||
enqueued_second = queue.submit(
|
||||
_request(
|
||||
key="older-event",
|
||||
source="workbuddy",
|
||||
session="older-session",
|
||||
event_at=anchor + timedelta(minutes=5),
|
||||
)
|
||||
)
|
||||
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="starve-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (enqueued_second.job_id,)
|
||||
queue.settle(
|
||||
enqueued_second.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="starve-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (enqueued_first.job_id,)
|
||||
|
||||
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
|
||||
window = db.execute(
|
||||
"SELECT window_anchor FROM memory_write_schedule_state WHERE singleton=1"
|
||||
).fetchone()
|
||||
assert window is not None
|
||||
assert str(window[0]) == (anchor + timedelta(hours=1)).astimezone(UTC).isoformat()
|
||||
|
||||
|
||||
def test_same_session_uses_event_time_then_queue_sequence(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
base = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
later = queue.submit(
|
||||
_request(key="same-session-later", source="cursor", event_at=base + timedelta(hours=1))
|
||||
)
|
||||
earlier = queue.submit(_request(key="same-session-earlier", source="cursor", event_at=base))
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="event-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (earlier.job_id,)
|
||||
queue.settle(
|
||||
earlier.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="event-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (later.job_id,)
|
||||
|
||||
|
||||
def test_equal_event_time_uses_queue_sequence_inside_one_session(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
event_at = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
first = queue.submit(
|
||||
_request(key="equal-first", source="cursor", event_at=event_at)
|
||||
)
|
||||
second = queue.submit(
|
||||
_request(key="equal-second", source="cursor", event_at=event_at)
|
||||
)
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="equal-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (first.job_id,)
|
||||
queue.settle(
|
||||
first.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="equal-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (second.job_id,)
|
||||
@@ -2103,7 +2103,7 @@ def test_migration_49_resets_only_cursor_history_when_conversation_time_is_missi
|
||||
|
||||
result = asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
assert result.applied == (49, 50, 51)
|
||||
assert result.applied == (49, 50, 51, 52, 53, 54)
|
||||
with main_database(path).read() as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM source_cursors").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT COUNT(*) FROM live_source_cursors").fetchone()[0] == 1
|
||||
@@ -2136,7 +2136,7 @@ def test_upgrade_drops_obsolete_hy_node_permissions_without_touching_memories(
|
||||
|
||||
result = asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
assert result.applied == (51,)
|
||||
assert result.applied == (51, 52, 53, 54)
|
||||
with sqlite3.connect(path) as connection:
|
||||
tables = {
|
||||
row[0]
|
||||
@@ -2147,3 +2147,125 @@ def test_upgrade_drops_obsolete_hy_node_permissions_without_touching_memories(
|
||||
assert "hy_node_permissions" not in tables
|
||||
assert "governed_memories" in tables
|
||||
assert "evidence" in tables
|
||||
|
||||
|
||||
def test_migration_53_backfills_write_job_event_time_from_person_turn(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
import sqlite3
|
||||
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore
|
||||
|
||||
path = tmp_path / "legacy-write-jobs.sqlite3"
|
||||
with sqlite3.connect(path) as connection:
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE memory_write_jobs (
|
||||
job_id TEXT PRIMARY KEY,
|
||||
queue_seq INTEGER NOT NULL,
|
||||
session_key TEXT NOT NULL,
|
||||
source_key TEXT NOT NULL,
|
||||
batch_id TEXT NOT NULL,
|
||||
idempotency_key TEXT NOT NULL UNIQUE,
|
||||
payload_json TEXT NOT NULL,
|
||||
evidence_ids_json TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
priority TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL,
|
||||
max_attempts INTEGER NOT NULL,
|
||||
lease_owner TEXT,
|
||||
lease_expires_at TEXT,
|
||||
hy_invoked INTEGER NOT NULL,
|
||||
last_error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE evidence (
|
||||
evidence_id TEXT PRIMARY KEY,
|
||||
role TEXT NOT NULL,
|
||||
written_by TEXT NOT NULL,
|
||||
created_at TEXT,
|
||||
imported_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE memory_write_job_evidence (
|
||||
job_id TEXT NOT NULL,
|
||||
evidence_id TEXT NOT NULL,
|
||||
PRIMARY KEY (job_id, evidence_id)
|
||||
);
|
||||
INSERT INTO memory_write_jobs VALUES (
|
||||
'job-1',1,'session','cursor','batch','idem','{}','[]',
|
||||
'pending','normal',0,3,NULL,NULL,0,NULL,
|
||||
'2026-10-18T08:00:00+00:00','2026-10-18T08:00:00+00:00'
|
||||
);
|
||||
INSERT INTO evidence VALUES (
|
||||
'user-1','user','person','2026-10-16T01:02:03+00:00',
|
||||
'2026-10-18T08:00:00+00:00'
|
||||
);
|
||||
INSERT INTO memory_write_job_evidence VALUES ('job-1','user-1');
|
||||
"""
|
||||
)
|
||||
|
||||
migration = next(item for item in MIGRATIONS if item.version == 53)
|
||||
result = asyncio.run(migrate(SqliteMigrationStore(path), (migration,)))
|
||||
|
||||
assert result.applied == (53,)
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT event_at,time_quality FROM memory_write_jobs"
|
||||
).fetchone() == ("2026-10-16T01:02:03+00:00", "approximate")
|
||||
assert connection.execute(
|
||||
"SELECT 1 FROM sqlite_master "
|
||||
"WHERE type='table' AND name='memory_write_schedule_state'"
|
||||
).fetchone() == (1,)
|
||||
|
||||
|
||||
def test_migration_54_preserves_legacy_possibly_sent_tasks(tmp_path) -> None:
|
||||
import sqlite3
|
||||
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
through_53 = tuple(item for item in MIGRATIONS if item.version <= 53)
|
||||
asyncio.run(migrate(SqliteMigrationStore(path), through_53))
|
||||
with sqlite3.connect(path) as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO memory_tasks (
|
||||
task_id,request_id,user_id,kind,input_ref,status,
|
||||
profile_id,profile_revision,profile_snapshot_json,
|
||||
attempts,created_at,updated_at,hy_invocation_count
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,0,?,?,0)
|
||||
""",
|
||||
(
|
||||
"legacy-sent",
|
||||
"legacy-sent-request",
|
||||
"local",
|
||||
"extract",
|
||||
"evidence",
|
||||
"running",
|
||||
"profile",
|
||||
1,
|
||||
"{}",
|
||||
"2026-10-18T00:00:00+00:00",
|
||||
"2026-10-18T00:00:00+00:00",
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE memory_task_continuous_permissions "
|
||||
"(task_id TEXT PRIMARY KEY)"
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_task_continuous_permissions VALUES (?)",
|
||||
("legacy-sent",),
|
||||
)
|
||||
|
||||
asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT hy_invocation_count FROM memory_tasks WHERE task_id='legacy-sent'"
|
||||
).fetchone() == (1,)
|
||||
assert connection.execute(
|
||||
"SELECT 1 FROM sqlite_master "
|
||||
"WHERE type='table' AND name='memory_task_continuous_permissions'"
|
||||
).fetchone() is None
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
|
||||
@@ -9,13 +11,23 @@ import pytest
|
||||
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.durable_queue import DurableWriteQueue
|
||||
from one_memory.memory_server.durable_queue import (
|
||||
DurableWriteDrain,
|
||||
DurableWriteQueue,
|
||||
execute_job,
|
||||
)
|
||||
from one_memory.memory_server.unified_entry import (
|
||||
UnifiedWriteIntake,
|
||||
)
|
||||
|
||||
|
||||
def _record(store: EvidenceStore, *, conversation_id: str = "c1", message_id: str = "m1"):
|
||||
def _record(
|
||||
store: EvidenceStore,
|
||||
*,
|
||||
conversation_id: str = "c1",
|
||||
message_id: str = "m1",
|
||||
verbatim: str = "durable intake",
|
||||
):
|
||||
return store.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
@@ -24,7 +36,7 @@ def _record(store: EvidenceStore, *, conversation_id: str = "c1", message_id: st
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=datetime.now(UTC),
|
||||
verbatim="durable intake",
|
||||
verbatim=verbatim,
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
@@ -42,6 +54,51 @@ def test_evidence_submission_creates_one_durable_row(tmp_path) -> None:
|
||||
assert receipt.evidence_id == record.evidence_id
|
||||
assert receipt.request_id == f"entry:{record.evidence_id}"
|
||||
assert intake.queue.get(receipt.request_id) is not None
|
||||
with sqlite3.connect(path) as connection:
|
||||
event_at, quality = connection.execute(
|
||||
"SELECT event_at,time_quality FROM memory_write_jobs WHERE job_id=?",
|
||||
(receipt.record.job_id,),
|
||||
).fetchone()
|
||||
assert datetime.fromisoformat(event_at) == record.created_at
|
||||
assert quality == "exact"
|
||||
|
||||
|
||||
def test_cursor_turn_timestamp_orders_job_without_reordering_evidence(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
file_mtime = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
turn_time = datetime(2026, 10, 16, 0, 24, tzinfo=UTC)
|
||||
record = store.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
conversation_id="cursor-session",
|
||||
message_id="cursor-user",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=file_mtime,
|
||||
verbatim="timestamped Cursor turn",
|
||||
origin_meta={
|
||||
EvidenceOrigin.FILE: OriginMetadata(
|
||||
raw_meta={
|
||||
"time_basis": "file_mtime",
|
||||
"turn_event_at": turn_time.isoformat(),
|
||||
}
|
||||
)
|
||||
},
|
||||
)
|
||||
).record
|
||||
intake = UnifiedWriteIntake(DurableWriteQueue(path, evidence=store))
|
||||
|
||||
receipt = intake.submit_evidence(record, first_round=1, last_round=1)
|
||||
|
||||
with sqlite3.connect(path) as connection:
|
||||
event_at, quality = connection.execute(
|
||||
"SELECT event_at,time_quality FROM memory_write_jobs WHERE job_id=?",
|
||||
(receipt.record.job_id,),
|
||||
).fetchone()
|
||||
assert record.created_at == file_mtime
|
||||
assert datetime.fromisoformat(event_at) == turn_time
|
||||
assert quality == "exact"
|
||||
|
||||
|
||||
def test_a_long_branch_conversation_id_still_enqueues(tmp_path) -> None:
|
||||
@@ -127,6 +184,136 @@ def test_replaying_the_same_evidence_does_not_create_a_second_row(tmp_path) -> N
|
||||
assert replay.enqueued is False
|
||||
|
||||
|
||||
def test_rewound_harness_revision_appends_a_job_with_immutable_input(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
intake = UnifiedWriteIntake(DurableWriteQueue(path, evidence=store))
|
||||
old = _record(store, verbatim="old branch")
|
||||
old_job = intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
|
||||
revised = _record(store, verbatim="new branch")
|
||||
new_job = intake.submit_evidence(revised, first_round=1, last_round=1)
|
||||
|
||||
assert revised.evidence_id == old.evidence_id
|
||||
assert revised.revision == old.revision + 1
|
||||
assert new_job.record.job_id != old_job.record.job_id
|
||||
assert intake.queue.payload_for(old_job.record.job_id).input_records[0].verbatim == "old branch"
|
||||
assert intake.queue.payload_for(new_job.record.job_id).input_records[0].verbatim == "new branch"
|
||||
assert intake.queue.unsubmitted_evidence_ids() == ()
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM memory_write_jobs").fetchone()[0] == 2
|
||||
|
||||
|
||||
def test_revision_committed_before_job_crash_is_discovered_for_reconcile(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
intake = UnifiedWriteIntake(queue)
|
||||
old = _record(store, verbatim="old branch")
|
||||
intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
|
||||
revised = _record(store, verbatim="new branch")
|
||||
|
||||
assert revised.evidence_id == old.evidence_id
|
||||
assert revised.revision == old.revision + 1
|
||||
assert queue.unsubmitted_evidence_ids() == (revised.evidence_id,)
|
||||
|
||||
|
||||
def test_late_old_revision_snapshot_does_not_cover_the_current_revision(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
intake = UnifiedWriteIntake(queue)
|
||||
old = _record(store, verbatim="old branch")
|
||||
revised = _record(store, verbatim="new branch")
|
||||
|
||||
intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
|
||||
assert revised.evidence_id == old.evidence_id
|
||||
assert revised.revision == old.revision + 1
|
||||
assert queue.unsubmitted_evidence_ids() == (revised.evidence_id,)
|
||||
|
||||
|
||||
def test_old_branch_job_executes_its_frozen_revision_after_harness_rewind(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
intake = UnifiedWriteIntake(queue)
|
||||
old = _record(store, verbatim="old branch")
|
||||
old_job = intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
revised = _record(store, verbatim="new branch")
|
||||
intake.submit_evidence(revised, first_round=1, last_round=1)
|
||||
drain = DurableWriteDrain(queue, execute=lambda _job_id: asyncio.sleep(0), owner="frozen")
|
||||
assert drain.claim(limit=1) == (old_job.record.job_id,)
|
||||
|
||||
captured = []
|
||||
|
||||
class Supervisor:
|
||||
async def process_records(self, records, *, before_hy_invocation):
|
||||
captured.extend(records)
|
||||
await before_hy_invocation()
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
old_job.record.job_id,
|
||||
queue=queue,
|
||||
supervisor=Supervisor(),
|
||||
evidence=store,
|
||||
owner="frozen",
|
||||
)
|
||||
)
|
||||
|
||||
assert [record.verbatim for record in captured] == ["old branch"]
|
||||
assert queue.get(old_job.request_id).state == "completed"
|
||||
|
||||
|
||||
def test_reopening_reads_an_old_id_only_job_without_rewriting_it(tmp_path) -> None:
|
||||
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
record = _record(store)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
legacy = queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id=record.conversation_id,
|
||||
source_id=record.source_id,
|
||||
batch_identity=BatchIdentity(
|
||||
internal_session_id=record.conversation_id,
|
||||
first_round=1,
|
||||
last_round=1,
|
||||
),
|
||||
evidence_ids=[record.evidence_id],
|
||||
idempotency_key="legacy-id-only",
|
||||
)
|
||||
)
|
||||
with sqlite3.connect(path) as connection, connection:
|
||||
payload = json.loads(
|
||||
connection.execute(
|
||||
"SELECT payload_json FROM memory_write_jobs WHERE job_id=?",
|
||||
(legacy.job_id,),
|
||||
).fetchone()[0]
|
||||
)
|
||||
payload.pop("input_records", None)
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET payload_json=? WHERE job_id=?",
|
||||
(json.dumps(payload, sort_keys=True), legacy.job_id),
|
||||
)
|
||||
assert queue.payload_for(legacy.job_id).input_records == ()
|
||||
|
||||
reopened = DurableWriteQueue(path, evidence=store)
|
||||
|
||||
assert reopened.payload_for(legacy.job_id).input_records == ()
|
||||
with sqlite3.connect(path) as connection:
|
||||
persisted = json.loads(
|
||||
connection.execute(
|
||||
"SELECT payload_json FROM memory_write_jobs WHERE job_id=?",
|
||||
(legacy.job_id,),
|
||||
).fetchone()[0]
|
||||
)
|
||||
assert "input_records" not in persisted
|
||||
|
||||
|
||||
def test_committed_evidence_without_submit_is_the_red_state(tmp_path) -> None:
|
||||
"""The exact crash window the C0 wiring has to close.
|
||||
|
||||
|
||||
@@ -1500,14 +1500,8 @@ def test_restart_splits_a_sent_continuous_task_from_an_unsent_one(tmp_path) -> N
|
||||
input_ref="evidence-unsent",
|
||||
profile=profile,
|
||||
)
|
||||
with sqlite3.connect(store.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE memory_task_continuous_permissions (task_id TEXT PRIMARY KEY)"
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_task_continuous_permissions VALUES (?)", (sent.task_id,)
|
||||
)
|
||||
store.claim(sent.task_id)
|
||||
_sent_task, sent_token = store.claim(sent.task_id)
|
||||
store.record_hy_invocation(sent.task_id, runner_token=sent_token)
|
||||
store.claim(unsent.task_id)
|
||||
|
||||
recovered = store.recover_running()
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from one_memory.model_request_audit import (
|
||||
AuditedModelTransport,
|
||||
ModelRequestAuditStore,
|
||||
audit_precipitation_request,
|
||||
)
|
||||
|
||||
|
||||
def _transport(tmp_path, response):
|
||||
store = ModelRequestAuditStore(tmp_path / "one-memory.sqlite3")
|
||||
return store, AuditedModelTransport(
|
||||
store,
|
||||
profile_id="profile-1",
|
||||
profile_revision=2,
|
||||
provider_kind="openai_compatible",
|
||||
model="deepseek-flash",
|
||||
inner=httpx.MockTransport(response),
|
||||
)
|
||||
|
||||
|
||||
def test_records_one_physical_request_with_provider_usage_and_no_content(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
private = "private conversation text"
|
||||
monkeypatch.setattr(
|
||||
"one_memory.model_request_audit._current_prompt_stage",
|
||||
lambda: "coding_judge",
|
||||
)
|
||||
|
||||
def response(_request):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "provider-response-1",
|
||||
"usage": {
|
||||
"prompt_tokens": 120,
|
||||
"completion_tokens": 30,
|
||||
"total_tokens": 150,
|
||||
"prompt_tokens_details": {"cached_tokens": 80},
|
||||
"completion_tokens_details": {"reasoning_tokens": 10},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
store, transport = _transport(tmp_path, response)
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
with audit_precipitation_request("memory-request-1"):
|
||||
result = await client.post(
|
||||
"https://example.test/chat/completions",
|
||||
json={"model": "deepseek-flash", "messages": [{"content": private}]},
|
||||
)
|
||||
assert result.status_code == 200
|
||||
assert result.json()["id"] == "provider-response-1"
|
||||
|
||||
asyncio.run(scenario())
|
||||
report = store.report(request_id="memory-request-1")
|
||||
assert report["summary"] == {
|
||||
"request_count": 1,
|
||||
"succeeded": 1,
|
||||
"failed": 0,
|
||||
"in_flight": 0,
|
||||
"prompt_tokens": 120,
|
||||
"completion_tokens": 30,
|
||||
"total_tokens": 150,
|
||||
"cached_prompt_tokens": 80,
|
||||
"reasoning_tokens": 10,
|
||||
"avg_prompt_tokens": 120.0,
|
||||
"avg_completion_tokens": 30.0,
|
||||
}
|
||||
row = report["requests"][0]
|
||||
assert row["provider_request_id"] == "provider-response-1"
|
||||
assert row["request_kind"] == "chat_completion"
|
||||
assert row["prompt_stage"] == "coding_judge"
|
||||
assert row["profile_revision"] == 2
|
||||
assert report["by_minute"][0]["prompt_stage"] == "coding_judge"
|
||||
assert report["by_minute"][0]["total_tokens"] == 150
|
||||
assert report["by_minute"][0]["avg_prompt_tokens"] == 120.0
|
||||
assert private.encode() not in store.database.read_bytes()
|
||||
|
||||
|
||||
def test_requests_outside_precipitation_are_not_recorded(tmp_path):
|
||||
store, transport = _transport(tmp_path, lambda _request: httpx.Response(200, json={}))
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
await client.post("https://example.test/chat/completions", json={})
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert store.report()["summary"]["request_count"] == 0
|
||||
|
||||
|
||||
def test_precipitation_context_reaches_the_sdk_worker_thread(tmp_path):
|
||||
store, transport = _transport(
|
||||
tmp_path,
|
||||
lambda _request: httpx.Response(
|
||||
200,
|
||||
json={"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}},
|
||||
),
|
||||
)
|
||||
|
||||
def worker():
|
||||
async def request():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
await client.post("https://example.test/chat/completions", json={})
|
||||
|
||||
asyncio.run(request())
|
||||
|
||||
async def scenario():
|
||||
with audit_precipitation_request("threaded-memory-request"):
|
||||
await asyncio.to_thread(worker)
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert store.report(request_id="threaded-memory-request")["summary"]["total_tokens"] == 3
|
||||
|
||||
|
||||
def test_failure_is_recorded_without_error_message(tmp_path):
|
||||
secret = "provider leaked private detail"
|
||||
|
||||
def response(_request):
|
||||
raise httpx.ConnectError(secret)
|
||||
|
||||
store, transport = _transport(tmp_path, response)
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
with audit_precipitation_request("memory-request-failed"):
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
await client.post("https://example.test/embeddings", json={})
|
||||
|
||||
asyncio.run(scenario())
|
||||
report = store.report(request_id="memory-request-failed")
|
||||
row = report["requests"][0]
|
||||
assert row["outcome"] == "failed"
|
||||
assert row["error_type"] == "ConnectError"
|
||||
assert row["request_kind"] == "embedding"
|
||||
assert secret.encode() not in store.database.read_bytes()
|
||||
|
||||
|
||||
def test_missing_usage_stays_unknown_in_detail_and_zero_in_summary(tmp_path):
|
||||
store, transport = _transport(
|
||||
tmp_path,
|
||||
lambda _request: httpx.Response(200, content=json.dumps({"id": "r"}).encode()),
|
||||
)
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
with audit_precipitation_request("memory-request-no-usage"):
|
||||
await client.post("https://example.test/chat/completions", json={})
|
||||
|
||||
asyncio.run(scenario())
|
||||
report = store.report(request_id="memory-request-no-usage")
|
||||
assert report["requests"][0]["total_tokens"] is None
|
||||
assert report["summary"]["total_tokens"] == 0
|
||||
|
||||
|
||||
def test_store_schema_matches_migration_and_supports_parallel_starts(tmp_path):
|
||||
store = ModelRequestAuditStore(tmp_path / "one-memory.sqlite3")
|
||||
|
||||
async def start(index):
|
||||
return await asyncio.to_thread(
|
||||
store.start,
|
||||
request_id=f"request-{index}",
|
||||
profile_id="profile",
|
||||
profile_revision=1,
|
||||
provider_kind="openai_compatible",
|
||||
model="model",
|
||||
request_kind="chat_completion",
|
||||
prompt_stage="other",
|
||||
)
|
||||
|
||||
async def scenario():
|
||||
return await asyncio.gather(*(start(index) for index in range(8)))
|
||||
|
||||
ids = asyncio.run(scenario())
|
||||
assert len(set(ids)) == 8
|
||||
with sqlite3.connect(store.database) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM model_request_audit").fetchone() == (8,)
|
||||
@@ -3,7 +3,12 @@ import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
from one_memory.api import _run_backfill
|
||||
from one_memory.api import (
|
||||
_history_import_gate_released,
|
||||
_run_backfill,
|
||||
_sedimentation_claim_enabled,
|
||||
_source_writes_on,
|
||||
)
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.config_state import ConfigStore
|
||||
@@ -92,3 +97,61 @@ def test_source_pause_skips_history_until_the_source_is_reenabled(tmp_path) -> N
|
||||
asyncio.run(_run_backfill(app))
|
||||
|
||||
assert app.state.memory_write_queue.unsubmitted_evidence_ids() == ()
|
||||
|
||||
|
||||
def test_the_manual_source_is_admitted_by_grant_without_a_toggle(tmp_path) -> None:
|
||||
"""A source that is not a connected app has no toggle to read.
|
||||
|
||||
`one-memory-manual` is admitted by its grant and never appears in the
|
||||
configuration document, so any gate that reads the per-app toggle answers
|
||||
False for it while the write gate answers True. The drain's claim gate used
|
||||
to be that gate, and every manual note's job was skipped on every tick.
|
||||
"""
|
||||
config = ConfigStore(tmp_path)
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
config_store=config,
|
||||
source_grant_store=SimpleNamespace(
|
||||
receipt=lambda **_kwargs: SimpleNamespace(revoked_at=None)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert _source_writes_on(app, "one-memory-manual") is True
|
||||
assert config.sync_enabled("one-memory-manual") is False
|
||||
|
||||
assert _sedimentation_claim_enabled(app, "one-memory-manual") is True
|
||||
config.set_sedimentation_enabled(False)
|
||||
assert _sedimentation_claim_enabled(app, "one-memory-manual") is False
|
||||
|
||||
|
||||
def test_first_import_gate_waits_for_every_selected_history_job() -> None:
|
||||
state = SimpleNamespace(
|
||||
status="importing",
|
||||
selected_source_ids=("cursor", "codex"),
|
||||
started_at=datetime(2026, 10, 18, tzinfo=UTC),
|
||||
)
|
||||
jobs = [
|
||||
SimpleNamespace(
|
||||
operation="history",
|
||||
source_id="cursor",
|
||||
created_at=datetime(2026, 10, 18, tzinfo=UTC),
|
||||
status="completed",
|
||||
),
|
||||
SimpleNamespace(
|
||||
operation="history",
|
||||
source_id="codex",
|
||||
created_at=datetime(2026, 10, 18, tzinfo=UTC),
|
||||
status="running",
|
||||
),
|
||||
]
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
onboarding_store=SimpleNamespace(get=lambda: state),
|
||||
import_job_store=SimpleNamespace(list_jobs=lambda: jobs),
|
||||
)
|
||||
)
|
||||
|
||||
assert _history_import_gate_released(app) is False
|
||||
jobs[1].status = "failed"
|
||||
assert _history_import_gate_released(app) is True
|
||||
@@ -19,7 +19,12 @@ def test_production_serve_passes_only_supported_app_options(monkeypatch, tmp_pat
|
||||
|
||||
def run(application, **kwargs):
|
||||
assert application is marker
|
||||
assert kwargs == {"host": "127.0.0.1", "port": 12345, "reload": False}
|
||||
assert kwargs == {
|
||||
"host": "127.0.0.1",
|
||||
"port": 12345,
|
||||
"reload": False,
|
||||
"access_log": False,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(cli, "load_settings", lambda: settings)
|
||||
monkeypatch.setattr(log_file, "start_logging_to_file", lambda _: None)
|
||||
@@ -46,7 +51,12 @@ def test_maintenance_serve_does_not_own_the_runtime(monkeypatch, tmp_path):
|
||||
|
||||
def run(application, **kwargs):
|
||||
assert application is marker
|
||||
assert kwargs == {"host": "127.0.0.1", "port": 12346, "reload": False}
|
||||
assert kwargs == {
|
||||
"host": "127.0.0.1",
|
||||
"port": 12346,
|
||||
"reload": False,
|
||||
"access_log": False,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(cli, "load_settings", lambda: settings)
|
||||
monkeypatch.setattr(log_file, "start_logging_to_file", lambda _: None)
|
||||
|
||||
@@ -13,7 +13,7 @@ from one_memory.imports.adapters import (
|
||||
SourceCursor,
|
||||
)
|
||||
from one_memory.imports.jobs import ImportJobStore, ImportStatus
|
||||
from one_memory.imports.models import WrittenBy
|
||||
from one_memory.imports.models import EvidenceOrigin, WrittenBy
|
||||
from one_memory.imports.runner import ImportRunner
|
||||
from one_memory.imports.sources.claude_code import create_claude_code_adapter
|
||||
from one_memory.imports.sources.codex import create_codex_adapter
|
||||
@@ -131,6 +131,9 @@ def test_cursor_store_timestamp_envelope_keeps_only_the_user_query_text(tmp_path
|
||||
|
||||
assert [item.evidence.verbatim for item in records] == [query]
|
||||
assert records[0].evidence.written_by is WrittenBy.PERSON
|
||||
meta = records[0].evidence.origin_meta[EvidenceOrigin.FILE].raw_meta
|
||||
assert meta["time_basis"] == "file_mtime"
|
||||
assert meta["turn_event_at"] == "2026-09-22T23:34:00+00:00"
|
||||
|
||||
|
||||
def test_cursor_transcript_timestamp_envelope_keeps_only_the_user_query_text(tmp_path) -> None:
|
||||
@@ -163,6 +166,9 @@ def test_cursor_transcript_timestamp_envelope_keeps_only_the_user_query_text(tmp
|
||||
|
||||
assert [item.evidence.verbatim for item in records] == [query]
|
||||
assert records[0].evidence.written_by is WrittenBy.PERSON
|
||||
meta = records[0].evidence.origin_meta[EvidenceOrigin.FILE].raw_meta
|
||||
assert meta["time_basis"] == "file_mtime"
|
||||
assert meta["turn_event_at"] == "2026-09-22T23:34:00+00:00"
|
||||
|
||||
|
||||
def test_cursor_transcript_uses_file_mtime_with_stable_record_order(tmp_path) -> None:
|
||||
|
||||
@@ -191,10 +191,12 @@ from one_memory import cli
|
||||
from one_memory.config import Settings
|
||||
|
||||
served_on = []
|
||||
served_with = []
|
||||
|
||||
|
||||
def fake_run(application, *, host=None, port=None, **_kwargs):
|
||||
served_on.append(port)
|
||||
served_with.append(_kwargs)
|
||||
|
||||
|
||||
uvicorn.run = fake_run
|
||||
@@ -219,6 +221,7 @@ json.dump(
|
||||
{
|
||||
"printed": printed,
|
||||
"served_on": served_on,
|
||||
"served_with": served_with,
|
||||
"port": int(match.group(1)) if match else None,
|
||||
},
|
||||
sys.stdout,
|
||||
@@ -249,3 +252,4 @@ json.dump(
|
||||
# Port 0 means "any free port", so the reported value must be a real one.
|
||||
assert payload["port"] > 0
|
||||
assert payload["served_on"] == [payload["port"]]
|
||||
assert payload["served_with"] == [{"reload": False, "access_log": False}]
|
||||
@@ -132,19 +132,22 @@ def test_raw_authors_keep_exact_body_and_do_not_mutate_evidence(role, author):
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"role": "user", "written_by": WrittenBy.AGENT},
|
||||
{"role": "assistant", "written_by": WrittenBy.PERSON},
|
||||
{"role": "tool", "written_by": WrittenBy.TOOL},
|
||||
{"role": "system"},
|
||||
{"written_by": WrittenBy.UNKNOWN},
|
||||
{"content_kind": EvidenceContentKind.MEMORY_CONTENT},
|
||||
],
|
||||
)
|
||||
def test_raw_and_author_qualification_rejects_without_tokenizing(change):
|
||||
def test_raw_role_qualification_rejects_without_tokenizing(change):
|
||||
with pytest.raises(MappedContentError, match="mapped_content_ineligible"):
|
||||
prepare_mapped_content(record("ordinary body", **change))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("author", list(WrittenBy))
|
||||
def test_host_author_guess_does_not_change_protocol_role_qualification(author):
|
||||
content = prepare_mapped_content(record("ordinary body", role="user", written_by=author))
|
||||
assert content.state == "ready"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,reason",
|
||||
[
|
||||
|
||||
@@ -25,7 +25,6 @@ def _message(
|
||||
message_id=message_id or f"m{ordinal}",
|
||||
ordinal=ordinal,
|
||||
role=role,
|
||||
written_by="person" if role == "user" else "model",
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ Dev history: when the model extracted no fact, the coding extractor stored the
|
||||
raw dialogue as a working fact, and with no words of the person's it took the
|
||||
first 600 characters of the rendered trace. MemHub showed that as the memory
|
||||
content, which is where `[0] assistant: ... [1] tool(?): ...` came from
|
||||
(D87). The SDK fix remains present in 2.0.3: it does not store that fallback by
|
||||
(D87). The SDK fix remains present in 2.0.4: it does not store that fallback by
|
||||
default.
|
||||
|
||||
This pins the default here, so a future bump that turns it back on, or our own
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
"""Stopping model-driven sedimentation without losing the configuration.
|
||||
|
||||
The switch is the user's cost control: while it is off nothing may reach the
|
||||
model, but neither the model configuration nor the already imported
|
||||
conversations may be touched, and work that was already queued has to survive
|
||||
so starting again resumes it instead of beginning from nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import SecretStr
|
||||
|
||||
from one_memory.api import create_app
|
||||
from one_memory.config import Settings
|
||||
from one_memory.extraction_settings import ExtractionSettingsStore
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridgeError
|
||||
from one_memory.imports.models import EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceInput, EvidenceStore
|
||||
from one_memory.memory_processing import MemoryProcessingSupervisor
|
||||
from one_memory.memory_tasks import MemoryTaskKind, MemoryTaskStatus, MemoryTaskStore
|
||||
from one_memory.model_profiles import (
|
||||
ModelCapabilities,
|
||||
ModelProfileInput,
|
||||
ModelProfileStore,
|
||||
ProviderKind,
|
||||
)
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
NOW = datetime(2026, 9, 9, tzinfo=UTC)
|
||||
|
||||
|
||||
class _Flag:
|
||||
"""The user's switch, standing in for `ConfigStore.sedimentation_enabled`."""
|
||||
|
||||
def __init__(self, value: bool = True) -> None:
|
||||
self.value = value
|
||||
|
||||
def __call__(self) -> bool:
|
||||
return self.value
|
||||
|
||||
|
||||
def test_sedimentation_route_persists_and_resumes_without_restart(tmp_path) -> None:
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=tmp_path / "home",
|
||||
api_token=SecretStr("test"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
with TestClient(app, headers={"Authorization": "Bearer test"}) as client:
|
||||
assert client.get("/api/v1/sedimentation").json()["enabled"] is True
|
||||
stopped = client.put(
|
||||
"/api/v1/sedimentation", json={"enabled": False}
|
||||
)
|
||||
assert stopped.status_code == 200
|
||||
assert stopped.json()["enabled"] is False
|
||||
assert client.get("/api/v1/sedimentation").json()["enabled"] is False
|
||||
resumed = client.put("/api/v1/sedimentation", json={"enabled": True})
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json()["enabled"] is True
|
||||
|
||||
|
||||
def test_switch_answers_before_the_resume_finishes(tmp_path, monkeypatch) -> None:
|
||||
"""The PUT is a decision; the walk over the backlog happens after it.
|
||||
|
||||
Awaiting that walk is what turned "turn sedimentation back on" into a
|
||||
request that never returned. The renderer gave up at its thirty-second
|
||||
timeout and reported a failure for a switch that had in fact been saved,
|
||||
which is the worst of both: the user believes the model is still stopped
|
||||
while it is already running.
|
||||
"""
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=tmp_path / "home",
|
||||
api_token=SecretStr("test"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
released = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
with TestClient(app, headers={"Authorization": "Bearer test"}) as client:
|
||||
supervisor = app.state.memory_processing_supervisor
|
||||
|
||||
async def slow_resume() -> None:
|
||||
await asyncio.get_running_loop().run_in_executor(None, released.wait, 10.0)
|
||||
finished.set()
|
||||
|
||||
monkeypatch.setattr(supervisor, "resume_allowed", slow_resume)
|
||||
assert client.put("/api/v1/sedimentation", json={"enabled": False}).status_code == 200
|
||||
resumed = client.put("/api/v1/sedimentation", json={"enabled": True})
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json()["enabled"] is True
|
||||
# Answered while the resume is still walking the backlog.
|
||||
assert finished.is_set() is False
|
||||
released.set()
|
||||
|
||||
|
||||
def test_a_failing_resume_does_not_fail_the_switch(tmp_path, monkeypatch) -> None:
|
||||
"""Resuming is opportunistic; the user's choice is already stored."""
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=tmp_path / "home",
|
||||
api_token=SecretStr("test"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
with TestClient(app, headers={"Authorization": "Bearer test"}) as client:
|
||||
supervisor = app.state.memory_processing_supervisor
|
||||
|
||||
async def broken_resume() -> None:
|
||||
raise RuntimeError("hy_memory_processing_unavailable")
|
||||
|
||||
monkeypatch.setattr(supervisor, "resume_allowed", broken_resume)
|
||||
assert client.put("/api/v1/sedimentation", json={"enabled": False}).status_code == 200
|
||||
resumed = client.put("/api/v1/sedimentation", json={"enabled": True})
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json()["enabled"] is True
|
||||
|
||||
|
||||
def _stores(tmp_path, flag: _Flag):
|
||||
database_path = tmp_path / "one-memory.sqlite3"
|
||||
evidence = EvidenceStore(database_path)
|
||||
tasks = MemoryTaskStore(database_path)
|
||||
profiles = ModelProfileStore(database_path)
|
||||
secrets = InMemorySecretStore()
|
||||
profile = profiles.create(
|
||||
ModelProfileInput(
|
||||
name="Local",
|
||||
provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
base_url="https://example.com/v1",
|
||||
model="test-model",
|
||||
embedding_provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
embedding_model="builtin",
|
||||
embedding_dimensions=512,
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
profiles.record_capabilities(
|
||||
profile.profile_id,
|
||||
expected_revision=profile.revision,
|
||||
capabilities=ModelCapabilities(
|
||||
endpoint_reachable=True,
|
||||
model_available=True,
|
||||
structured_output=True,
|
||||
embedding_available=True,
|
||||
),
|
||||
secrets=secrets,
|
||||
)
|
||||
active = profiles.activate(profile.profile_id, secrets)
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=tasks,
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=evidence,
|
||||
governance=None, # type: ignore[arg-type]
|
||||
data_dir=tmp_path,
|
||||
extraction_settings=ExtractionSettingsStore(database_path),
|
||||
sedimentation_enabled=flag,
|
||||
)
|
||||
return evidence, tasks, active, supervisor, profiles, secrets
|
||||
|
||||
|
||||
def _task(evidence, tasks, active, *, index: int, age_days: int = 1):
|
||||
record = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
workspace="/Users/me/project",
|
||||
conversation_id=f"conv-{index}",
|
||||
message_id=f"m{index}",
|
||||
correlation_id=f"m{index}",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=NOW - timedelta(days=age_days),
|
||||
verbatim="I prefer concise answers.",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
return tasks.create(
|
||||
request_id=f"req-{index}",
|
||||
user_id="local",
|
||||
kind=MemoryTaskKind.EXTRACT,
|
||||
input_ref=record.evidence_id,
|
||||
profile=active,
|
||||
)
|
||||
|
||||
|
||||
def test_stopped_switch_classifies_no_work_and_keeps_it_pending(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
|
||||
assert supervisor._classify([task]) == []
|
||||
# Parked, not failed: nothing was sent to the model, so a stopped switch
|
||||
# must not read as a broken pipeline.
|
||||
assert tasks.require(task.task_id).status is MemoryTaskStatus.PENDING
|
||||
|
||||
flag.value = True
|
||||
assert len(supervisor._classify([task])) == 1
|
||||
|
||||
|
||||
def test_stopped_switch_refuses_direct_durable_processing(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, flag
|
||||
)
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
record = evidence.get(task.input_ref)
|
||||
assert record is not None
|
||||
|
||||
with pytest.raises(HyMemoryBridgeError, match="model_sedimentation_disabled"):
|
||||
asyncio.run(supervisor.process_records((record,)))
|
||||
|
||||
assert tasks.require(task.task_id).status is MemoryTaskStatus.PENDING
|
||||
|
||||
|
||||
def test_pause_returns_a_claimed_task_attempt(tmp_path) -> None:
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, _supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, flag
|
||||
)
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
claimed, token = tasks.claim(task.task_id)
|
||||
assert claimed.attempts == 1
|
||||
|
||||
released = tasks.release_unstarted(
|
||||
task.task_id,
|
||||
runner_token=token,
|
||||
error_code="model_sedimentation_disabled",
|
||||
)
|
||||
|
||||
assert released.status is MemoryTaskStatus.PENDING
|
||||
assert released.attempts == 0
|
||||
|
||||
|
||||
def test_stopping_after_scheduler_classification_starts_no_worker(tmp_path) -> None:
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, flag
|
||||
)
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
work = supervisor._classify([task])
|
||||
assert len(work) == 1
|
||||
supervisor._queue.add(work[0])
|
||||
|
||||
flag.value = False
|
||||
asyncio.run(supervisor._work())
|
||||
|
||||
current = tasks.require(task.task_id)
|
||||
assert current.status is MemoryTaskStatus.PENDING
|
||||
assert current.attempts == 0
|
||||
|
||||
|
||||
def test_stopped_switch_enqueues_nothing(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, _tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
record = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
workspace="/Users/me/project",
|
||||
conversation_id="conv-live",
|
||||
message_id="m-live",
|
||||
correlation_id="m-live",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=NOW,
|
||||
verbatim="Remember this.",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
|
||||
assert supervisor.enqueue_evidence(record) is None
|
||||
assert supervisor._debounces == {}
|
||||
|
||||
|
||||
def test_stopped_switch_prepares_no_backfill(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
# No task is queued yet: backfill is what would create one for this
|
||||
# conversation, so the stopped switch must keep it from creating any.
|
||||
# A settled turn (question plus answer): an unanswered user tail is not
|
||||
# backfill material, so the conversation needs both records.
|
||||
for message_id, role, written_by, verbatim in (
|
||||
("m-old", "user", WrittenBy.PERSON, "Remember this too."),
|
||||
("m-old-reply", "assistant", WrittenBy.AGENT, "Saved."),
|
||||
):
|
||||
evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
workspace="/Users/me/project",
|
||||
conversation_id="conv-old",
|
||||
message_id=message_id,
|
||||
correlation_id=message_id,
|
||||
role=role,
|
||||
written_by=written_by,
|
||||
created_at=NOW - timedelta(days=1),
|
||||
verbatim=verbatim,
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
)
|
||||
|
||||
assert supervisor.prepare_backfill() == []
|
||||
|
||||
flag.value = True
|
||||
assert len(supervisor.prepare_backfill()) == 1
|
||||
|
||||
|
||||
def test_absent_switch_keeps_sedimenting(tmp_path) -> None:
|
||||
"""A Core without the setting behaves exactly as it did before it existed."""
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, _Flag(True)
|
||||
)
|
||||
supervisor.sedimentation_enabled = None
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
|
||||
assert len(supervisor._classify([task])) == 1
|
||||
|
||||
|
||||
def test_stopping_does_not_disturb_the_model_configuration(tmp_path) -> None:
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, supervisor, profiles, secrets = _stores(tmp_path, flag)
|
||||
_task(evidence, tasks, active, index=1)
|
||||
|
||||
flag.value = False
|
||||
assert supervisor.prepare_backfill() == []
|
||||
# Still configured: starting again is a switch, not a re-entry of a key.
|
||||
assert profiles.active(secrets) is not None
|
||||
|
||||
|
||||
def test_bounded_backfill_returns_when_the_queue_is_already_full(tmp_path) -> None:
|
||||
"""A full queue means nothing to add, not "wait until it drains".
|
||||
|
||||
The wait used to sleep inside the scan lock, so whoever asked for the scan
|
||||
— the resume behind a switch toggle, and through it an HTTP request —
|
||||
waited for the whole backlog to clear. A saturated queue never clears by
|
||||
waiting, so that request never came back.
|
||||
"""
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
for index in range(8):
|
||||
task = _task(evidence, tasks, active, index=index)
|
||||
supervisor._queue.add(supervisor._classify([task])[0])
|
||||
|
||||
started = time.monotonic()
|
||||
asyncio.run(
|
||||
asyncio.wait_for(supervisor.backfill_off_thread(cohort_size=8), timeout=3)
|
||||
)
|
||||
assert time.monotonic() - started < 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__]))
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Why a source cannot be read, named separately from whether it can.
|
||||
|
||||
`authorized=false` was one answer to six different situations, so the UI could
|
||||
only offer every recovery at once and people reinstalled plugins for a problem
|
||||
in the source grant (issue #29). These cover the naming, and the capture-side
|
||||
refusal that used to be silent and unrecorded.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.imports.access import SourceAccessError, SourceAccessErrorCode
|
||||
from one_memory.imports.grants import SourceAuthorizationState, SourceGrantStore
|
||||
from one_memory.imports.sources.registry import RootCandidateKind
|
||||
from one_memory.live_capture import (
|
||||
CaptureRejection,
|
||||
capture_rejection,
|
||||
record_capture_rejection,
|
||||
)
|
||||
from one_memory.memory_server.config_state import ConfigStore
|
||||
|
||||
|
||||
def build_store(tmp_path: Path) -> tuple[SourceGrantStore, Path, Path]:
|
||||
home = tmp_path / "home"
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
return SourceGrantStore(database, home=home), database, home
|
||||
|
||||
|
||||
def _grant(store: SourceGrantStore, home: Path, name: str) -> Path:
|
||||
root = home / name
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
store.grant(
|
||||
user_id="local",
|
||||
source_id="workbuddy",
|
||||
root=root,
|
||||
candidate_kind=RootCandidateKind.DEFAULT,
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
def test_no_grant_is_named_missing(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=home / "workbuddy"
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.MISSING_GRANT
|
||||
|
||||
|
||||
def test_a_live_grant_on_the_current_candidate_is_ok(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.OK
|
||||
|
||||
|
||||
def test_a_revoked_grant_is_not_a_missing_one(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
store.revoke(user_id="local", source_id="workbuddy")
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.REVOKED_GRANT
|
||||
|
||||
|
||||
def test_a_rebuilt_directory_is_not_a_moved_candidate(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
root.rmdir()
|
||||
root.mkdir()
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.SOURCE_IDENTITY_CHANGED
|
||||
|
||||
|
||||
def test_a_missing_directory_is_named_unavailable(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
root.rmdir()
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.SOURCE_ROOT_UNAVAILABLE
|
||||
|
||||
|
||||
def test_a_grant_pointing_somewhere_else_is_a_moved_candidate(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
elsewhere = home / "workbuddy-elsewhere"
|
||||
elsewhere.mkdir()
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=elsewhere
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.CANDIDATE_CHANGED
|
||||
|
||||
|
||||
def test_a_grant_still_fails_closed_when_a_state_is_named(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
root.rmdir()
|
||||
root.mkdir()
|
||||
|
||||
# Naming the reason is not granting access: the read path still refuses,
|
||||
# and says so through the same code it always did.
|
||||
with pytest.raises(SourceAccessError) as captured:
|
||||
store.authorized_root(user_id="local", source_id="workbuddy")
|
||||
assert captured.value.code is SourceAccessErrorCode.SOURCE_IDENTITY_CHANGED
|
||||
|
||||
|
||||
def test_capture_rejection_names_sync_disabled(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
|
||||
assert (
|
||||
capture_rejection(database, "workbuddy") is CaptureRejection.SYNC_DISABLED
|
||||
)
|
||||
|
||||
|
||||
def test_capture_rejection_names_a_missing_then_a_revoked_grant(tmp_path) -> None:
|
||||
_store, database, _home = build_store(tmp_path)
|
||||
ConfigStore(database.parent).set_sync("workbuddy", True)
|
||||
|
||||
assert capture_rejection(database, "workbuddy") is CaptureRejection.GRANT_MISSING
|
||||
|
||||
store, _database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
store.revoke(user_id="local", source_id="workbuddy")
|
||||
|
||||
assert capture_rejection(database, "workbuddy") is CaptureRejection.GRANT_REVOKED
|
||||
|
||||
|
||||
def test_capture_rejection_is_none_when_a_turn_may_be_captured(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
ConfigStore(database.parent).set_sync("workbuddy", True)
|
||||
|
||||
assert capture_rejection(database, "workbuddy") is None
|
||||
|
||||
|
||||
def test_the_recorded_rejection_carries_no_turn_and_no_path(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
secret = "深绿色的按钮,不要告诉别人"
|
||||
marker = home / "secret.txt"
|
||||
marker.write_text(secret, encoding="utf8")
|
||||
|
||||
record_capture_rejection(database, "workbuddy", CaptureRejection.SYNC_DISABLED)
|
||||
record_capture_rejection(database, "workbuddy", CaptureRejection.SYNC_DISABLED)
|
||||
|
||||
record = json.loads(
|
||||
(
|
||||
database.parent
|
||||
/ "logs"
|
||||
/ "capture-rejection-workbuddy-sync_disabled.json"
|
||||
).read_text(encoding="utf8")
|
||||
)
|
||||
assert record["count"] == 2
|
||||
assert record["source_id"] == "workbuddy"
|
||||
assert record["reason"] == "sync_disabled"
|
||||
assert secret not in json.dumps(record, ensure_ascii=False)
|
||||
assert str(home) not in json.dumps(record, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_recording_a_rejection_never_breaks_the_host_turn(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
# A data directory that cannot be written must stay silent: this runs
|
||||
# inside somebody's turn, where raising would print into the conversation.
|
||||
blocked = tmp_path / "blocked"
|
||||
blocked.write_text("not a directory", encoding="utf8")
|
||||
|
||||
record_capture_rejection(
|
||||
blocked / "one-memory.sqlite3",
|
||||
"workbuddy",
|
||||
CaptureRejection.GRANT_REVOKED,
|
||||
)
|
||||
@@ -195,11 +195,19 @@ def test_connection_retry_reuses_completed_job_and_checks_root_snapshot(tmp_path
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("session_count", "limited"), [(200, False), (201, True)])
|
||||
@pytest.mark.parametrize(
|
||||
("turn_count", "age_days", "limited"),
|
||||
[
|
||||
(200, 4, False),
|
||||
(201, 1, False),
|
||||
(201, 4, True),
|
||||
],
|
||||
)
|
||||
def test_onboarding_applies_large_history_window_to_all_sources(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
session_count,
|
||||
turn_count,
|
||||
age_days,
|
||||
limited,
|
||||
):
|
||||
monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
|
||||
@@ -210,8 +218,22 @@ def test_onboarding_applies_large_history_window_to_all_sources(
|
||||
codex = home / ".codex/sessions"
|
||||
claude.mkdir(parents=True)
|
||||
codex.mkdir(parents=True)
|
||||
for index in range(session_count):
|
||||
(claude / f"{index:03}.jsonl").write_text("")
|
||||
timestamp = (datetime.now(UTC) - timedelta(days=age_days)).isoformat()
|
||||
(claude / "large-session.jsonl").write_text(
|
||||
"".join(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "user",
|
||||
"sessionId": "large-session",
|
||||
"uuid": f"user-{index}",
|
||||
"timestamp": timestamp,
|
||||
"message": {"content": f"turn {index}"},
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
for index in range(turn_count)
|
||||
)
|
||||
)
|
||||
(codex / "one.jsonl").write_text("")
|
||||
app = create_app(
|
||||
Settings(
|
||||
@@ -240,7 +262,7 @@ def test_onboarding_applies_large_history_window_to_all_sources(
|
||||
"/api/v1/sources/connect",
|
||||
json={
|
||||
"source_ids": ["claude-code", "codex"],
|
||||
"request_id": f"history-window-{session_count}",
|
||||
"request_id": f"history-window-{turn_count}-{age_days}",
|
||||
"source_roots": {
|
||||
"claude-code": claude_root,
|
||||
"codex": codex_root,
|
||||
|
||||
@@ -39,7 +39,11 @@ def record(
|
||||
source_id=source,
|
||||
conversation_id=f"c{index % 2}",
|
||||
message_id=str(index),
|
||||
role="assistant" if author == WrittenBy.AGENT else "user",
|
||||
role=(
|
||||
"assistant"
|
||||
if author == WrittenBy.AGENT
|
||||
else ("tool" if author == WrittenBy.TOOL else "user")
|
||||
),
|
||||
written_by=author,
|
||||
created_at=time,
|
||||
verbatim=text or f"original {index}",
|
||||
@@ -83,8 +87,8 @@ def test_receipt_counts_saved_records_without_inventing_dates_or_other_sources(t
|
||||
data = response.json()
|
||||
assert data["source_id"] == "claude-code"
|
||||
assert data["messages"] == 5 and data["conversations"] == 2
|
||||
assert data["person_messages"] == 2 and data["agent_messages"] == 1
|
||||
assert data["unknown_author_messages"] == 1 and data["tool_messages"] == 1
|
||||
assert data["person_messages"] == 3 and data["agent_messages"] == 1
|
||||
assert data["unknown_author_messages"] == 0 and data["tool_messages"] == 1
|
||||
assert data["unknown_time_messages"] == 4
|
||||
assert len(data["examples"]) == 3
|
||||
assert "excluded-source" not in response.text and "excluded-user" not in response.text
|
||||
|
||||
@@ -34,8 +34,8 @@ def test_a_heartbeat_is_not_a_question():
|
||||
assert last_row_opens_an_unanswered_turn(("user", "person", "<heartbeat>ping")) is False
|
||||
|
||||
|
||||
def test_a_message_written_by_the_agent_as_user_is_not_a_question():
|
||||
assert last_row_opens_an_unanswered_turn(("user", "agent", "继续")) is False
|
||||
def test_protocol_user_role_opens_a_turn_regardless_of_legacy_author_guess():
|
||||
assert last_row_opens_an_unanswered_turn(("user", "agent", "继续")) is True
|
||||
|
||||
|
||||
def test_a_conversation_with_no_rows_is_not_unanswered():
|
||||
|
||||
@@ -124,7 +124,7 @@ def test_the_sdk_strips_tool_messages_on_the_chat_path():
|
||||
|
||||
|
||||
def test_the_sdk_still_cuts_tool_results_the_way_this_assumes():
|
||||
"""D74 counts 2 KiB per tool record because the SDK cuts there.
|
||||
"""MemHub's window estimate follows its configured SDK tool ceiling.
|
||||
|
||||
If an SDK upgrade moves that boundary or stops dropping tool records on
|
||||
the chat path, `_HY_TOOL_CHARACTERS` no longer describes what HY reads and
|
||||
@@ -141,14 +141,14 @@ def test_the_sdk_still_cuts_tool_results_the_way_this_assumes():
|
||||
from one_memory.memory_processing import _HY_TOOL_CHARACTERS
|
||||
|
||||
assert DEFAULT_MAX_TOOL_RESULT_BYTES == 2048
|
||||
assert _HY_TOOL_CHARACTERS == 2048
|
||||
assert _HY_TOOL_CHARACTERS == 512
|
||||
long = "z" * 20_000
|
||||
cut = truncate_tool_result_text(long)
|
||||
assert len(cut) <= DEFAULT_MAX_TOOL_RESULT_BYTES + 128
|
||||
cut = truncate_tool_result_text(long, max_bytes=_HY_TOOL_CHARACTERS)
|
||||
assert len(cut.encode()) <= _HY_TOOL_CHARACTERS + 128
|
||||
|
||||
from hy_memory.coding.preproc import truncate_messages
|
||||
|
||||
messages = [ChatMessage(role="tool", content=long)]
|
||||
truncated = truncate_messages(messages)
|
||||
truncated = truncate_messages(messages, max_bytes=_HY_TOOL_CHARACTERS)
|
||||
assert sum(len(item.content or "") for item in truncated) < len(long)
|
||||
assert _HY_TOOL_CHARACTERS >= DEFAULT_HEAD_BYTES + DEFAULT_TAIL_BYTES
|
||||
assert _HY_TOOL_CHARACTERS < DEFAULT_HEAD_BYTES + DEFAULT_TAIL_BYTES
|
||||
Generated
+5
-5
@@ -991,7 +991,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "hy-memory"
|
||||
version = "2.0.3"
|
||||
version = "2.0.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "chromadb" },
|
||||
@@ -1008,9 +1008,9 @@ dependencies = [
|
||||
{ name = "scikit-learn" },
|
||||
{ name = "tiktoken" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/12/10e91468d1331eff8380e6b32c1e8784413660a299005ce07ac3e60e90bd/hy_memory-2.0.3.tar.gz", hash = "sha256:cdebd46b16ebe3cb04c89475f9044d30c6cc9065796be369f5d9607410a5c96b", size = 823596, upload-time = "2026-10-17T11:06:23.093Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1f/d0/44af626b3991a49f787074ebf7474ab68d7adf93a048555ffcdd3092d215/hy_memory-2.0.4.tar.gz", hash = "sha256:ae68837a55afdd50143003608e586350da36c5bd90f4dafcf460c34377179a11", size = 824206, upload-time = "2026-10-17T13:47:29.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/af/ed/bac8c15eb91baeaeb15a9061d7937ffd02ebe3384757addf0ca09660e592/hy_memory-2.0.3-py3-none-any.whl", hash = "sha256:763d6388ea83762e06c51aecd8e5d8e76211000f67355f65a760eb0a17ede4c5", size = 887157, upload-time = "2026-10-17T11:06:20.994Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/68/1c6512a7c3c41cad2731df40416a5331cbc95763feb4983fb59a91869cc7/hy_memory-2.0.4-py3-none-any.whl", hash = "sha256:1d6fd4120185cc6cbcf04f9d81d5bf47e33c467f52787e5d175a412d4e687cd1", size = 888026, upload-time = "2026-10-17T13:47:27.062Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1801,7 +1801,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "one-memory"
|
||||
version = "1.0.3"
|
||||
version = "1.0.4"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiomysql" },
|
||||
@@ -1837,7 +1837,7 @@ requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.141.1" },
|
||||
{ name = "filelock", specifier = ">=3.32.4" },
|
||||
{ name = "httpx", specifier = ">=0.28.1" },
|
||||
{ name = "hy-memory", specifier = "==2.0.3" },
|
||||
{ name = "hy-memory", specifier = "==2.0.4" },
|
||||
{ name = "keyring", specifier = ">=25.7.0" },
|
||||
{ name = "numpy", specifier = "==2.5.2" },
|
||||
{ name = "onnxruntime", specifier = "==1.23.2" },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@one-memory/desktop",
|
||||
"version": "1.0.3",
|
||||
"version": "1.0.4",
|
||||
"private": true,
|
||||
"main": "dist/main.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { resolveWebResource } from "./app-protocol";
|
||||
import {
|
||||
isStartupScreenRequest,
|
||||
resolveWebResource,
|
||||
STARTUP_SCREEN_PATH,
|
||||
} from "./app-protocol";
|
||||
import { STARTUP_SCREEN_URL } from "./startup-screen";
|
||||
|
||||
describe("packaged web protocol", () => {
|
||||
it("serves only files inside the packaged web root", () => {
|
||||
@@ -20,4 +25,14 @@ describe("packaged web protocol", () => {
|
||||
resolveWebResource(root, "one-memory://app/%2F..%2Fsecret"),
|
||||
).toBeNull();
|
||||
});
|
||||
it("recognizes the startup route and never maps it onto a file", () => {
|
||||
const root = path.resolve("/bundle/web");
|
||||
expect(isStartupScreenRequest(STARTUP_SCREEN_URL)).toBe(true);
|
||||
expect(isStartupScreenRequest("one-memory://app/index.html")).toBe(false);
|
||||
expect(isStartupScreenRequest("one-memory://other/__startup")).toBe(false);
|
||||
// The placeholder is served from memory, so a file of that name inside the
|
||||
// web bundle must never be reachable through it.
|
||||
expect(resolveWebResource(root, STARTUP_SCREEN_URL)).toBeNull();
|
||||
expect(STARTUP_SCREEN_PATH).toBe("/__startup");
|
||||
});
|
||||
});
|
||||
@@ -5,12 +5,28 @@ import { net, protocol } from "electron";
|
||||
|
||||
export const APP_ORIGIN = "one-memory://app";
|
||||
|
||||
/** The route the startup placeholder is served from. It is never a file: the
|
||||
* placeholder has to load before anything in the web bundle is trusted. */
|
||||
export const STARTUP_SCREEN_PATH = "/__startup";
|
||||
|
||||
export function isStartupScreenRequest(requestUrl: string): boolean {
|
||||
const url = new URL(requestUrl);
|
||||
return (
|
||||
url.protocol === "one-memory:" &&
|
||||
url.host === "app" &&
|
||||
url.pathname === STARTUP_SCREEN_PATH
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveWebResource(
|
||||
webRoot: string,
|
||||
requestUrl: string,
|
||||
): string | null {
|
||||
const url = new URL(requestUrl);
|
||||
if (url.protocol !== "one-memory:" || url.host !== "app") return null;
|
||||
// Reserved for the in-memory startup placeholder: a file of that name in the
|
||||
// web bundle is not the placeholder and must not be reachable as one.
|
||||
if (isStartupScreenRequest(requestUrl)) return null;
|
||||
const pathname = decodeURIComponent(url.pathname);
|
||||
const requested = pathname === "/" ? "index.html" : pathname.slice(1);
|
||||
const resolvedRoot = path.resolve(webRoot);
|
||||
@@ -22,8 +38,17 @@ export function resolveWebResource(
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export function installAppProtocol(webRoot: string): void {
|
||||
export function installAppProtocol(
|
||||
webRoot: string,
|
||||
options: { startupScreen?: () => string } = {},
|
||||
): void {
|
||||
protocol.handle("one-memory", (request) => {
|
||||
if (options.startupScreen && isStartupScreenRequest(request.url)) {
|
||||
return new Response(options.startupScreen(), {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html; charset=utf-8" },
|
||||
});
|
||||
}
|
||||
const resource = resolveWebResource(webRoot, request.url);
|
||||
if (!resource) {
|
||||
return new Response("Not found", { status: 404 });
|
||||
|
||||
@@ -50,6 +50,10 @@ interface Overrides {
|
||||
whenReady?: Promise<void>;
|
||||
windowGate?: Promise<void>;
|
||||
windowError?: Error;
|
||||
/** Give the lifecycle a startup placeholder, as the packaged App does. */
|
||||
startupWindow?: boolean;
|
||||
startupWindowError?: Error;
|
||||
revealError?: Error;
|
||||
}
|
||||
|
||||
function fixture(background = false, lock = true, overrides: Overrides = {}) {
|
||||
@@ -71,6 +75,15 @@ function fixture(background = false, lock = true, overrides: Overrides = {}) {
|
||||
if (overrides.windowError) throw overrides.windowError;
|
||||
windows.push(liveWindow());
|
||||
});
|
||||
// The startup placeholder is a window the lifecycle raises before the
|
||||
// services exist, then hands back to be swapped onto the real app.
|
||||
const showStartupWindow = vi.fn(async () => {
|
||||
if (overrides.startupWindowError) throw overrides.startupWindowError;
|
||||
windows.push(liveWindow());
|
||||
});
|
||||
const revealStartupWindow = vi.fn(async () => {
|
||||
if (overrides.revealError) throw overrides.revealError;
|
||||
});
|
||||
const prepare = vi.fn(() => {
|
||||
if (overrides.prepareError) throw overrides.prepareError;
|
||||
});
|
||||
@@ -84,6 +97,9 @@ function fixture(background = false, lock = true, overrides: Overrides = {}) {
|
||||
initialize,
|
||||
stop,
|
||||
dispose,
|
||||
...(overrides.startupWindow
|
||||
? { showStartupWindow, revealStartupWindow }
|
||||
: {}),
|
||||
createWindow,
|
||||
getWindows: () => windows,
|
||||
installTray,
|
||||
@@ -118,6 +134,9 @@ function fixture(background = false, lock = true, overrides: Overrides = {}) {
|
||||
dispose,
|
||||
windows,
|
||||
createWindow,
|
||||
showStartupWindow,
|
||||
revealStartupWindow,
|
||||
startupWindow: overrides.startupWindow === true,
|
||||
prepare,
|
||||
installTray,
|
||||
onError,
|
||||
@@ -218,6 +237,76 @@ describe("desktop background lifecycle with injected Electron", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("desktop lifecycle startup window", () => {
|
||||
it("raises the placeholder before initialization and swaps it after", async () => {
|
||||
const f = fixture(false, true, { startupWindow: true });
|
||||
f.ready.resolve();
|
||||
await f.flush();
|
||||
// The whole point: a window exists while `initialize` is still pending,
|
||||
// because that is the part that takes tens of seconds.
|
||||
expect(f.initialize).toHaveBeenCalledTimes(1);
|
||||
expect(f.showStartupWindow).toHaveBeenCalledTimes(1);
|
||||
expect(f.windows).toHaveLength(1);
|
||||
expect(f.revealStartupWindow).not.toHaveBeenCalled();
|
||||
expect(f.createWindow).not.toHaveBeenCalled();
|
||||
f.starting.resolve();
|
||||
await f.flush();
|
||||
expect(f.revealStartupWindow).toHaveBeenCalledTimes(1);
|
||||
// The placeholder is reused, not replaced: two windows would flash.
|
||||
expect(f.createWindow).not.toHaveBeenCalled();
|
||||
expect(f.windows).toHaveLength(1);
|
||||
expect(f.windows[0].show).toHaveBeenCalled();
|
||||
});
|
||||
it("keeps no placeholder for a background launch", async () => {
|
||||
const f = fixture(true, true, { startupWindow: true });
|
||||
await f.boot();
|
||||
expect(f.initialize).toHaveBeenCalledTimes(1);
|
||||
expect(f.showStartupWindow).not.toHaveBeenCalled();
|
||||
expect(f.revealStartupWindow).not.toHaveBeenCalled();
|
||||
expect(f.windows).toHaveLength(0);
|
||||
});
|
||||
it("opens the real app when the placeholder was closed during startup", async () => {
|
||||
const f = fixture(false, true, { startupWindow: true });
|
||||
f.ready.resolve();
|
||||
await f.flush();
|
||||
expect(f.windows).toHaveLength(1);
|
||||
f.windows.length = 0;
|
||||
f.starting.resolve();
|
||||
await f.flush();
|
||||
expect(f.revealStartupWindow).toHaveBeenCalledTimes(1);
|
||||
expect(f.createWindow).toHaveBeenCalledTimes(1);
|
||||
expect(f.windows).toHaveLength(1);
|
||||
});
|
||||
it("quitting during initialization leaves the placeholder unrevealed", async () => {
|
||||
const f = fixture(false, true, { startupWindow: true });
|
||||
f.ready.resolve();
|
||||
await f.flush();
|
||||
expect(f.showStartupWindow).toHaveBeenCalledTimes(1);
|
||||
f.quit();
|
||||
f.starting.resolve();
|
||||
await f.flush();
|
||||
expect(f.revealStartupWindow).not.toHaveBeenCalled();
|
||||
expect(f.createWindow).not.toHaveBeenCalled();
|
||||
});
|
||||
it("still opens the real app when the placeholder cannot be raised", async () => {
|
||||
const f = fixture(false, true, {
|
||||
startupWindow: true,
|
||||
startupWindowError: new Error("startup_window_unavailable"),
|
||||
});
|
||||
f.ready.resolve();
|
||||
await f.flush();
|
||||
// The placeholder is a courtesy, not a dependency: a launch that cannot
|
||||
// draw it must still bring up the services and the real window.
|
||||
expect(f.initialize).toHaveBeenCalledTimes(1);
|
||||
expect(f.onError).not.toHaveBeenCalled();
|
||||
f.starting.resolve();
|
||||
await f.flush();
|
||||
expect(f.revealStartupWindow).not.toHaveBeenCalled();
|
||||
expect(f.createWindow).toHaveBeenCalledTimes(1);
|
||||
expect(f.windows).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("desktop lifecycle failure paths", () => {
|
||||
it("a failed start is reported once and opens no window on any request", async () => {
|
||||
const f = fixture();
|
||||
|
||||
Loaded 100 of 166 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user