Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5c6c63f7d3 |
No files matched your search
@@ -1123,7 +1123,6 @@ class Extractor:
|
||||
max_tokens=self._llm_config.agent_max_tokens,
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
audit_stage="extract",
|
||||
)
|
||||
self._call_count += 1
|
||||
total_tokens += response.tokens_used
|
||||
@@ -1234,7 +1233,6 @@ class Extractor:
|
||||
max_tokens=self._llm_config.agent_max_tokens,
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
audit_stage="extract",
|
||||
)
|
||||
self._call_count += 1
|
||||
total_tokens += response.tokens_used
|
||||
|
||||
@@ -34,7 +34,6 @@ import os
|
||||
import weakref
|
||||
|
||||
from ..config import MemoryConfig
|
||||
from ..utils.model_call_context import model_call_stage
|
||||
from ..core.workload import llm_slot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -349,7 +348,6 @@ class LLMProvider:
|
||||
stop: List[str] = None,
|
||||
tools: Optional[List[Dict[str, Any]]] = None,
|
||||
tool_choice: Optional[Any] = None,
|
||||
audit_stage: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> LLMResponse:
|
||||
"""
|
||||
@@ -388,16 +386,15 @@ class LLMProvider:
|
||||
len(prompt), _content_fingerprint(prompt),
|
||||
)
|
||||
|
||||
with model_call_stage(audit_stage):
|
||||
response = await self._call_llm(
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
stop=stop,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
**kwargs
|
||||
)
|
||||
response = await self._call_llm(
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
stop=stop,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
self._total_calls += 1
|
||||
self._total_tokens += response.tokens_used
|
||||
@@ -598,7 +595,6 @@ class LLMProvider:
|
||||
temperature: Optional[float] = None,
|
||||
tools: Optional[List[Dict[str, Any]]] = None,
|
||||
tool_choice: Optional[Any] = None,
|
||||
audit_stage: Optional[str] = None,
|
||||
) -> LLMResponse:
|
||||
"""
|
||||
多轮 messages 调用(支持 tool results 回传)。
|
||||
@@ -626,14 +622,13 @@ class LLMProvider:
|
||||
f"msgs={len(messages)} tools={len(tools) if tools else 0}"
|
||||
)
|
||||
|
||||
with model_call_stage(audit_stage):
|
||||
response = await self._chat_completions(
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
response = await self._chat_completions(
|
||||
messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
|
||||
self._total_calls += 1
|
||||
self._total_tokens += response.tokens_used
|
||||
|
||||
@@ -669,7 +669,6 @@ class MemoryReconciler:
|
||||
response = await llm.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=500,
|
||||
audit_stage="reconcile",
|
||||
)
|
||||
text = response.content.strip()
|
||||
if "```json" in text:
|
||||
@@ -1384,7 +1383,6 @@ class MemoryReconciler:
|
||||
response = await llm.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=dyn_max_tokens,
|
||||
audit_stage="reconcile",
|
||||
)
|
||||
except Exception as e:
|
||||
# LLM 彻底不可用(provider 已按 max_retries 重试过,网关 500 /
|
||||
|
||||
@@ -919,7 +919,6 @@ class CodingMemoryExtractor:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
audit_stage="coding_extract",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[coding-extract] LLM call failed: {e}")
|
||||
|
||||
@@ -571,7 +571,6 @@ class WorkingFactReconciler:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
audit_stage="coding_fact_reconcile",
|
||||
)
|
||||
return self._parse_ops(resp.content)
|
||||
|
||||
@@ -638,7 +637,6 @@ class WorkingFactReconciler:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=_CHAIN_SUMMARY_MAX_TOKENS,
|
||||
audit_stage="coding_fact_summary",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[fact-reconcile] chain summary failed: {e}")
|
||||
|
||||
@@ -29,9 +29,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_JUDGE_MAX_TURNS = 20
|
||||
_JUDGE_HEAD_TURNS = 5
|
||||
_JUDGE_USER_CONTENT_LIMIT = 1_000
|
||||
_JUDGE_USER_CONTENT_HEAD = 200
|
||||
_JUDGE_USER_CONTENT_TAIL = 200
|
||||
_JUDGE_MAX_QUERY_CHARS = 300
|
||||
_JUDGE_MAX_QUERIES = 20
|
||||
|
||||
@@ -189,7 +186,6 @@ async def classify_messages_scene(
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
audit_stage="coding_judge",
|
||||
)
|
||||
_elapsed = round((time.perf_counter() - _t0) * 1000, 2)
|
||||
result = _parse_scene_json(resp.content)
|
||||
@@ -635,13 +631,9 @@ def _format_turns(summary: List[Dict[str, Any]]) -> str:
|
||||
rows = []
|
||||
for t in summary:
|
||||
user = t.get("user", "") or ""
|
||||
if len(user) > _JUDGE_USER_CONTENT_LIMIT:
|
||||
omitted = len(user) - _JUDGE_USER_CONTENT_HEAD - _JUDGE_USER_CONTENT_TAIL
|
||||
user = (
|
||||
user[:_JUDGE_USER_CONTENT_HEAD]
|
||||
+ f"\n[...{omitted} chars omitted...]\n"
|
||||
+ user[-_JUDGE_USER_CONTENT_TAIL:]
|
||||
)
|
||||
# 单 turn 用户文本截到 200 字符
|
||||
if len(user) > 200:
|
||||
user = user[:200] + "..."
|
||||
tools = t.get("tools", []) or []
|
||||
rows.append(
|
||||
f"- turn {t.get('turn', '?')}: user={user!r}, tools={tools}"
|
||||
|
||||
@@ -310,7 +310,6 @@ class CodingMemoryReconciler:
|
||||
resp = await self.llm_provider.complete(
|
||||
prompt=prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
audit_stage="coding_reconcile",
|
||||
)
|
||||
return self._parse_ops(resp.content), prompt, resp.content, resp
|
||||
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
"""Context propagated to host-provided HTTP transports for cost accounting."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar, Token
|
||||
|
||||
_model_call_stage: ContextVar[str] = ContextVar(
|
||||
"hy_memory_model_call_stage",
|
||||
default="other",
|
||||
)
|
||||
|
||||
|
||||
def current_model_call_stage() -> str:
|
||||
return _model_call_stage.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def model_call_stage(stage: str | None) -> Iterator[None]:
|
||||
normalized = (stage or "other").strip().lower() or "other"
|
||||
token: Token[str] = _model_call_stage.set(normalized)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_model_call_stage.reset(token)
|
||||
@@ -80,11 +80,7 @@ class PipelineLogWriter:
|
||||
split_namespace_user,
|
||||
)
|
||||
|
||||
# Providers define total_tokens as prompt+completion. Adding all
|
||||
# three fields doubled every cost estimate in this JSONL.
|
||||
tokens = (total_tokens or 0) or (
|
||||
(prompt_tokens or 0) + (completion_tokens or 0)
|
||||
)
|
||||
tokens = (prompt_tokens or 0) + (completion_tokens or 0) + (total_tokens or 0)
|
||||
namespace, uid = split_namespace_user(user_id or "")
|
||||
if not is_llm_content_logging_enabled():
|
||||
prompt = redact_content(prompt)
|
||||
|
||||
+4
-16
@@ -1,20 +1,8 @@
|
||||
# MemHub
|
||||
# MemHub / One Memory
|
||||
|
||||
MemHub是一个本地优先的共享记忆产品。它把用户明确授权的AI对话和编码记录汇总到一份本地记忆中,再由用户分别授权给支持的AI工具调用。你可以继续使用熟悉的客户端,同时保留跨工具延续的上下文。Hy Memory是MemHub使用的Python SDK和记忆引擎。
|
||||
|
||||
# 💡❤️ MemHub 刚刚开源,欢迎一起优化共建! ❤️💡
|
||||
|
||||
**# 💭 MemHub处于preview阶段,当前公开版本可以下载和体验,部分平台制品、安装路径与宿主体验仍在持续优化验证。安装前请先查看Release里的实际制品和版本说明。**
|
||||
|
||||
> 🙋 如果你遇到问题/有建议:欢迎提交Issue,并附上MemHub版本、操作系统与芯片架构、复现步骤、预期结果、实际结果和经过脱敏的日志。请不要上传APIKey、真实对话、个人路径或其他敏感信息。
|
||||
|
||||
> 我们会公开跟进可复现的问题,持续发布修复,并根据真实反馈完善安装、同步和召回体验。
|
||||
|
||||
**# 感谢每一条反馈,期待你和我们一起打磨MemHub的体验 :)**
|
||||
|
||||
> 💗 MemHub has just been open-sourced and is under active development. The current public release is available for download andevaluation. Some platform packages, installation paths, and host integrations are still being completed and verified. Check the assets
|
||||
and release notes before installing.
|
||||
If you run into a problem, please open an Issue with the version, platform,reproduction steps, and sanitized logs, We wilreviewreports, publish fixes, and keep the support matrix up to date.
|
||||
MemHub 是一个本地优先的共享记忆产品:把用户授权的多个 AI 产品中的历史对话和新增内容
|
||||
汇总到同一份记忆,并让这些 AI 在独立授权后复用它。Hy Memory 是 MemHub 使用的 Python
|
||||
SDK 和记忆引擎。One Memory / `one-memory` 是历史模块、数据目录、服务和兼容命令名称。
|
||||
|
||||
## 发行与平台合同
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
"dataLicense": "CC0-1.0",
|
||||
"SPDXID": "SPDXRef-DOCUMENT",
|
||||
"name": "one-memory",
|
||||
"documentNamespace": "https://one-memory.local/spdx/2026-10-17T13:52:48Z",
|
||||
"documentNamespace": "https://one-memory.local/spdx/2026-10-17T11:07:57Z",
|
||||
"creationInfo": {
|
||||
"created": "2026-10-17T13:52:48Z",
|
||||
"created": "2026-10-17T11:07:57Z",
|
||||
"creators": [
|
||||
"Tool: one-memory-generate-sbom"
|
||||
]
|
||||
@@ -2335,7 +2335,7 @@
|
||||
{
|
||||
"SPDXID": "SPDXRef-Package-hy-memory",
|
||||
"name": "hy-memory",
|
||||
"versionInfo": "2.0.4",
|
||||
"versionInfo": "2.0.3",
|
||||
"downloadLocation": "NOASSERTION",
|
||||
"filesAnalyzed": false,
|
||||
"licenseConcluded": "Apache-2.0",
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
MemHub itself is licensed under Apache-2.0 (see `LICENSE`).
|
||||
It bundles the runtime dependencies listed below.
|
||||
Generated from the installed dependency tree on 2026-10-17T13:52:48Z
|
||||
Generated from the installed dependency tree on 2026-10-17T11:07:57Z
|
||||
by `python3 scripts/generate_sbom.py`. Do not edit by hand.
|
||||
|
||||
Per-package license texts are not reproduced here yet; the license
|
||||
column is filled in as part of release review, and the SBOM records
|
||||
`NOASSERTION` until then rather than guessing.
|
||||
|
||||
## hy-memory 2.0.4
|
||||
## hy-memory 2.0.3
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@one-memory/cli",
|
||||
"version": "1.0.4",
|
||||
"version": "1.0.3",
|
||||
"description": "MemHub bootstrap CLI for installing and managing the local-first shared memory runtime",
|
||||
"license": "Apache-2.0",
|
||||
"repository": {
|
||||
|
||||
@@ -384,101 +384,6 @@ describe("unified context search", () => {
|
||||
expect(streams.stdout()).not.toContain("must-not-print");
|
||||
});
|
||||
|
||||
it("counts characters as code points so an emoji hit stays grounded", async () => {
|
||||
// Core reports used_characters with Python's len(), which counts code
|
||||
// points: this text is 10 characters there and 12 UTF-16 units here.
|
||||
const text = "消费保 📍 扩展 📡";
|
||||
expect(text.length).toBe(12);
|
||||
expect(Array.from(text).length).toBe(10);
|
||||
transport.post.mockResolvedValue({
|
||||
status: 200,
|
||||
data: {
|
||||
...RESPONSE,
|
||||
paths: {
|
||||
...RESPONSE.paths,
|
||||
memory: {
|
||||
...RESPONSE.paths.memory,
|
||||
status: "empty",
|
||||
returned_count: 0,
|
||||
},
|
||||
},
|
||||
items: [
|
||||
{
|
||||
...(RESPONSE.items[0] as Record<string, unknown>),
|
||||
messages: [
|
||||
{
|
||||
...(
|
||||
RESPONSE.items[0] as {
|
||||
messages: Array<Record<string, unknown>>;
|
||||
}
|
||||
).messages[0],
|
||||
text,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
used_characters: 10,
|
||||
},
|
||||
});
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
expect(await run(["recall", "test", "--json"])).toBe(0);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(JSON.parse(streams.stdout())).toMatchObject({
|
||||
state: "grounded",
|
||||
used_characters: 10,
|
||||
items: [{ kind: "raw", messages: [{ text }] }],
|
||||
});
|
||||
});
|
||||
|
||||
it("still rejects a response whose character total does not match", async () => {
|
||||
// The same text counted as UTF-16 units is exactly what Core does not mean,
|
||||
// so the cross-check must keep failing instead of accepting both totals.
|
||||
const text = "消费保 📍 扩展 📡";
|
||||
transport.post.mockResolvedValue({
|
||||
status: 200,
|
||||
data: {
|
||||
...RESPONSE,
|
||||
paths: {
|
||||
...RESPONSE.paths,
|
||||
memory: {
|
||||
...RESPONSE.paths.memory,
|
||||
status: "empty",
|
||||
returned_count: 0,
|
||||
},
|
||||
},
|
||||
items: [
|
||||
{
|
||||
...(RESPONSE.items[0] as Record<string, unknown>),
|
||||
messages: [
|
||||
{
|
||||
...(
|
||||
RESPONSE.items[0] as {
|
||||
messages: Array<Record<string, unknown>>;
|
||||
}
|
||||
).messages[0],
|
||||
text,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
used_characters: 12,
|
||||
},
|
||||
});
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
expect(await run(["recall", "test", "--json"])).toBe(1);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(JSON.parse(streams.stdout())).toMatchObject({
|
||||
state: "invalid",
|
||||
code: "response_invalid",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports endpoint denial without exposing the response body", async () => {
|
||||
transport.post.mockResolvedValue({
|
||||
status: 403,
|
||||
|
||||
@@ -181,9 +181,9 @@ function validateContextResponse(
|
||||
items.push(item);
|
||||
characters +=
|
||||
item.kind === "memory"
|
||||
? characterCount(item.content)
|
||||
? String(item.content).length
|
||||
: (item.messages as Array<Record<string, unknown>>).reduce(
|
||||
(sum, message) => sum + characterCount(message.text),
|
||||
(sum, message) => sum + String(message.text).length,
|
||||
0,
|
||||
);
|
||||
}
|
||||
@@ -330,17 +330,6 @@ function record(value: unknown): Record<string, unknown> | null {
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count characters the way Core does. Python's len() counts Unicode code
|
||||
* points; String.prototype.length counts UTF-16 code units, so an emoji outside
|
||||
* the BMP is one character to Core and two here. Comparing the two totals made
|
||||
* every response containing an emoji fail the used_characters cross-check and
|
||||
* surface as response_invalid.
|
||||
*/
|
||||
function characterCount(value: unknown): number {
|
||||
return Array.from(String(value)).length;
|
||||
}
|
||||
|
||||
function boundedText(
|
||||
value: unknown,
|
||||
maximum: number,
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
||||
|
||||
[project]
|
||||
name = "one-memory"
|
||||
version = "1.0.4"
|
||||
version = "1.0.3"
|
||||
description = "MemHub local-first memory core"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
@@ -12,7 +12,7 @@ dependencies = [
|
||||
"fastapi>=0.141.1",
|
||||
"filelock>=3.32.4",
|
||||
# Public SDK version validated against MemHub's permission and recall contracts.
|
||||
"hy-memory==2.0.4",
|
||||
"hy-memory==2.0.3",
|
||||
# Retrieval's derived vector index directly consumes these locked versions.
|
||||
"chromadb==1.5.9",
|
||||
"numpy==2.5.2",
|
||||
|
||||
@@ -100,13 +100,7 @@ from one_memory.imports.access import (
|
||||
SourceAccessError,
|
||||
root_identity_matches,
|
||||
)
|
||||
from one_memory.imports.adapters import (
|
||||
ScanFailure,
|
||||
ScannedEvidence,
|
||||
ScanOptions,
|
||||
SourceAdapter,
|
||||
SyncCompleteness,
|
||||
)
|
||||
from one_memory.imports.adapters import SourceAdapter, SyncCompleteness
|
||||
from one_memory.imports.candidates import (
|
||||
SourceCandidateError,
|
||||
SourceCandidateSelection,
|
||||
@@ -117,11 +111,7 @@ from one_memory.imports.connections import (
|
||||
ConnectionIntentReceipt,
|
||||
SourceConnectionStore,
|
||||
)
|
||||
from one_memory.imports.grants import (
|
||||
SourceAuthorizationState,
|
||||
SourceGrantReceipt,
|
||||
SourceGrantStore,
|
||||
)
|
||||
from one_memory.imports.grants import SourceGrantReceipt, SourceGrantStore
|
||||
from one_memory.imports.host_packages import CapabilityState, HostPackageCapabilities
|
||||
from one_memory.imports.jobs import (
|
||||
ImportIssueRecord,
|
||||
@@ -200,7 +190,6 @@ from one_memory.model_profiles import (
|
||||
ModelProfileStore,
|
||||
ModelProfileUnavailableError,
|
||||
)
|
||||
from one_memory.model_request_audit import AuditedModelTransport, ModelRequestAuditStore
|
||||
from one_memory.model_request_budget import BudgetedModelTransport, ModelRequestBudget
|
||||
from one_memory.onboarding import OnboardingState, OnboardingStore
|
||||
from one_memory.recall import GovernedRecallTarget, RecallReceipt, RecallRequest
|
||||
@@ -292,9 +281,6 @@ class SourceResponse(BaseModel):
|
||||
candidate_revision: int = 0
|
||||
candidate_change_allowed: bool = False
|
||||
candidate_change_reason: str | None = None
|
||||
#: Why `authorized` is false, so the UI can ask for the one thing that
|
||||
#: would fix it instead of offering every recovery at once.
|
||||
authorization_state: SourceAuthorizationState = SourceAuthorizationState.OK
|
||||
|
||||
|
||||
class PreviewSourceCandidateRequest(BaseModel):
|
||||
@@ -327,22 +313,6 @@ class SourceSyncState(BaseModel):
|
||||
sync_enabled: bool
|
||||
|
||||
|
||||
class SedimentationRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
enabled: StrictBool
|
||||
|
||||
|
||||
class SedimentationState(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
enabled: bool
|
||||
#: What the switch means for the model bill and for what still runs. Read
|
||||
#: by the app so the setting can say so without hardcoding the wording.
|
||||
pending_retained: bool = True
|
||||
imports_unaffected: bool = True
|
||||
|
||||
|
||||
class DetectionResponse(BaseModel):
|
||||
source_id: str
|
||||
detected: bool
|
||||
@@ -369,7 +339,7 @@ class ConnectSourcesRequest(BaseModel):
|
||||
import_upload_id: UUID | None = None
|
||||
|
||||
|
||||
ONBOARDING_LARGE_HISTORY_USER_TURNS = 200
|
||||
ONBOARDING_LARGE_HISTORY_SESSIONS = 200
|
||||
ONBOARDING_HISTORY_WINDOW = timedelta(days=3)
|
||||
|
||||
|
||||
@@ -916,7 +886,6 @@ def create_app(
|
||||
)
|
||||
app.state.import_job_store = ImportJobStore(database_path)
|
||||
app.state.onboarding_store = OnboardingStore(database_path)
|
||||
app.state.history_import_gate_released = False
|
||||
app.state.extraction_settings_store = ExtractionSettingsStore(database_path)
|
||||
app.state.source_grant_store = SourceGrantStore(
|
||||
database_path,
|
||||
@@ -939,7 +908,6 @@ def create_app(
|
||||
app.state.source_permission_locks = {
|
||||
source_id: asyncio.Lock() for source_id in app.state.source_catalog
|
||||
}
|
||||
app.state.sedimentation_lock = asyncio.Lock()
|
||||
_follow_workbuddy_home(app, runtime_settings.source_home)
|
||||
app.state.source_detection = {}
|
||||
app.state.secret_store = secret_store or KeyringSecretStore()
|
||||
@@ -1006,7 +974,6 @@ def create_app(
|
||||
else None
|
||||
)
|
||||
app.state.model_request_budget = model_request_budget
|
||||
app.state.model_request_audit_store = ModelRequestAuditStore(database_path)
|
||||
app.state.memory_answerer = MemoryAnswerer(
|
||||
governance=app.state.memory_governance_store,
|
||||
filter_memories=app.state.source_read_access.memories,
|
||||
@@ -1049,17 +1016,10 @@ def create_app(
|
||||
profile=profile,
|
||||
api_key=api_key,
|
||||
data_dir=data_dir,
|
||||
http_transport_factory=lambda: AuditedModelTransport(
|
||||
app.state.model_request_audit_store,
|
||||
profile_id=profile.profile_id,
|
||||
profile_revision=profile.revision,
|
||||
provider_kind=str(profile.provider_kind),
|
||||
model=profile.model,
|
||||
inner=(
|
||||
BudgetedModelTransport(model_request_budget)
|
||||
if model_request_budget is not None
|
||||
else None
|
||||
),
|
||||
http_transport_factory=(
|
||||
(lambda: BudgetedModelTransport(model_request_budget))
|
||||
if model_request_budget is not None
|
||||
else None
|
||||
),
|
||||
),
|
||||
# `None` means "whatever the active profile allows". Pinned to
|
||||
@@ -1072,7 +1032,6 @@ def create_app(
|
||||
max_concurrent_tasks=runtime_settings.precipitation_max_concurrent_tasks,
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
service_levels=app.state.service_levels,
|
||||
sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(),
|
||||
)
|
||||
# C2 owns the single production budget (handle 5s + reserve 250ms,
|
||||
# total 5.25s, lease 10s, renew 1s, claim 250ms). The composition
|
||||
@@ -1105,12 +1064,8 @@ def create_app(
|
||||
mark_invoked: Callable[[], Awaitable[bool]],
|
||||
) -> bool:
|
||||
"""Serialize pause with the last boundary before an HY call."""
|
||||
source_lock = app.state.source_permission_locks.setdefault(
|
||||
source_id, asyncio.Lock()
|
||||
)
|
||||
async with app.state.sedimentation_lock, source_lock:
|
||||
if not _config_store(app).sedimentation_enabled():
|
||||
raise RuntimeError("model_sedimentation_disabled")
|
||||
lock = app.state.source_permission_locks.setdefault(source_id, asyncio.Lock())
|
||||
async with lock:
|
||||
if not _source_writes_on(app, source_id):
|
||||
raise RuntimeError("model_content_source_disabled")
|
||||
return await mark_invoked()
|
||||
@@ -1148,18 +1103,8 @@ def create_app(
|
||||
+ 10
|
||||
),
|
||||
reconcile_unknown=reconcile_unknown_writes,
|
||||
source_writes_permitted=lambda source_id: _sedimentation_claim_enabled(
|
||||
app, source_id
|
||||
),
|
||||
# The claim gate has to be the same question the write gate
|
||||
# asks, not a second answer to it. It used to be
|
||||
# `sync_enabled`, the per-app toggle, and `one-memory-manual` is
|
||||
# not a connected app: it has no entry in the configuration
|
||||
# document, so that predicate answers False for it while the
|
||||
# write gate admits it by its grant. Every manual note's job was
|
||||
# therefore skipped on every drain tick — the note stayed
|
||||
# "saved, waiting" with no task behind it, forever.
|
||||
source_claim_enabled=lambda source_id: _sedimentation_claim_enabled(app, source_id),
|
||||
source_writes_permitted=lambda source_id: _source_writes_on(app, source_id),
|
||||
source_claim_enabled=lambda source_id: _config_store(app).sync_enabled(source_id),
|
||||
)
|
||||
app.state.memory_write_drain = memory_write_drain
|
||||
app.state.memory_write_drain_task = watched_task(
|
||||
@@ -1188,7 +1133,10 @@ def create_app(
|
||||
for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True):
|
||||
if (
|
||||
record not in chunk
|
||||
or not any(item.role == "user" for item in chunk)
|
||||
or not any(
|
||||
item.role == "user" and item.written_by.value == "person"
|
||||
for item in chunk
|
||||
)
|
||||
or not any(item.role == "assistant" for item in chunk)
|
||||
):
|
||||
continue
|
||||
@@ -1801,30 +1749,6 @@ def create_app(
|
||||
"events": events,
|
||||
}
|
||||
|
||||
@app.get("/api/v1/diagnostics/model-usage", dependencies=protected)
|
||||
def diagnostic_model_usage(
|
||||
since: datetime | None = Query(default=None), # noqa: B008
|
||||
request_id: str | None = Query( # noqa: B008
|
||||
default=None, min_length=1, max_length=512
|
||||
),
|
||||
limit: int = Query(default=200, ge=1, le=2_000), # noqa: B008
|
||||
) -> dict[str, object]:
|
||||
"""Return content-free precipitation usage for provider reconciliation."""
|
||||
|
||||
normalized_since = (
|
||||
(since if since.tzinfo is not None else since.replace(tzinfo=UTC))
|
||||
if since is not None
|
||||
else None
|
||||
)
|
||||
report = app.state.model_request_audit_store.report(
|
||||
since=normalized_since.astimezone(UTC).isoformat()
|
||||
if normalized_since is not None
|
||||
else None,
|
||||
request_id=request_id,
|
||||
limit=limit,
|
||||
)
|
||||
return {"schema_version": 1, **report}
|
||||
|
||||
@app.get(
|
||||
"/api/v1/runtime/identity",
|
||||
response_model=HealthResponse,
|
||||
@@ -2013,11 +1937,6 @@ def create_app(
|
||||
candidate_change_reason=app.state.source_candidate_store.change_reason(
|
||||
registration
|
||||
),
|
||||
authorization_state=grant_store.authorization_state(
|
||||
user_id="local",
|
||||
source_id=source_id,
|
||||
candidate=registration.candidate.path,
|
||||
),
|
||||
)
|
||||
for source_id, registration in _catalog(app).items()
|
||||
]
|
||||
@@ -2329,39 +2248,24 @@ def create_app(
|
||||
raise HTTPException(status_code=409, detail=error.code) from None
|
||||
if grant is None:
|
||||
raise HTTPException(status_code=409, detail="grant_required")
|
||||
try:
|
||||
await _prepare_source_resume(source_id)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_prepare_failed") from error
|
||||
await _prepare_source_resume(source_id)
|
||||
else:
|
||||
try:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_prepare_failed") from error
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
try:
|
||||
await asyncio.to_thread(_config_store(app).set_sync, source_id, request.enabled)
|
||||
except BaseException as error:
|
||||
except BaseException:
|
||||
if request.enabled:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
else:
|
||||
await _prepare_source_resume(source_id)
|
||||
await _activate_source_resume(source_id)
|
||||
await _refresh_live_watch(app)
|
||||
raise _sync_refused("sync_config_write_failed") from error
|
||||
raise
|
||||
try:
|
||||
if request.enabled:
|
||||
try:
|
||||
await _activate_source_resume(source_id)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_resume_failed") from error
|
||||
try:
|
||||
await _refresh_live_watch(app)
|
||||
except BaseException as error:
|
||||
raise _sync_refused("sync_watcher_failed") from error
|
||||
await _activate_source_resume(source_id)
|
||||
await _refresh_live_watch(app)
|
||||
except BaseException:
|
||||
# Rolls back the desired state and settles the watcher exactly as
|
||||
# before; the only change is that the raise above now carries a
|
||||
# code saying which stage failed.
|
||||
await asyncio.to_thread(_config_store(app).set_sync, source_id, previous)
|
||||
if request.enabled:
|
||||
await _live_sync(app).stop_source(source_id)
|
||||
@@ -2492,8 +2396,6 @@ def create_app(
|
||||
async def connect_sources(
|
||||
request: ConnectSourcesRequest,
|
||||
) -> ConnectSourcesResponse:
|
||||
if request.onboarding:
|
||||
app.state.history_import_gate_released = False
|
||||
source_ids = tuple(dict.fromkeys(request.source_ids))
|
||||
registrations = [_require_registration(app, source_id) for source_id in source_ids]
|
||||
if request.request_id is None:
|
||||
@@ -3659,48 +3561,6 @@ def create_app(
|
||||
def put_extraction_settings(request: ExtractionWindowUpdate) -> ExtractionSettings:
|
||||
return _memory_processing_supervisor(app).set_extraction_window(request.window_preset)
|
||||
|
||||
@app.get(
|
||||
"/api/v1/sedimentation",
|
||||
response_model=SedimentationState,
|
||||
dependencies=protected,
|
||||
)
|
||||
def get_sedimentation() -> SedimentationState:
|
||||
"""Whether the model may still turn conversations into memories."""
|
||||
return SedimentationState(enabled=_config_store(app).sedimentation_enabled())
|
||||
|
||||
@app.put(
|
||||
"/api/v1/sedimentation",
|
||||
response_model=SedimentationState,
|
||||
dependencies=protected,
|
||||
)
|
||||
async def put_sedimentation(request: SedimentationRequest) -> SedimentationState:
|
||||
"""Start or stop model-driven sedimentation, keeping the configuration.
|
||||
|
||||
Stopping is the user's cost control: nothing further goes to the model
|
||||
and already queued work waits rather than failing, so starting again
|
||||
resumes it. Importing conversations is not affected either way.
|
||||
"""
|
||||
store = _config_store(app)
|
||||
changed = False
|
||||
async with app.state.sedimentation_lock:
|
||||
previous = store.sedimentation_enabled()
|
||||
if previous is not request.enabled:
|
||||
await asyncio.to_thread(store.set_sedimentation_enabled, request.enabled)
|
||||
changed = True
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"enabled": request.enabled},
|
||||
)
|
||||
if changed and request.enabled:
|
||||
# Work parked by the stop is picked up again rather than waiting
|
||||
# for the next natural dispatch, but not on this request's time:
|
||||
# see _resume_sedimentation.
|
||||
_resume_sedimentation(app)
|
||||
return SedimentationState(enabled=store.sedimentation_enabled())
|
||||
|
||||
@app.post(
|
||||
"/api/v1/memory-processing/manual-extract",
|
||||
response_model=ManualExtractReceipt,
|
||||
@@ -4061,7 +3921,6 @@ async def _authorize_answer_generation(
|
||||
# The explicit question is this operation's input, not a standing permission for history.
|
||||
if not question.strip() or not grounding:
|
||||
raise MemoryAnswerError("memory_answer_generation_not_authorized")
|
||||
|
||||
def check() -> None:
|
||||
if _model_profile_store(app).active_identity() != (
|
||||
profile.profile_id,
|
||||
@@ -4085,7 +3944,6 @@ async def _authorize_answer_generation(
|
||||
await recheck()
|
||||
return recheck
|
||||
|
||||
|
||||
class _LazyASGIApp:
|
||||
"""Compatibility ASGI app that is built only when a server imports it.
|
||||
|
||||
@@ -4117,16 +3975,6 @@ class _LazyASGIApp:
|
||||
app = _LazyASGIApp()
|
||||
|
||||
|
||||
def _sync_refused(code: str) -> HTTPException:
|
||||
"""A sync refusal that says which stage refused.
|
||||
|
||||
Turning the switch on takes four steps: authorize, prepare, write, resume.
|
||||
The UI used to be told only that the switch had sprung back, and every
|
||||
stage needs a different fix, so each one gets its own code.
|
||||
"""
|
||||
return HTTPException(status_code=409, detail=code)
|
||||
|
||||
|
||||
def _granted_roots(
|
||||
source_ids: Iterable[str],
|
||||
grant_store: SourceGrantStore,
|
||||
@@ -4178,7 +4026,6 @@ def _source_response(
|
||||
totals: tuple[int, int],
|
||||
*,
|
||||
candidate_change_reason: str | None = None,
|
||||
authorization_state: SourceAuthorizationState = SourceAuthorizationState.OK,
|
||||
) -> SourceResponse:
|
||||
location = inspect_source_location(
|
||||
registration.descriptor.source_id, registration.candidate.path
|
||||
@@ -4391,7 +4238,7 @@ async def _onboarding_history_after(
|
||||
home: Path,
|
||||
now: datetime | None = None,
|
||||
) -> datetime | None:
|
||||
"""Return one shared cutoff when any selected harness has large, old history.
|
||||
"""Return one shared cutoff when any selected harness has large history.
|
||||
|
||||
The cutoff is decided before any history job is created, then persisted on
|
||||
every accepted job. This is deliberately global: independently limiting
|
||||
@@ -4400,8 +4247,6 @@ async def _onboarding_history_after(
|
||||
"""
|
||||
if not request.onboarding:
|
||||
return None
|
||||
anchor = now or datetime.now(UTC)
|
||||
cutoff = anchor - ONBOARDING_HISTORY_WINDOW
|
||||
revisions = request.source_candidate_revisions or {}
|
||||
for registration in registrations:
|
||||
source_id = registration.descriptor.source_id
|
||||
@@ -4429,52 +4274,29 @@ async def _onboarding_history_after(
|
||||
home=home,
|
||||
)
|
||||
adapter = registration.create_adapter(grant)
|
||||
if await _has_large_old_history(adapter, cutoff=cutoff):
|
||||
return cutoff
|
||||
count_sessions = cast(
|
||||
Callable[..., Awaitable[int]] | None,
|
||||
getattr(adapter, "count_sessions", None),
|
||||
)
|
||||
if count_sessions is not None:
|
||||
session_count = await count_sessions(limit=ONBOARDING_LARGE_HISTORY_SESSIONS + 1)
|
||||
else:
|
||||
pending = _install_state(adapter)
|
||||
if pending is None:
|
||||
continue
|
||||
session_count = (await pending).session_count
|
||||
except (OSError, SourceAccessError):
|
||||
# A supported source whose metadata count cannot be completed is
|
||||
# treated conservatively. The later connection still owns the
|
||||
# precise authorization error, while onboarding never falls back
|
||||
# to an unbounded history import.
|
||||
return cutoff
|
||||
session_count = ONBOARDING_LARGE_HISTORY_SESSIONS + 1
|
||||
if session_count > ONBOARDING_LARGE_HISTORY_SESSIONS:
|
||||
anchor = now or datetime.now(UTC)
|
||||
return anchor - ONBOARDING_HISTORY_WINDOW
|
||||
return None
|
||||
|
||||
|
||||
async def _has_large_old_history(adapter: SourceAdapter, *, cutoff: datetime) -> bool:
|
||||
"""Probe one authorized Harness with the same parsers the import will use.
|
||||
|
||||
Stop as soon as both policy facts are proven: more than 200 genuine person
|
||||
turns and at least one record older than the three-day cutoff. Counting
|
||||
parsed Evidence instead of transcript files prevents one giant coding
|
||||
session from bypassing the limit, while reusing the adapters keeps host
|
||||
envelopes and tool-shaped user records out of the count.
|
||||
"""
|
||||
|
||||
user_turns = 0
|
||||
has_old_data = False
|
||||
async for event in adapter.scan(ScanOptions()):
|
||||
if isinstance(event, ScanFailure):
|
||||
# A partial probe must never turn into an unbounded first import.
|
||||
return True
|
||||
if not isinstance(event, ScannedEvidence):
|
||||
continue
|
||||
observed = event.evidence.created_at
|
||||
if observed is None and event.cursor.observed_modified_ns > 0:
|
||||
observed = datetime.fromtimestamp(
|
||||
event.cursor.observed_modified_ns / 1_000_000_000,
|
||||
tz=UTC,
|
||||
)
|
||||
if observed is not None:
|
||||
if observed.tzinfo is None:
|
||||
observed = observed.replace(tzinfo=UTC)
|
||||
has_old_data = has_old_data or observed.astimezone(UTC) < cutoff
|
||||
if event.evidence.role == "user":
|
||||
user_turns += 1
|
||||
if user_turns > ONBOARDING_LARGE_HISTORY_USER_TURNS and has_old_data:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _install_state(adapter: object) -> Awaitable[SourceInstallState] | None:
|
||||
"""Ask an adapter for the two-level answer when it has one.
|
||||
|
||||
@@ -4766,7 +4588,7 @@ async def _keep_loop_heartbeat(watchdog: LoopWatchdog) -> None:
|
||||
async def _emit_resource_snapshots(interval_seconds: float = 60.0) -> None:
|
||||
while True:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="core.resources",
|
||||
event="core.resource_snapshot",
|
||||
outcome="observed",
|
||||
@@ -4791,33 +4613,6 @@ def _start_backfill(app: FastAPI) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _resume_sedimentation(app: FastAPI) -> None:
|
||||
"""Pick the work parked by a stop back up, off the caller's clock.
|
||||
|
||||
Turning the switch back on is a decision, not a job the caller waits for:
|
||||
the choice is already stored, and what follows it is maintenance over every
|
||||
unfinished task followed by a scan of the whole queue. On a real backlog
|
||||
that runs for many minutes, so awaiting it here left the PUT unanswered
|
||||
until the client gave up — the renderer showed "启用未能确认" for a switch
|
||||
that had in fact been saved, and the switch only appeared to turn on once
|
||||
the follow-up read came back. Measured: one toggle, twenty-plus minutes,
|
||||
no response, client timeout at thirty seconds.
|
||||
|
||||
Repeated toggles reuse the running walk; it reads the switch as it goes, so
|
||||
a stop part-way through is honoured on the next batch instead of needing a
|
||||
cancel.
|
||||
"""
|
||||
running = cast("asyncio.Task[None] | None", getattr(app.state, "sedimentation_resume", None))
|
||||
if running is not None and not running.done():
|
||||
return
|
||||
app.state.sedimentation_resume = watched_task(
|
||||
asyncio.get_running_loop(),
|
||||
lambda: _memory_processing_supervisor(app).resume_allowed(),
|
||||
component="memory.sedimentation",
|
||||
name="one-memory-sedimentation-resume",
|
||||
)
|
||||
|
||||
|
||||
def _start_visibility_recompute(app: FastAPI) -> None:
|
||||
"""Keep the stored visibility conclusions current, in the background.
|
||||
|
||||
@@ -5061,51 +4856,6 @@ def _source_writes_on(app: FastAPI, source_id: str) -> bool:
|
||||
return _source_processing_enabled(app, source_id)
|
||||
|
||||
|
||||
def _sedimentation_claim_enabled(app: FastAPI, source_id: str) -> bool:
|
||||
"""Claim queued model work only while both global and source gates allow it."""
|
||||
|
||||
return (
|
||||
_config_store(app).sedimentation_enabled()
|
||||
and _history_import_gate_released(app)
|
||||
and _source_writes_on(app, source_id)
|
||||
)
|
||||
|
||||
|
||||
def _history_import_gate_released(app: FastAPI) -> bool:
|
||||
"""Hold first sedimentation until every accepted history import is terminal."""
|
||||
|
||||
if getattr(app.state, "history_import_gate_released", False):
|
||||
return True
|
||||
onboarding = getattr(app.state, "onboarding_store", None)
|
||||
jobs = getattr(app.state, "import_job_store", None)
|
||||
if onboarding is None or jobs is None:
|
||||
return True
|
||||
state = onboarding.get()
|
||||
if state.status != "importing" or not state.selected_source_ids:
|
||||
app.state.history_import_gate_released = True
|
||||
return True
|
||||
selected = set(state.selected_source_ids)
|
||||
matching = [
|
||||
job
|
||||
for job in jobs.list_jobs()
|
||||
if job.operation == "history"
|
||||
and job.source_id in selected
|
||||
and (state.started_at is None or job.created_at >= state.started_at)
|
||||
]
|
||||
if {job.source_id for job in matching} != selected:
|
||||
return False
|
||||
active = {
|
||||
ImportStatus.PENDING,
|
||||
ImportStatus.RUNNING,
|
||||
ImportStatus.PAUSED,
|
||||
ImportStatus.INTERRUPTED,
|
||||
}
|
||||
released = all(job.status not in active for job in matching)
|
||||
if released:
|
||||
app.state.history_import_gate_released = True
|
||||
return released
|
||||
|
||||
|
||||
def _model_prober(app: FastAPI) -> ModelProber:
|
||||
return cast(ModelProber, app.state.model_prober)
|
||||
|
||||
|
||||
@@ -175,22 +175,6 @@ async def in_interactive_read[T](
|
||||
return await in_read_pool(interactive_read_pool(), work, *args, **kwargs)
|
||||
|
||||
|
||||
def _retrieve[T](future: "asyncio.Future[T]") -> None:
|
||||
"""Mark a shielded read's outcome as seen.
|
||||
|
||||
`in_read_pool` shields the wrapped worker, so a caller that gives up and
|
||||
cancels leaves the read running to completion. When that read then fails,
|
||||
nothing awaits the future any more: asyncio reports the orphan at GC as
|
||||
"Future exception was never retrieved", the loop handler turns it into a
|
||||
`core.loop.task_exception` error diagnostic, and a lane that degraded on
|
||||
its own budget -- correctly, and already reported -- looks like a crashed
|
||||
task on top of it. Retrieving here keeps the diagnostics honest. Awaiting
|
||||
still raises for callers that are still there, so no failure is hidden.
|
||||
"""
|
||||
if not future.cancelled():
|
||||
future.exception()
|
||||
|
||||
|
||||
async def in_read_pool[T](
|
||||
pool: ThreadPoolExecutor,
|
||||
work: Callable[..., T],
|
||||
@@ -209,9 +193,7 @@ async def in_read_pool[T](
|
||||
_interactive_reads -= 1
|
||||
|
||||
worker.add_done_callback(released)
|
||||
pending = asyncio.wrap_future(worker)
|
||||
pending.add_done_callback(_retrieve)
|
||||
return await asyncio.shield(pending)
|
||||
return await asyncio.shield(asyncio.wrap_future(worker))
|
||||
|
||||
|
||||
async def wait_for_interactive_reads() -> None:
|
||||
|
||||
@@ -266,13 +266,7 @@ def serve(*, maintenance: bool = False, reader: bool = False) -> None:
|
||||
)
|
||||
application = create_reader_app(reader_settings)
|
||||
print(f"reader_port:{port}", file=sys.stderr, flush=True)
|
||||
uvicorn.run(
|
||||
application,
|
||||
host=settings.host,
|
||||
port=port,
|
||||
reload=False,
|
||||
access_log=False,
|
||||
)
|
||||
uvicorn.run(application, host=settings.host, port=port, reload=False)
|
||||
return
|
||||
|
||||
from one_memory.api import create_app
|
||||
@@ -303,21 +297,9 @@ def serve(*, maintenance: bool = False, reader: bool = False) -> None:
|
||||
)
|
||||
if maintenance:
|
||||
with CoreRuntime(settings, publish_receipt=False):
|
||||
uvicorn.run(
|
||||
application,
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
reload=False,
|
||||
access_log=False,
|
||||
)
|
||||
uvicorn.run(application, host=settings.host, port=settings.port, reload=False)
|
||||
else:
|
||||
uvicorn.run(
|
||||
application,
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
reload=False,
|
||||
access_log=False,
|
||||
)
|
||||
uvicorn.run(application, host=settings.host, port=settings.port, reload=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -196,7 +196,7 @@ def read_raw_context(
|
||||
if _prefers_declared_facts(request.query)
|
||||
else "1"
|
||||
)
|
||||
person_authored = "CASE WHEN e.role='user' THEN 1 ELSE 0 END"
|
||||
person_authored = "CASE WHEN e.written_by='person' THEN 1 ELSE 0 END"
|
||||
query_echo = (
|
||||
"CASE WHEN trim(e.verbatim)=? OR trim(e.verbatim)=? THEN 0 ELSE 1 END"
|
||||
if _prefers_declared_facts(request.query)
|
||||
@@ -470,8 +470,9 @@ def revalidate_raw_context(
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
break
|
||||
if (row["role"], created) != (
|
||||
if (row["role"], row["written_by"], created) != (
|
||||
message.role,
|
||||
message.written_by,
|
||||
message.created_at,
|
||||
):
|
||||
break
|
||||
|
||||
@@ -66,12 +66,12 @@ def _field(text: str) -> ResumeField | None:
|
||||
return None
|
||||
|
||||
|
||||
def _raw_class(role: str, evidence_id: str, accepted: frozenset[str]) -> EvidenceClass:
|
||||
if role == "user":
|
||||
def _raw_class(written_by: str, evidence_id: str, accepted: frozenset[str]) -> EvidenceClass:
|
||||
if written_by == "person":
|
||||
return "person_statement"
|
||||
if role == "assistant":
|
||||
if written_by == "agent":
|
||||
return "agent_report"
|
||||
if role == "tool" and evidence_id in accepted:
|
||||
if written_by == "tool" and evidence_id in accepted:
|
||||
return "system_acceptance"
|
||||
return "tool_output"
|
||||
|
||||
@@ -118,7 +118,7 @@ def assemble_resume_packet(
|
||||
field=_field(message.text) or "unclassified",
|
||||
text=message.text,
|
||||
evidence_class=_raw_class(
|
||||
message.role,
|
||||
message.written_by,
|
||||
message.evidence_id,
|
||||
verified_system_evidence_ids,
|
||||
),
|
||||
|
||||
@@ -345,7 +345,7 @@ def _lexical_raw_rows(
|
||||
if context_raw._prefers_declared_facts(query.query)
|
||||
else "1"
|
||||
)
|
||||
person_authored = "CASE WHEN e.role='user' THEN 1 ELSE 0 END"
|
||||
person_authored = "CASE WHEN e.written_by='person' THEN 1 ELSE 0 END"
|
||||
query_echo = (
|
||||
"CASE WHEN trim(e.verbatim)=? OR trim(e.verbatim)=? THEN 0 ELSE 1 END"
|
||||
if context_raw._prefers_declared_facts(query.query)
|
||||
@@ -507,7 +507,7 @@ def _raw_item(row: sqlite3.Row, start1: int, matched_by: tuple[str, ...]) -> Raw
|
||||
"source_id": row["source_id"],
|
||||
"conversation_id": row["conversation_id"],
|
||||
"role": row["role"],
|
||||
"written_by": "person" if row["role"] == "user" else "agent",
|
||||
"written_by": "person" if row["written_by"] == "person" else "agent",
|
||||
"event_time": event_time.isoformat() if event_time else None,
|
||||
"spans": [{"start_cp": start, "end_cp": start + len(text), "text": text}],
|
||||
"matched_by": list(matched_by),
|
||||
|
||||
@@ -1039,7 +1039,7 @@ def close_main_databases() -> None:
|
||||
def log_main_database_stats(moment: str) -> None:
|
||||
"""One line in core.log: how many connections are open and who waited."""
|
||||
stats = main_database_stats()
|
||||
logger.debug(
|
||||
logger.info(
|
||||
"main_database_connections moment=%s %s",
|
||||
moment,
|
||||
" ".join(f"{key}={value}" for key, value in stats.items()),
|
||||
|
||||
@@ -19,7 +19,6 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@@ -140,10 +139,6 @@ _DETAIL_KEYS = frozenset(
|
||||
"accounted_ms",
|
||||
"unaccounted_ms",
|
||||
"conversation_hash",
|
||||
# The sedimentation switch: a bounded boolean, the same shape as
|
||||
# `spawned` and `index_complete`. Left out of this list it did not
|
||||
# drop a log line, it turned the switch's own request into a 500.
|
||||
"enabled",
|
||||
}
|
||||
)
|
||||
_trace_id: ContextVar[str | None] = ContextVar("one_memory_trace_id", default=None)
|
||||
@@ -654,24 +649,12 @@ def diagnostic_event(**values: object) -> None:
|
||||
active = _active
|
||||
if active is None:
|
||||
return
|
||||
if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(
|
||||
logging.DEBUG
|
||||
):
|
||||
return
|
||||
try:
|
||||
active.write(**values) # type: ignore[arg-type]
|
||||
except OSError:
|
||||
# Diagnostics must not become a new product outage when a disk is full
|
||||
# or the log directory is temporarily unavailable.
|
||||
return
|
||||
except ValueError:
|
||||
# Same promise, same reason, for a rejected payload instead of a failed
|
||||
# write. An unlisted `details` key once turned every external recall
|
||||
# into a 500 and then killed the process, and the sedimentation switch
|
||||
# into a permanent failure. The payload itself is still wrong and is
|
||||
# still caught in CI by tests/test_diagnostic_payload_keys.py; what
|
||||
# must not happen is the caller paying for it.
|
||||
return
|
||||
|
||||
|
||||
def current_session_id() -> str | None:
|
||||
|
||||
@@ -33,7 +33,6 @@ from one_memory.hy_runtime_config import (
|
||||
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
|
||||
from one_memory.hy_store_concurrency import native_store_read_slot
|
||||
from one_memory.model_profiles import ModelProfileSnapshot, ProviderKind, attach_disable_thinking
|
||||
from one_memory.model_request_audit import audit_precipitation_request
|
||||
from one_memory.semantic_deadline import SemanticMemoryHit
|
||||
|
||||
#: D102: how relevant HY has to call a hit before it counts. Raised from 0.25
|
||||
@@ -144,22 +143,6 @@ class HyMemoryBridgeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES = 512
|
||||
|
||||
|
||||
def coding_tool_result_max_bytes() -> int:
|
||||
"""Return the per-tool payload ceiling used by MemHub's HY client."""
|
||||
|
||||
raw = os.environ.get("MEMORY_CODING_TOOL_RESULT_MAX_BYTES")
|
||||
if raw is None:
|
||||
return MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
return value if value > 0 else MEMHUB_DEFAULT_CODING_TOOL_RESULT_MAX_BYTES
|
||||
|
||||
|
||||
def _exclusive[**P, T](
|
||||
method: Callable[P, Coroutine[Any, Any, T]],
|
||||
) -> Callable[P, Coroutine[Any, Any, T]]:
|
||||
@@ -431,9 +414,8 @@ class HyMemoryBridge:
|
||||
|
||||
@_exclusive
|
||||
async def ingest(self, request: HyMemoryIngestRequest) -> HyMemoryIngestResult:
|
||||
with audit_precipitation_request(request.request_id):
|
||||
async with self._write_lock:
|
||||
return await self._ingest(request)
|
||||
async with self._write_lock:
|
||||
return await self._ingest(request)
|
||||
|
||||
async def _ingest(self, request: HyMemoryIngestRequest) -> HyMemoryIngestResult:
|
||||
started = monotonic()
|
||||
@@ -449,7 +431,9 @@ class HyMemoryBridge:
|
||||
guards = getattr(client, "_one_memory_write_guards", ())
|
||||
from one_memory.hy_response_guard import HyReconcileGuard
|
||||
|
||||
reconcile_guards = tuple(guard for guard in guards if isinstance(guard, HyReconcileGuard))
|
||||
reconcile_guards = tuple(
|
||||
guard for guard in guards if isinstance(guard, HyReconcileGuard)
|
||||
)
|
||||
for guard in reconcile_guards:
|
||||
guard.begin_write()
|
||||
before = [len(guard.failures) for guard in guards]
|
||||
@@ -560,9 +544,8 @@ class HyMemoryBridge:
|
||||
request: HyMemoryIngestRequest,
|
||||
contents: Sequence[str],
|
||||
) -> HyMemoryIngestResult:
|
||||
with audit_precipitation_request(request.request_id):
|
||||
async with self._write_lock:
|
||||
return await self._ingest_extracted(request, contents)
|
||||
async with self._write_lock:
|
||||
return await self._ingest_extracted(request, contents)
|
||||
|
||||
async def _ingest_extracted(
|
||||
self,
|
||||
@@ -1127,7 +1110,7 @@ class HyMemoryBridge:
|
||||
self.profile,
|
||||
api_key=self.api_key,
|
||||
data_dir=self.data_dir,
|
||||
http_transport_factory=self._http_transport_factory,
|
||||
http_transport_factory=self._http_transport_factory,
|
||||
)
|
||||
|
||||
await _joined_thread(initialize)
|
||||
@@ -1154,7 +1137,7 @@ class HyMemoryBridge:
|
||||
)
|
||||
raise HyMemoryBridgeError("hy_memory_init_failed") from error
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="hy.client",
|
||||
event="hy.client_initialization.completed",
|
||||
outcome="succeeded",
|
||||
@@ -1345,7 +1328,9 @@ async def _with_native_mutation_delta(
|
||||
"""
|
||||
|
||||
retired = set(result.retired_memory_ids)
|
||||
superseded = {key: tuple(values) for key, values in result.superseded_memory_ids.items()}
|
||||
superseded = {
|
||||
key: tuple(values) for key, values in result.superseded_memory_ids.items()
|
||||
}
|
||||
chat_targets = {
|
||||
key
|
||||
for guard in reconcile_guards
|
||||
@@ -1353,8 +1338,7 @@ async def _with_native_mutation_delta(
|
||||
if isinstance(key, str) and key
|
||||
}
|
||||
|
||||
operations = raw.get("ops")
|
||||
for item in operations if isinstance(operations, list) else ():
|
||||
for item in raw.get("ops", ()) if isinstance(raw.get("ops"), list) else ():
|
||||
if not isinstance(item, Mapping):
|
||||
continue
|
||||
if item.get("action") == "DELETE":
|
||||
@@ -1362,8 +1346,7 @@ async def _with_native_mutation_delta(
|
||||
if isinstance(key, str) and key:
|
||||
retired.add(key)
|
||||
|
||||
fact_operations = raw.get("fact_ops")
|
||||
for item in fact_operations if isinstance(fact_operations, list) else ():
|
||||
for item in raw.get("fact_ops", ()) if isinstance(raw.get("fact_ops"), list) else ():
|
||||
if not isinstance(item, Mapping) or item.get("action") != "SUPERSEDE":
|
||||
continue
|
||||
old, new = item.get("fact_id"), item.get("superseded_by")
|
||||
@@ -1384,7 +1367,9 @@ async def _with_native_mutation_delta(
|
||||
if isinstance(replacement, str) and replacement:
|
||||
superseded[key] = (replacement,)
|
||||
elif isinstance(replacement, list):
|
||||
values = tuple(value for value in replacement if isinstance(value, str) and value)
|
||||
values = tuple(
|
||||
value for value in replacement if isinstance(value, str) and value
|
||||
)
|
||||
if values:
|
||||
superseded[key] = values
|
||||
|
||||
@@ -1712,7 +1697,6 @@ def build_hy_memory_config(
|
||||
config.history.db_path = str(memory_root / "history.sqlite3")
|
||||
config.coding.enable = True
|
||||
config.coding.db_path = str(memory_root / "coding.sqlite3")
|
||||
config.coding.tool_result_max_bytes = coding_tool_result_max_bytes()
|
||||
config.recall.episode_recall_enabled = True
|
||||
config.recall.episode_summary_enabled = False
|
||||
config.recall.dynamic_episode_enabled = True
|
||||
|
||||
@@ -14,7 +14,7 @@ from pathlib import Path
|
||||
from typing import Literal, cast
|
||||
|
||||
HY_MEMORY_DISTRIBUTION = "hy-memory"
|
||||
HY_MEMORY_VERSION = "2.0.4"
|
||||
HY_MEMORY_VERSION = "2.0.3"
|
||||
HY_MEMORY_NAMESPACE = "governed-v2"
|
||||
DiagnosticLogLevel = Literal["error", "warning", "info", "debug"]
|
||||
|
||||
@@ -31,12 +31,11 @@ _COLLECTION_LOGGERS = (
|
||||
"httpx",
|
||||
"httpcore",
|
||||
)
|
||||
_NOISY_LIBRARY_LOGGERS = ("httpx", "httpcore", "watchfiles", "jieba")
|
||||
|
||||
# One Memory deliberately stays on ultra because its existing, user-visible
|
||||
# profile/graph flow calls digest(), which the pinned public SDK exposes only
|
||||
# for ultra clients. Reader and non-reranker behavior remain pinned for
|
||||
# compatibility with public SDK 2.0.4.
|
||||
# compatibility with public SDK 2.0.3.
|
||||
HY_MEMORY_MODE = "ultra"
|
||||
# The SDK only recognises `legacy` and `hybrid` (see
|
||||
# hy_memory/pipelines/_retrieval/config.py: `ALL_READERS = (READER_LEGACY,
|
||||
@@ -140,18 +139,6 @@ def apply_diagnostic_log_level(level: DiagnosticLogLevel) -> None:
|
||||
for name in tuple(logging.Logger.manager.loggerDict):
|
||||
if any(name == root or name.startswith(f"{root}.") for root in _COLLECTION_LOGGERS):
|
||||
logging.getLogger(name).setLevel(selected)
|
||||
# Successful HTTP exchanges, file-watch notifications and tokenizer
|
||||
# initialization are useful only while actively debugging. At INFO they
|
||||
# dominated core.log and the in-app panel without describing product work.
|
||||
noisy_level = selected if selected == logging.DEBUG else max(selected, logging.WARNING)
|
||||
for root in _NOISY_LIBRARY_LOGGERS:
|
||||
logging.getLogger(root).setLevel(noisy_level)
|
||||
for name in tuple(logging.Logger.manager.loggerDict):
|
||||
if any(
|
||||
name == root or name.startswith(f"{root}.")
|
||||
for root in _NOISY_LIBRARY_LOGGERS
|
||||
):
|
||||
logging.getLogger(name).setLevel(noisy_level)
|
||||
|
||||
os.environ["MEMORY_LOG_LEVEL"] = logging.getLevelName(selected)
|
||||
os.environ["MEMORY_PIPELINE_TRACE_ENABLED"] = "true" if level == "debug" else "false"
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
|
||||
|
||||
@@ -61,7 +61,17 @@ class HyWriteBindings:
|
||||
seed.workspace,
|
||||
)
|
||||
or record.content_kind is not EvidenceContentKind.RAW_CONVERSATION
|
||||
or record.role not in {"user", "assistant", "tool"}
|
||||
or (record.role, record.written_by)
|
||||
not in {
|
||||
# D98: the same change as the turn assembly. A sub agent's
|
||||
# question is a `user` message marked written by the
|
||||
# agent; excluding it here would make the source context
|
||||
# disagree with what was actually sent.
|
||||
("user", WrittenBy.PERSON),
|
||||
("user", WrittenBy.AGENT),
|
||||
("assistant", WrittenBy.AGENT),
|
||||
("tool", WrittenBy.TOOL),
|
||||
}
|
||||
):
|
||||
raise ValueError("hy_memory_write_verification_context_scope")
|
||||
self.all_sources: list[dict[str, Any]] = [
|
||||
|
||||
@@ -152,7 +152,7 @@ class HyStepTimer:
|
||||
if value is None:
|
||||
continue
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component=STEP_COMPONENTS[name],
|
||||
event="hy.step.completed",
|
||||
outcome="succeeded",
|
||||
@@ -168,7 +168,7 @@ class HyStepTimer:
|
||||
)
|
||||
model = measured.get("model")
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="hy.bridge",
|
||||
event="hy.steps.summary",
|
||||
outcome="succeeded",
|
||||
|
||||
@@ -105,7 +105,7 @@ async def iterate_in_worker[T](
|
||||
# was unwinding with, and a diagnostic is not worth an import.
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.async_bridge",
|
||||
event="import.source_read_finished",
|
||||
outcome="succeeded",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import sqlite3
|
||||
from contextlib import AbstractContextManager
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
@@ -21,23 +20,6 @@ CURSOR_MAX_SOURCE_FILE_BYTES = 64 * 1024 * 1024 * 1024
|
||||
GRANT_SCHEMA = source_grants_ddl("sqlite")
|
||||
|
||||
|
||||
class SourceAuthorizationState(StrEnum):
|
||||
"""Why a source cannot be read right now, or `ok` when it can.
|
||||
|
||||
The API used to answer `authorized=false` for all of these at once, which
|
||||
is how a revoked grant, a rebuilt history directory and a moved candidate
|
||||
all looked like the same bug and all got the same advice. The values carry
|
||||
no paths and no user content, because they are written to diagnostics.
|
||||
"""
|
||||
|
||||
OK = "ok"
|
||||
MISSING_GRANT = "missing_grant"
|
||||
REVOKED_GRANT = "revoked_grant"
|
||||
SOURCE_IDENTITY_CHANGED = "source_identity_changed"
|
||||
SOURCE_ROOT_UNAVAILABLE = "source_root_unavailable"
|
||||
CANDIDATE_CHANGED = "candidate_changed"
|
||||
|
||||
|
||||
class SourceGrantReceipt(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
@@ -164,46 +146,6 @@ class SourceGrantStore:
|
||||
raise SourceAccessError(SourceAccessErrorCode.SOURCE_IDENTITY_CHANGED)
|
||||
return authorized
|
||||
|
||||
def authorization_state(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
source_id: str,
|
||||
candidate: Path,
|
||||
) -> SourceAuthorizationState:
|
||||
"""Why reading this source would be refused.
|
||||
|
||||
Read-only and fails the same way `authorized_root` does: this names a
|
||||
state for the UI and for diagnostics, it does not widen access. The
|
||||
candidate is compared by path, so a grant that still points somewhere
|
||||
else reads as a moved candidate rather than as a rebuilt directory —
|
||||
the two need different fixes.
|
||||
"""
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM source_grants WHERE user_id=? AND source_id=?",
|
||||
(user_id, source_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return SourceAuthorizationState.MISSING_GRANT
|
||||
if row["revoked_at"] is not None or row["revision"] <= 0:
|
||||
return SourceAuthorizationState.REVOKED_GRANT
|
||||
granted = Path(row["requested_path"])
|
||||
if not granted.exists():
|
||||
return SourceAuthorizationState.SOURCE_ROOT_UNAVAILABLE
|
||||
try:
|
||||
if not root_identity_matches(granted, row["root_identity"]):
|
||||
return SourceAuthorizationState.SOURCE_IDENTITY_CHANGED
|
||||
except SourceAccessError:
|
||||
# Unreadable or gone between the two calls: there is no grant to
|
||||
# repair, only a directory to make available again.
|
||||
return SourceAuthorizationState.SOURCE_ROOT_UNAVAILABLE
|
||||
# Same directory, but the app now looks somewhere else. Re-authorizing
|
||||
# the old grant would not help; the candidate has to be granted.
|
||||
if granted != candidate:
|
||||
return SourceAuthorizationState.CANDIDATE_CHANGED
|
||||
return SourceAuthorizationState.OK
|
||||
|
||||
def revoke(self, *, user_id: str, source_id: str) -> SourceGrantReceipt:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
with self._connect(writing=True) as connection:
|
||||
|
||||
@@ -106,7 +106,7 @@ class PollingLiveSyncSupervisor:
|
||||
level=(
|
||||
"warn"
|
||||
if value.state in {LiveSyncState.DEGRADED, LiveSyncState.UNAVAILABLE}
|
||||
else "debug"
|
||||
else "info"
|
||||
),
|
||||
component="import.live_sync",
|
||||
event="live_sync.state_changed",
|
||||
|
||||
@@ -156,7 +156,7 @@ class FileWatchLiveSync:
|
||||
if source is not None
|
||||
):
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.live_watch",
|
||||
event="live_sync.hook_received",
|
||||
outcome="queued",
|
||||
@@ -190,7 +190,7 @@ class FileWatchLiveSync:
|
||||
try:
|
||||
await self._trigger_past(source_id, observed)
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.live_watch",
|
||||
event="live_sync.hook_completed",
|
||||
outcome="written",
|
||||
|
||||
@@ -79,8 +79,6 @@ class EvidenceInput(BaseModel):
|
||||
source_ordinal: str | None = Field(default=None, min_length=1, max_length=256)
|
||||
content_kind: EvidenceContentKind = EvidenceContentKind.RAW_CONVERSATION
|
||||
role: str = Field(min_length=1, max_length=128)
|
||||
# Legacy import/storage metadata only. Protocol `role` is authoritative;
|
||||
# this value must not affect identity, admission, scheduling or ownership.
|
||||
written_by: WrittenBy
|
||||
created_at: datetime | None = None
|
||||
verbatim: str = Field(min_length=1, max_length=MAX_VERBATIM_CHARACTERS)
|
||||
@@ -151,22 +149,17 @@ class EvidenceInput(BaseModel):
|
||||
)
|
||||
)
|
||||
else:
|
||||
# `role` is the protocol-level speaker identity. `written_by` was a
|
||||
# parser guess and differed between two imports of the same host
|
||||
# message (notably sub-agent/user and unknown/user), minting two
|
||||
# Evidence rows. Keep it out of every new identity.
|
||||
#
|
||||
# No workspace and no source_ordinal in the derived key: the hook
|
||||
# lane has neither (it reports no workspace and a constant ordinal),
|
||||
# so including either gave the same message a different key per lane
|
||||
# and stored it twice. Both are record metadata here, not identity.
|
||||
identity = "\0".join(
|
||||
(
|
||||
"derived-v4",
|
||||
"derived-v3",
|
||||
self.user_id,
|
||||
self.source_id,
|
||||
self.conversation_id,
|
||||
self.role,
|
||||
self.written_by,
|
||||
self.content_hash,
|
||||
)
|
||||
)
|
||||
@@ -175,54 +168,6 @@ class EvidenceInput(BaseModel):
|
||||
# two rows. The capture time is record metadata, not identity.
|
||||
return hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
@property
|
||||
def written_by_derived_v3_idempotency_key(self) -> str:
|
||||
"""The last identity that treated the parser's author guess as canonical.
|
||||
|
||||
New builds key by protocol role. During an upgrade this key lets a
|
||||
rescan find and reuse the row already stored by derived-v3 instead of
|
||||
inserting the same host message under a second Evidence id.
|
||||
"""
|
||||
identity = "\0".join(
|
||||
(
|
||||
"derived-v3",
|
||||
self.user_id,
|
||||
self.source_id,
|
||||
self.conversation_id,
|
||||
self.written_by,
|
||||
self.content_hash,
|
||||
)
|
||||
)
|
||||
return hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
@property
|
||||
def written_by_derived_v3_idempotency_keys(self) -> tuple[str, ...]:
|
||||
"""Every legacy author guess this protocol role may have carried."""
|
||||
guesses = {
|
||||
"user": (
|
||||
WrittenBy.PERSON,
|
||||
WrittenBy.AGENT,
|
||||
WrittenBy.UNKNOWN,
|
||||
),
|
||||
"assistant": (WrittenBy.AGENT, WrittenBy.UNKNOWN, WrittenBy.PERSON),
|
||||
"tool": (WrittenBy.TOOL, WrittenBy.UNKNOWN),
|
||||
}.get(self.role, tuple(WrittenBy))
|
||||
ordered = (self.written_by, *guesses)
|
||||
keys: list[str] = []
|
||||
for written_by in dict.fromkeys(ordered):
|
||||
identity = "\0".join(
|
||||
(
|
||||
"derived-v3",
|
||||
self.user_id,
|
||||
self.source_id,
|
||||
self.conversation_id,
|
||||
written_by,
|
||||
self.content_hash,
|
||||
)
|
||||
)
|
||||
keys.append(hashlib.sha256(identity.encode()).hexdigest())
|
||||
return tuple(keys)
|
||||
|
||||
@property
|
||||
def legacy_message_idempotency_key(self) -> str | None:
|
||||
"""The pre-workspace key for a message-shaped input, when one exists.
|
||||
@@ -267,13 +212,7 @@ class EvidenceInput(BaseModel):
|
||||
"""
|
||||
return self.derived_v2_idempotency_key("derived-v2", workspace_agnostic=False)
|
||||
|
||||
def derived_v2_idempotency_key(
|
||||
self,
|
||||
label: str,
|
||||
*,
|
||||
workspace_agnostic: bool,
|
||||
written_by: WrittenBy | None = None,
|
||||
) -> str:
|
||||
def derived_v2_idempotency_key(self, label: str, *, workspace_agnostic: bool) -> str:
|
||||
identity = "\0".join(
|
||||
(
|
||||
label,
|
||||
@@ -281,7 +220,7 @@ class EvidenceInput(BaseModel):
|
||||
self.source_id,
|
||||
*([] if workspace_agnostic else [self.workspace or "\uffffnull"]),
|
||||
self.conversation_id,
|
||||
written_by or self.written_by,
|
||||
self.written_by,
|
||||
self.created_at.isoformat() if self.created_at else "",
|
||||
self.source_ordinal or "",
|
||||
# v2 hashed the cleaned body; v1 predates raw_verbatim entirely,
|
||||
@@ -291,26 +230,6 @@ class EvidenceInput(BaseModel):
|
||||
)
|
||||
return hashlib.sha256(identity.encode()).hexdigest()
|
||||
|
||||
@property
|
||||
def older_written_by_idempotency_keys(self) -> tuple[str, ...]:
|
||||
keys: list[str] = []
|
||||
for written_by in WrittenBy:
|
||||
keys.append(
|
||||
self.derived_v2_idempotency_key(
|
||||
"derived-v1",
|
||||
workspace_agnostic=True,
|
||||
written_by=written_by,
|
||||
)
|
||||
)
|
||||
keys.append(
|
||||
self.derived_v2_idempotency_key(
|
||||
"derived-v2",
|
||||
workspace_agnostic=False,
|
||||
written_by=written_by,
|
||||
)
|
||||
)
|
||||
return tuple(keys)
|
||||
|
||||
@property
|
||||
def evidence_id(self) -> str:
|
||||
return str(uuid5(EVIDENCE_NAMESPACE, self.idempotency_key))
|
||||
@@ -330,7 +249,6 @@ class EvidenceRecord(BaseModel):
|
||||
source_ordinal: str | None
|
||||
content_kind: EvidenceContentKind
|
||||
role: str
|
||||
# Retained while old databases and export formats still carry the column.
|
||||
written_by: WrittenBy
|
||||
created_at: datetime | None
|
||||
verbatim: str
|
||||
|
||||
@@ -199,7 +199,7 @@ class ImportRunner:
|
||||
raise ValueError("Import job source does not match adapter source")
|
||||
job, runner_token = await in_the_background(self.job_store.claim, job_id)
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.parser",
|
||||
event="import.parse.started",
|
||||
outcome="started",
|
||||
@@ -310,7 +310,7 @@ class ImportRunner:
|
||||
if event.kind == "progress" and str(event.phase) != last_phase:
|
||||
phase = str(event.phase)
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.parser",
|
||||
event="import.phase_changed",
|
||||
outcome="changed",
|
||||
@@ -516,7 +516,7 @@ class ImportRunner:
|
||||
async def _emit_parse_result(self, job: ImportJob, started: float) -> None:
|
||||
succeeded = job.status is ImportStatus.COMPLETED
|
||||
diagnostic_event(
|
||||
level="debug" if succeeded else "warn",
|
||||
level="info" if succeeded else "warn",
|
||||
component="import.parser",
|
||||
event="import.parse.completed",
|
||||
outcome="succeeded" if succeeded else str(job.status),
|
||||
|
||||
@@ -141,18 +141,11 @@ def parse_codex_record(
|
||||
if role == "user":
|
||||
written_by = _user_author(record, payload, session)
|
||||
# Machine inputs may quote a whole transcript, including user_query tags.
|
||||
# Protocol role is authoritative; host author metadata does not change
|
||||
# how a user message is cleaned.
|
||||
if not (
|
||||
content.startswith(
|
||||
"The following is the Codex agent history whose request action you are assessing."
|
||||
)
|
||||
and ">>> TRANSCRIPT START" in content
|
||||
and ">>> TRANSCRIPT END" in content
|
||||
):
|
||||
# Session provenance is not proof of who wrote any particular future turn.
|
||||
if written_by is WrittenBy.PERSON:
|
||||
content = strip_host_wrappers(content)
|
||||
if not content:
|
||||
return ParsedSkip(ScanIssueCode.NO_TEXT)
|
||||
if not content:
|
||||
return ParsedSkip(ScanIssueCode.NO_TEXT)
|
||||
return ParsedMessage(
|
||||
conversation_id=conversation_from_relative_path(context.relative_path),
|
||||
message_id=first_string(payload.get("id"), record.get("id")),
|
||||
|
||||
@@ -355,7 +355,7 @@ class JsonlSourceAdapter:
|
||||
source_file, checkpoint, header = await asyncio.to_thread(
|
||||
self._prepare, listed, options
|
||||
)
|
||||
_log.debug(
|
||||
_log.info(
|
||||
"import_file_read source=%s file=%d/%d file_bytes=%d planned_bytes=%d",
|
||||
self.descriptor.source_id,
|
||||
index,
|
||||
@@ -376,7 +376,7 @@ class JsonlSourceAdapter:
|
||||
else DEFAULT_IMPORT_MAX_FILE_BYTES
|
||||
)
|
||||
if source_file.size > budget:
|
||||
_log.debug(
|
||||
_log.info(
|
||||
"import_file_budgeted source=%s file=%d/%d file_bytes=%d budget=%d path=%s",
|
||||
self.descriptor.source_id,
|
||||
index,
|
||||
|
||||
@@ -7,7 +7,7 @@ import sqlite3
|
||||
import stat
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import UTC, datetime, timedelta, timezone
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from functools import partial
|
||||
|
||||
from pydantic import JsonValue
|
||||
@@ -82,11 +82,6 @@ _CURSOR_HOST_CONTEXT_START = re.compile(
|
||||
)
|
||||
_CURSOR_USER_QUERY = re.compile(r"<user_query\b")
|
||||
_CURSOR_TIMESTAMP_PREFIX = re.compile(r"<timestamp>[\s\S]*?</timestamp>\s*")
|
||||
_CURSOR_TIMESTAMP_VALUE = re.compile(
|
||||
r"<timestamp>\s*"
|
||||
r"([A-Za-z]+,\s+[A-Za-z]{3}\s+\d{1,2},\s+\d{4},\s+\d{1,2}:\d{2}\s+[AP]M)"
|
||||
r"\s+\(UTC([+-]\d{1,2})(?::(\d{2}))?\)\s*</timestamp>",
|
||||
)
|
||||
_CURSOR_IMAGE_FILES_PREFIX = re.compile(r"\[Image\]\s*<image_files>[\s\S]*?</image_files>\s*")
|
||||
_CURSOR_BARE_IMAGE_PREFIX = re.compile(r"(?:\[Image\]\s*)+")
|
||||
|
||||
@@ -128,24 +123,6 @@ def _strip_cursor_timestamp(content: str | None) -> str | None:
|
||||
return _CURSOR_TIMESTAMP_PREFIX.sub("", content, count=1) if content else None
|
||||
|
||||
|
||||
def _cursor_host_timestamp(content: str | None) -> datetime | None:
|
||||
"""Parse Cursor's user-visible timestamp envelope into UTC."""
|
||||
if not content:
|
||||
return None
|
||||
match = _CURSOR_TIMESTAMP_VALUE.search(content)
|
||||
if match is None:
|
||||
return None
|
||||
try:
|
||||
local = datetime.strptime(match[1], "%A, %b %d, %Y, %I:%M %p")
|
||||
hours = int(match[2])
|
||||
minutes = int(match[3] or "0")
|
||||
sign = -1 if hours < 0 else 1
|
||||
offset = timedelta(hours=hours, minutes=sign * minutes)
|
||||
return local.replace(tzinfo=timezone(offset)).astimezone(UTC)
|
||||
except (ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _strip_cursor_host_prefix(content: str | None) -> str | None:
|
||||
if not content:
|
||||
return content
|
||||
@@ -421,7 +398,10 @@ class CursorUnifiedSourceAdapter:
|
||||
if (
|
||||
isinstance(event, ScannedEvidence)
|
||||
and event.evidence.conversation_id in transcript_ids
|
||||
and not is_host_generated_user_turn(event.evidence.verbatim)
|
||||
and not (
|
||||
event.evidence.written_by is WrittenBy.AGENT
|
||||
and is_host_generated_user_turn(event.evidence.verbatim)
|
||||
)
|
||||
):
|
||||
yield ScanSkip(
|
||||
cursor=event.cursor,
|
||||
@@ -446,7 +426,10 @@ def _shadowed_by_transcript(event: ScanEvent, transcript_ids: set[str]) -> ScanE
|
||||
if (
|
||||
isinstance(event, ScannedEvidence)
|
||||
and event.evidence.conversation_id in transcript_ids
|
||||
and not is_host_generated_user_turn(event.evidence.verbatim)
|
||||
and not (
|
||||
event.evidence.written_by is WrittenBy.AGENT
|
||||
and is_host_generated_user_turn(event.evidence.verbatim)
|
||||
)
|
||||
):
|
||||
return ScanSkip(
|
||||
cursor=event.cursor,
|
||||
@@ -720,7 +703,6 @@ def _parse_transcript_record(
|
||||
project_name = path_parts[path_parts.index("projects") + 1] if "projects" in path_parts else ""
|
||||
session_name = context.relative_path.rsplit("/", 1)[-1].removesuffix(".jsonl")
|
||||
is_subagent = f"{project_name}\0{session_name}" in subagent_ids
|
||||
host_timestamp = _cursor_host_timestamp(content) if role == "user" else None
|
||||
created_at = _transcript_fallback_time(context)
|
||||
native_message_id = record.get("id") or message.get("id")
|
||||
message_id = (
|
||||
@@ -746,16 +728,7 @@ def _parse_transcript_record(
|
||||
created_at=created_at,
|
||||
workspace=workspace,
|
||||
record_type="cursor_transcript",
|
||||
origin_metadata=OriginMetadata(
|
||||
raw_meta={
|
||||
"time_basis": "file_mtime",
|
||||
**(
|
||||
{"turn_event_at": host_timestamp.isoformat()}
|
||||
if host_timestamp is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
),
|
||||
origin_metadata=OriginMetadata(raw_meta={"time_basis": "file_mtime"}),
|
||||
),
|
||||
tool_blocks_evidence(message.get("content"), block_types=TOOL_BLOCKS),
|
||||
)
|
||||
@@ -846,13 +819,6 @@ def _read_store_messages(
|
||||
reason_code=ScanIssueCode.UNKNOWN_RECORD,
|
||||
)
|
||||
continue
|
||||
host_timestamp = (
|
||||
_cursor_host_timestamp(content) if role == "user" else None
|
||||
)
|
||||
fallback_timestamp = datetime.fromtimestamp(
|
||||
source_file.modified_ns / 1_000_000_000,
|
||||
tz=UTC,
|
||||
) + timedelta(microseconds=rowid)
|
||||
parsed = ParsedMessage(
|
||||
conversation_id=conversation_id,
|
||||
message_id=str(record.get("id") or row["id"]),
|
||||
@@ -863,19 +829,8 @@ def _read_store_messages(
|
||||
else written_by_user
|
||||
),
|
||||
verbatim=cleaned,
|
||||
created_at=fallback_timestamp,
|
||||
workspace=_store_workspace(source_file),
|
||||
record_type="cursor_store_blob",
|
||||
origin_metadata=OriginMetadata(
|
||||
raw_meta={
|
||||
"time_basis": "file_mtime",
|
||||
**(
|
||||
{"turn_event_at": host_timestamp.isoformat()}
|
||||
if host_timestamp is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
),
|
||||
)
|
||||
yield ScannedEvidence(
|
||||
cursor=cursor,
|
||||
|
||||
@@ -57,8 +57,7 @@ _UNANSWERED_PREFIXES = (
|
||||
|
||||
|
||||
def _is_visible_user_row(role: str, written_by: str, verbatim: str) -> bool:
|
||||
del written_by # Legacy storage metadata; protocol role is authoritative.
|
||||
if role != "user":
|
||||
if role != "user" or written_by != WrittenBy.PERSON.value:
|
||||
return False
|
||||
return not verbatim.lstrip().lower().startswith(_UNANSWERED_PREFIXES)
|
||||
|
||||
@@ -469,54 +468,16 @@ class EvidenceStore:
|
||||
if anchored is not None:
|
||||
self._write_origins(connection, anchored, evidence.origin_meta, now)
|
||||
workspace, project = self._resolve_conversation_attribution(connection, evidence)
|
||||
existing = connection.execute(
|
||||
"SELECT role,revision FROM evidence WHERE evidence_id=?",
|
||||
(anchored,),
|
||||
).fetchone()
|
||||
if existing is None:
|
||||
raise RuntimeError("anchored evidence disappeared")
|
||||
if str(existing["role"]) != evidence.role:
|
||||
was_revised = True
|
||||
revision = int(existing["revision"]) + 1
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE evidence
|
||||
SET workspace=COALESCE(workspace,?),
|
||||
project=COALESCE(project,?),
|
||||
content_kind=?,role=?,created_at=COALESCE(?,created_at),
|
||||
verbatim=?,content_hash=?,revision=?,updated_at=?
|
||||
WHERE evidence_id=?
|
||||
""",
|
||||
(
|
||||
workspace,
|
||||
project,
|
||||
evidence.content_kind,
|
||||
evidence.role,
|
||||
evidence.created_at.isoformat() if evidence.created_at else None,
|
||||
evidence.verbatim,
|
||||
evidence.content_hash,
|
||||
revision,
|
||||
now,
|
||||
anchored,
|
||||
),
|
||||
)
|
||||
self._insert_current_version(
|
||||
connection,
|
||||
anchored,
|
||||
revision=revision,
|
||||
changed_at=now,
|
||||
)
|
||||
else:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE evidence
|
||||
SET workspace=COALESCE(workspace,?),
|
||||
project=COALESCE(project,?),
|
||||
updated_at=?
|
||||
WHERE evidence_id=?
|
||||
""",
|
||||
(workspace, project, now, anchored),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE evidence
|
||||
SET workspace = COALESCE(workspace, ?),
|
||||
project = COALESCE(project, ?),
|
||||
updated_at = ?
|
||||
WHERE evidence_id = ?
|
||||
""",
|
||||
(workspace, project, now, anchored),
|
||||
)
|
||||
record = self._get_with_connection(connection, anchored)
|
||||
if record is None:
|
||||
raise RuntimeError("anchored evidence disappeared")
|
||||
@@ -524,11 +485,7 @@ class EvidenceStore:
|
||||
from one_memory import ingest_journal
|
||||
|
||||
ingest_journal.record(connection, anchored, cause="import")
|
||||
return PutEvidenceResult(
|
||||
record=record,
|
||||
was_created=False,
|
||||
was_revised=was_revised,
|
||||
)
|
||||
return PutEvidenceResult(record=record, was_created=False, was_revised=False)
|
||||
insert = connection.execute(
|
||||
"""
|
||||
INSERT INTO evidence (
|
||||
@@ -846,20 +803,27 @@ class EvidenceStore:
|
||||
and same_scope[0]["idempotency_key"] != evidence.idempotency_key
|
||||
):
|
||||
return str(same_scope[0]["evidence_id"])
|
||||
elif (
|
||||
legacy_id := next(
|
||||
(
|
||||
anchored
|
||||
for key in (
|
||||
*evidence.written_by_derived_v3_idempotency_keys,
|
||||
*evidence.older_written_by_idempotency_keys,
|
||||
)
|
||||
if (anchored := claim_legacy_key(key)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
) is not None:
|
||||
elif (legacy_id := claim_legacy_key(evidence.legacy_derived_idempotency_key)) is not None:
|
||||
return legacy_id
|
||||
if not evidence.message_id and not evidence.correlation_id:
|
||||
# BUG-03 moved the derived key off the capture time and onto the
|
||||
# pre-cleaning text. A replay of a row stored under the old scheme
|
||||
# has to be recognized as the same message.
|
||||
replayed = connection.execute(
|
||||
"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE idempotency_key = ? AND user_id = ? AND source_id = ?
|
||||
AND conversation_id = ?
|
||||
""",
|
||||
(
|
||||
evidence.prior_derived_idempotency_key,
|
||||
evidence.user_id,
|
||||
evidence.source_id,
|
||||
evidence.conversation_id,
|
||||
),
|
||||
).fetchone()
|
||||
if replayed is not None:
|
||||
return str(replayed["evidence_id"])
|
||||
if (
|
||||
EvidenceOrigin.HOOK in evidence.origins
|
||||
and evidence.message_id is None
|
||||
@@ -874,7 +838,7 @@ class EvidenceStore:
|
||||
ON hook.evidence_id=e.evidence_id AND hook.origin='hook'
|
||||
WHERE e.user_id=? AND e.source_id=? AND e.conversation_id=?
|
||||
AND (e.workspace IS ? OR e.workspace IS NULL OR ? IS NULL)
|
||||
AND e.role=? AND e.content_hash=?
|
||||
AND e.role=? AND e.written_by=? AND e.content_hash=?
|
||||
AND e.message_id IS NOT NULL AND hook.evidence_id IS NULL
|
||||
AND ABS((julianday(e.created_at)-julianday(?))*86400.0) <= 5.0
|
||||
ORDER BY ABS((julianday(e.created_at)-julianday(?))*86400.0), e.evidence_id
|
||||
@@ -887,6 +851,7 @@ class EvidenceStore:
|
||||
evidence.workspace,
|
||||
evidence.workspace,
|
||||
evidence.role,
|
||||
evidence.written_by,
|
||||
evidence.content_hash,
|
||||
evidence.created_at.isoformat(),
|
||||
evidence.created_at.isoformat(),
|
||||
@@ -907,7 +872,7 @@ class EvidenceStore:
|
||||
AND (e.workspace IS ? OR e.workspace IS NULL OR ? IS NULL)
|
||||
-- Unary plus keeps the coverage index from looking better than
|
||||
-- the message index, which narrows to this exact text.
|
||||
AND e.role = ? AND e.content_hash = ?
|
||||
AND e.role = ? AND +e.written_by = ? AND e.content_hash = ?
|
||||
AND e.message_id IS NULL
|
||||
""",
|
||||
(
|
||||
@@ -917,6 +882,7 @@ class EvidenceStore:
|
||||
evidence.workspace,
|
||||
evidence.workspace,
|
||||
evidence.role,
|
||||
evidence.written_by,
|
||||
evidence.content_hash,
|
||||
),
|
||||
).fetchall()
|
||||
@@ -1281,7 +1247,7 @@ class EvidenceStore:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE user_id = 'local' AND role = 'user'
|
||||
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
|
||||
ORDER BY imported_at, evidence_id
|
||||
LIMIT ?
|
||||
""",
|
||||
@@ -1335,7 +1301,7 @@ class EvidenceStore:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE user_id = 'local' AND role = 'user'
|
||||
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
|
||||
AND ({predicates})
|
||||
ORDER BY imported_at, evidence_id
|
||||
LIMIT ?
|
||||
@@ -1367,7 +1333,7 @@ class EvidenceStore:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE user_id = 'local' AND role = 'user'
|
||||
WHERE user_id = 'local' AND role = 'user' AND written_by = 'person'
|
||||
AND workspace IS NOT NULL
|
||||
AND ({predicates})
|
||||
ORDER BY imported_at, evidence_id
|
||||
@@ -1510,7 +1476,7 @@ class EvidenceStore:
|
||||
)
|
||||
values: list[object] = [user_id, *(source_ids or ())]
|
||||
visible_user = """
|
||||
role='user' AND NOT (
|
||||
role='user' AND written_by='person' AND NOT (
|
||||
lower(ltrim(verbatim)) LIKE '<codex_internal_context%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<environment_context>%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<heartbeat>%' OR
|
||||
@@ -1519,7 +1485,7 @@ class EvidenceStore:
|
||||
)
|
||||
"""
|
||||
selected_visible_user = """
|
||||
e.role='user' AND NOT (
|
||||
e.role='user' AND e.written_by='person' AND NOT (
|
||||
lower(ltrim(e.verbatim)) LIKE '<codex_internal_context%' OR
|
||||
lower(ltrim(e.verbatim)) LIKE '<environment_context>%' OR
|
||||
lower(ltrim(e.verbatim)) LIKE '<heartbeat>%' OR
|
||||
@@ -1672,7 +1638,7 @@ class EvidenceStore:
|
||||
if seed is None:
|
||||
return ConversationTurnPage((), {}, 0, offset, limit)
|
||||
visible_user = """
|
||||
role='user' AND NOT (
|
||||
role='user' AND written_by='person' AND NOT (
|
||||
lower(ltrim(verbatim)) LIKE '<codex_internal_context%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<environment_context>%' OR
|
||||
lower(ltrim(verbatim)) LIKE '<heartbeat>%' OR
|
||||
@@ -1765,7 +1731,7 @@ class EvidenceStore:
|
||||
AND first.conversation_id = evidence.conversation_id
|
||||
AND first.workspace IS evidence.workspace
|
||||
AND (
|
||||
first.role = 'user'
|
||||
(first.role = 'user' AND first.written_by = 'person')
|
||||
OR first.content_kind = 'memory_content'
|
||||
)
|
||||
ORDER BY julianday(COALESCE(first.created_at, first.imported_at)),
|
||||
@@ -2159,6 +2125,8 @@ class EvidenceStore:
|
||||
(
|
||||
existing["content_kind"] != evidence.content_kind,
|
||||
existing["role"] != evidence.role,
|
||||
evidence.written_by is not WrittenBy.UNKNOWN
|
||||
and existing["written_by"] != evidence.written_by,
|
||||
existing["content_hash"] != evidence.content_hash,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -111,7 +111,7 @@ class ImportSupervisor:
|
||||
)
|
||||
)
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.supervisor",
|
||||
event="import.queued",
|
||||
outcome="queued",
|
||||
@@ -198,7 +198,7 @@ class ImportSupervisor:
|
||||
started = monotonic()
|
||||
failure_type: str | None = None
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="import.supervisor",
|
||||
event="import.started",
|
||||
outcome="started",
|
||||
|
||||
@@ -85,14 +85,15 @@ def _state(connection: sqlite3.Connection, evidence_id: str) -> dict[str, str |
|
||||
value = evidence.model_dump(mode="json")
|
||||
# The stored normalized content_hash is insufficient for exact-content leases.
|
||||
content = _digest(value["verbatim"])
|
||||
material = _digest({key: value[key] for key in ("verbatim", "role", "content_kind")})
|
||||
material = _digest(
|
||||
{key: value[key] for key in ("verbatim", "role", "written_by", "content_kind")}
|
||||
)
|
||||
origins = _digest(value["origin_meta"])
|
||||
identity = _digest(
|
||||
{
|
||||
key: item
|
||||
for key, item in value.items()
|
||||
if key
|
||||
not in {"verbatim", "written_by", "origin_meta", "imported_at", "updated_at"}
|
||||
if key not in {"verbatim", "origin_meta", "imported_at", "updated_at"}
|
||||
}
|
||||
)
|
||||
return dict(
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import TextIO, cast
|
||||
|
||||
@@ -105,13 +104,7 @@ def capture_hook(source_id: str, payload: object, database: Path) -> bool:
|
||||
# Cursor also runs ~/.claude/settings.json hooks; its own hook already files the turn.
|
||||
if source_id != "cursor" and "cursor_version" in payload:
|
||||
return False
|
||||
if not database.is_file():
|
||||
return False
|
||||
rejection = capture_rejection(database, source_id)
|
||||
if rejection is not None:
|
||||
# Silent, and the reason is counted rather than printed: a hook that
|
||||
# spoke would put this into the host's turn as text.
|
||||
record_capture_rejection(database, source_id, rejection)
|
||||
if not database.is_file() or not _sync_open(database, source_id):
|
||||
return False
|
||||
if source_id == "pi" and _is_automated_session(first_string(payload.get("cwd"))):
|
||||
return False
|
||||
@@ -146,79 +139,17 @@ def capture_hook(source_id: str, payload: object, database: Path) -> bool:
|
||||
|
||||
|
||||
def _sync_open(database: Path, source_id: str) -> bool:
|
||||
return capture_rejection(database, source_id) is None
|
||||
|
||||
|
||||
class CaptureRejection(StrEnum):
|
||||
"""Why a hook's turn was dropped, or nothing at all.
|
||||
|
||||
Capture fails open on purpose: a hook that raises or prints would be read
|
||||
back into the host's turn, so a refusal has to be silent and exit 0. It was
|
||||
also completely invisible, which is why "持续同步开着但一条都没进" could not be
|
||||
told apart from "capture 坏了". These values name the reason without any
|
||||
payload, path, session or message — the only part that may be recorded.
|
||||
"""
|
||||
|
||||
SYNC_DISABLED = "sync_disabled"
|
||||
GRANT_MISSING = "grant_missing"
|
||||
GRANT_REVOKED = "grant_revoked"
|
||||
|
||||
|
||||
def capture_rejection(database: Path, source_id: str) -> CaptureRejection | None:
|
||||
"""None when this turn may be captured, otherwise why it may not."""
|
||||
from one_memory.memory_server.config_state import ConfigStore
|
||||
|
||||
if not ConfigStore(database.parent).sync_enabled(source_id):
|
||||
return CaptureRejection.SYNC_DISABLED
|
||||
return False
|
||||
with main_database(database).read() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT revoked_at FROM source_grants WHERE user_id='local' AND source_id=?",
|
||||
"SELECT 1 FROM source_grants WHERE user_id='local' AND source_id=? "
|
||||
"AND revoked_at IS NULL",
|
||||
(source_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return CaptureRejection.GRANT_MISSING
|
||||
# Indexed, not named: this connection has no row factory, and a hook is
|
||||
# not the place to add one.
|
||||
if row[0] is not None:
|
||||
return CaptureRejection.GRANT_REVOKED
|
||||
return None
|
||||
|
||||
|
||||
def record_capture_rejection(
|
||||
database: Path,
|
||||
source_id: str,
|
||||
reason: CaptureRejection,
|
||||
*,
|
||||
window_seconds: float = 600.0,
|
||||
) -> None:
|
||||
"""Note one dropped turn, at most once per window per source and reason.
|
||||
|
||||
Best effort by construction: a hook runs inside somebody's turn, so a sink
|
||||
that is slow, full or read-only must never surface here. The record is a
|
||||
count and nothing else — no payload, no path, no session or conversation
|
||||
id — because this file is meant to be handed to support as is.
|
||||
|
||||
One small file per source and reason, rather than a line per turn: capture
|
||||
runs on every prompt, and a log that grew with the conversation would be a
|
||||
privacy problem wearing a diagnosis as a disguise.
|
||||
"""
|
||||
try:
|
||||
directory = database.parent / "logs"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
marker = directory / f"capture-rejection-{source_id}-{reason.value}.json"
|
||||
now = time.time()
|
||||
payload: dict[str, object] = {"source_id": source_id, "reason": reason.value}
|
||||
if marker.exists() and now - marker.stat().st_mtime < window_seconds:
|
||||
payload = json.loads(marker.read_text(encoding="utf8"))
|
||||
previous = payload.get("count", 1)
|
||||
payload["count"] = int(previous) + 1 if isinstance(previous, (str, int, float)) else 2
|
||||
else:
|
||||
payload["count"] = 1
|
||||
payload["window_started_at"] = datetime.fromtimestamp(now, UTC).isoformat()
|
||||
payload["last_at"] = payload["window_started_at"]
|
||||
marker.write_text(json.dumps(payload, sort_keys=True), encoding="utf8")
|
||||
except Exception:
|
||||
return
|
||||
return row is not None
|
||||
|
||||
|
||||
def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed: bool) -> None:
|
||||
@@ -236,7 +167,7 @@ def _submit_live_memory(store: EvidenceStore, record: EvidenceRecord, *, changed
|
||||
):
|
||||
if (
|
||||
record in chunk
|
||||
and any(item.role == "user" for item in chunk)
|
||||
and any(item.role == "user" and item.written_by is WrittenBy.PERSON for item in chunk)
|
||||
and any(item.role == "assistant" for item in chunk)
|
||||
):
|
||||
intake.submit_records(chunk, priority="live")
|
||||
|
||||
@@ -160,9 +160,7 @@ class MatterDiscovery:
|
||||
task_by_id: dict[str, MemoryTask] = {}
|
||||
for chunk in chunks:
|
||||
for record in chunk:
|
||||
task = await asyncio.to_thread(
|
||||
processor.tasks.task_for_evidence, record.evidence_id
|
||||
)
|
||||
task = await asyncio.to_thread(processor.tasks.task_for_evidence, record.evidence_id)
|
||||
if task is not None and task.status is MemoryTaskStatus.COMPLETED:
|
||||
task_by_id[task.task_id] = task
|
||||
tasks = tuple(sorted(task_by_id.values(), key=lambda task: (task.created_at, task.task_id)))
|
||||
|
||||
@@ -270,7 +270,7 @@ async def reconcile_current(
|
||||
(
|
||||
r
|
||||
for r in records
|
||||
if r.role == "user"
|
||||
if r.written_by == "person"
|
||||
and r.created_at
|
||||
and (r.source_id, r.conversation_id) in related
|
||||
),
|
||||
|
||||
@@ -51,19 +51,14 @@ def validate_claim(claim: Claim, records: Mapping[str, EvidenceRecord | SampleRe
|
||||
text = record.text if isinstance(record, SampleRecord) else record.verbatim
|
||||
if not citation.quote.strip() or citation.quote not in text:
|
||||
raise ValueError("citation quote is not in the source")
|
||||
author = {
|
||||
"user": "person",
|
||||
"assistant": "agent",
|
||||
"tool": "tool",
|
||||
}.get(record.role, "unknown")
|
||||
authors.add(author)
|
||||
authors.add(str(record.written_by))
|
||||
created = record.created_at
|
||||
citations.append(
|
||||
citation.model_copy(
|
||||
update={
|
||||
"revision": record.revision,
|
||||
"source_id": record.source_id,
|
||||
"written_by": author,
|
||||
"written_by": str(record.written_by),
|
||||
"created_at": created.isoformat() if isinstance(created, datetime) else created,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -290,7 +290,7 @@ def read_memory_library(
|
||||
COALESCE(
|
||||
MIN(
|
||||
CASE WHEN e.role = 'user'
|
||||
AND e.role = 'user'
|
||||
AND e.written_by IN ('person', 'user')
|
||||
THEN e.created_at END
|
||||
),
|
||||
MIN(e.created_at)
|
||||
|
||||
@@ -45,7 +45,6 @@ from one_memory.hy_memory_bridge import (
|
||||
HyMemoryOutput,
|
||||
HyMemoryTurn,
|
||||
builtin_retrieval_snapshot,
|
||||
coding_tool_result_max_bytes,
|
||||
decide_profile_graph_width,
|
||||
hy_memory_store_exists,
|
||||
inspect_profile_graph_width,
|
||||
@@ -53,7 +52,7 @@ from one_memory.hy_memory_bridge import (
|
||||
from one_memory.hy_runtime_config import hy_memory_version
|
||||
from one_memory.hy_source_context import HySourceContext, HySupportedMemory
|
||||
from one_memory.imports.access import opaque_workspace_id
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceOrigin, EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.runner import live_sync_arrival
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
@@ -163,7 +162,7 @@ def _report_stale_known_reasons(task_id: str, reasons: Mapping[str, int]) -> Non
|
||||
"""Emit one schema-valid scalar diagnostic per stale-memory reason."""
|
||||
for reason, count in sorted(reasons.items()):
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.context",
|
||||
event="known_memory_stale_omitted",
|
||||
outcome="omitted",
|
||||
@@ -174,21 +173,6 @@ def _report_stale_known_reasons(task_id: str, reasons: Mapping[str, int]) -> Non
|
||||
|
||||
MEMORY_INGEST_POLICY_VERSION = "hy-native-v23"
|
||||
UNIFIED_HY_AGENT_ID = "one-memory"
|
||||
#: Deterministic refusals raised before the payload reaches HY, so nothing was
|
||||
#: sent even though the invocation marker is already recorded by then: the
|
||||
#: write bindings are built inside `ingest` ahead of the call, and the input
|
||||
#: window is checked while the request is frozen.
|
||||
#:
|
||||
#: They are terminal rather than unknown. `unknown` here means "HY may hold a
|
||||
#: write we cannot confirm", and that is exactly what did not happen — parking
|
||||
#: the task on it hid the reason and stopped the queue behind it (issue #44).
|
||||
#: Kept in step with `durable_queue.PRE_SEND_REFUSAL_CODES`, which the durable
|
||||
#: write path applies to the same codes.
|
||||
PRE_SEND_REFUSAL_CODES = frozenset(
|
||||
{
|
||||
"hy_memory_input_exceeds_window",
|
||||
}
|
||||
)
|
||||
_BACKFILL_BATCH_SIZE = 8
|
||||
_BACKFILL_DRAIN_BUDGET_SECONDS = 330.0
|
||||
_HISTORY_MERGE_TURNS = 8
|
||||
@@ -296,7 +280,6 @@ class MemoryTaskRunner:
|
||||
*,
|
||||
timeout: float | None = None,
|
||||
before_hy_invocation: Callable[[], Awaitable[None]] | None = None,
|
||||
input_records: tuple[EvidenceRecord, ...] | None = None,
|
||||
) -> MemoryTask:
|
||||
started = monotonic()
|
||||
diagnostic_event(
|
||||
@@ -307,28 +290,13 @@ class MemoryTaskRunner:
|
||||
task_id=task_id,
|
||||
)
|
||||
try:
|
||||
if input_records is None:
|
||||
body = (
|
||||
self._run_without_content_permission(task_id)
|
||||
if before_hy_invocation is None
|
||||
else self._run_without_content_permission(
|
||||
task_id,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
)
|
||||
)
|
||||
else:
|
||||
body = (
|
||||
self._run_without_content_permission(
|
||||
task_id,
|
||||
input_records=input_records,
|
||||
)
|
||||
if before_hy_invocation is None
|
||||
else self._run_without_content_permission(
|
||||
task_id,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
input_records=input_records,
|
||||
)
|
||||
body = (
|
||||
self._run_without_content_permission(task_id)
|
||||
if before_hy_invocation is None
|
||||
else self._run_without_content_permission(
|
||||
task_id, before_hy_invocation=before_hy_invocation
|
||||
)
|
||||
)
|
||||
# The ceiling bounds how long this task holds its harness's slot:
|
||||
# a harness is serial, so one hung task starves every turn behind
|
||||
# it. It is insurance against a single bad call, not a way to run
|
||||
@@ -399,7 +367,7 @@ class MemoryTaskRunner:
|
||||
@staticmethod
|
||||
def _log_step(task_id: str, step: int, action: str, started: float) -> None:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.task",
|
||||
event="memory_task.step_completed",
|
||||
outcome="succeeded",
|
||||
@@ -413,7 +381,6 @@ class MemoryTaskRunner:
|
||||
task_id: str,
|
||||
*,
|
||||
before_hy_invocation: Callable[[], Awaitable[None]] | None = None,
|
||||
input_records: tuple[EvidenceRecord, ...] | None = None,
|
||||
) -> MemoryTask:
|
||||
# Every store call here is synchronous, and several are writes that
|
||||
# wait on the write lock for up to five seconds. Made from the loop,
|
||||
@@ -467,9 +434,7 @@ class MemoryTaskRunner:
|
||||
step_started = monotonic()
|
||||
frozen = await in_the_background(self.tasks.inputs, task.task_id)
|
||||
chunk: tuple[EvidenceRecord, ...] | None
|
||||
if input_records is not None:
|
||||
chunk = input_records
|
||||
elif frozen:
|
||||
if frozen:
|
||||
try:
|
||||
chunk = await in_the_background(self._frozen_chunk, task)
|
||||
except HyMemoryBridgeError as error:
|
||||
@@ -502,11 +467,8 @@ class MemoryTaskRunner:
|
||||
self._source_context, task, request.source_context
|
||||
)
|
||||
request = request.model_copy(update={"source_context": context})
|
||||
uses_frozen_input = bool(frozen) or input_records is not None
|
||||
grant_token = (
|
||||
await in_the_background(self._grant_token, task, chunk[0])
|
||||
if uses_frozen_input
|
||||
else None
|
||||
await in_the_background(self._grant_token, task, chunk[0]) if frozen else None
|
||||
)
|
||||
memory_contents = tuple(
|
||||
sanitize_model_input(record.verbatim)
|
||||
@@ -549,24 +511,19 @@ class MemoryTaskRunner:
|
||||
agent_id=request.agent_id,
|
||||
),
|
||||
)
|
||||
if uses_frozen_input:
|
||||
if input_records is None:
|
||||
await in_the_background(self._frozen_chunk, task)
|
||||
if frozen:
|
||||
await in_the_background(self._frozen_chunk, task)
|
||||
if grant_token != await in_the_background(self._grant_token, task, chunk[0]):
|
||||
raise HyMemoryBridgeError("hy_memory_source_changed")
|
||||
if request.source_context is not None:
|
||||
await in_the_background(
|
||||
self._check_source_context,
|
||||
request.source_context,
|
||||
skip_current=input_records is not None,
|
||||
)
|
||||
await in_the_background(self._check_source_context, request.source_context)
|
||||
# One thread hop for the whole batch rather than a dozen writes
|
||||
# made from the loop.
|
||||
step_started = monotonic()
|
||||
await in_the_background(
|
||||
self._record_outputs,
|
||||
task=task,
|
||||
chunk=chunk,
|
||||
chunk=self._original_records(chunk),
|
||||
outputs=outputs,
|
||||
request=request,
|
||||
derivation=derivation,
|
||||
@@ -576,13 +533,6 @@ class MemoryTaskRunner:
|
||||
self._log_step(task_id, 6, "governance", step_started)
|
||||
except HyMemoryBridgeError as error:
|
||||
code = str(error)
|
||||
if code == "model_sedimentation_disabled" and not hy_invoked:
|
||||
return await in_the_background(
|
||||
self.tasks.release_unstarted,
|
||||
task_id,
|
||||
runner_token=runner_token,
|
||||
error_code=code,
|
||||
)
|
||||
model_wait_errors = {
|
||||
"hy_memory_unavailable",
|
||||
"hy_memory_rate_limit",
|
||||
@@ -597,16 +547,6 @@ class MemoryTaskRunner:
|
||||
"hy_memory_model_timeout",
|
||||
}
|
||||
safe_terminal_after_invocation = {"hy_memory_import_no_readable_output"}
|
||||
if code in PRE_SEND_REFUSAL_CODES:
|
||||
# Raised before the call, so the marker only says "about to":
|
||||
# fail with the reason rather than parking on a write HY never
|
||||
# saw. See the constant and issue #44.
|
||||
return await in_the_background(
|
||||
self.tasks.fail,
|
||||
task_id,
|
||||
runner_token=runner_token,
|
||||
error_code=code,
|
||||
)
|
||||
if hy_invoked and (
|
||||
code == "hy_memory_evidence_changed"
|
||||
or (
|
||||
@@ -804,18 +744,8 @@ class MemoryTaskRunner:
|
||||
raise HyMemoryBridgeError("hy_memory_evidence_changed") from error
|
||||
return context
|
||||
|
||||
def _check_source_context(
|
||||
self,
|
||||
context: HySourceContext,
|
||||
*,
|
||||
skip_current: bool = False,
|
||||
) -> None:
|
||||
checked = (
|
||||
(*context.history, *context.supporting)
|
||||
if skip_current
|
||||
else (*context.current, *context.history, *context.supporting)
|
||||
)
|
||||
for original in checked:
|
||||
def _check_source_context(self, context: HySourceContext) -> None:
|
||||
for original in (*context.current, *context.history, *context.supporting):
|
||||
record = self.evidence.get(original.evidence_id)
|
||||
if record is None or not matches_evidence_ref(record, evidence_ref(original)):
|
||||
raise HyMemoryBridgeError("hy_memory_evidence_changed")
|
||||
@@ -916,7 +846,7 @@ class MemoryTaskRunner:
|
||||
# HY's own attribution. An undecided memory counts as the person's:
|
||||
# HY's prompt defaults that way.
|
||||
from_agent = output.owner == "agent" or all(
|
||||
record.role == "assistant"
|
||||
record.written_by is WrittenBy.AGENT
|
||||
and record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
for record in support
|
||||
)
|
||||
@@ -1086,20 +1016,15 @@ class MemoryTaskRunner:
|
||||
task: MemoryTask,
|
||||
chunk: tuple[EvidenceRecord, ...],
|
||||
) -> HyMemoryIngestRequest:
|
||||
if not chunk:
|
||||
# A task with no inputs has nothing to name the request after and
|
||||
# nothing to attribute an answer to. Left alone this reached
|
||||
# `chunk[0]` below and came out as IndexError, which no caller can
|
||||
# read: a reconcile pass catches domain errors per task and moves
|
||||
# on, so one such task ended the whole pass and left every other
|
||||
# unknown task unreconciled. On this machine 68 of 303 unknown
|
||||
# tasks had no inputs, and the pass raised on the first of them.
|
||||
raise HyMemoryBridgeError("hy_memory_task_without_inputs")
|
||||
has_raw = any(
|
||||
record.content_kind is EvidenceContentKind.RAW_CONVERSATION for record in chunk
|
||||
)
|
||||
source = next(
|
||||
(record for record in chunk if record.role == "user"),
|
||||
(
|
||||
record
|
||||
for record in chunk
|
||||
if record.role == "user" and record.written_by is WrittenBy.PERSON
|
||||
),
|
||||
chunk[0],
|
||||
)
|
||||
request_turns = tuple(
|
||||
@@ -1108,9 +1033,12 @@ class MemoryTaskRunner:
|
||||
if (
|
||||
record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and (
|
||||
# Protocol role is authoritative. A sub-agent question is
|
||||
# an ordinary user message regardless of a host parser's
|
||||
# former author guess.
|
||||
# D98: any `user` message goes, whoever the parser says
|
||||
# wrote it. A sub agent's question is ordinary
|
||||
# user-assistant data; the author field cannot tell who
|
||||
# wrote a message, and filtering on it dropped the sub
|
||||
# agent's turns. `written_by` still labels the display and
|
||||
# decides owner — it no longer decides what is sent.
|
||||
record.role == "user"
|
||||
# The assistant's side goes too. Without it 「就按你说的那个来」
|
||||
# arrives with nothing to resolve it against, and HY's own
|
||||
@@ -1121,7 +1049,7 @@ class MemoryTaskRunner:
|
||||
or record.role == "assistant"
|
||||
# Tool calls go as they happened; whether they make the
|
||||
# turn a coding turn is HY's call, not ours.
|
||||
or record.role == "tool"
|
||||
or (record.role == "tool" and record.written_by is WrittenBy.TOOL)
|
||||
)
|
||||
)
|
||||
or (not has_raw and record.content_kind is EvidenceContentKind.MEMORY_CONTENT)
|
||||
@@ -1290,7 +1218,6 @@ class MemoryProcessingSupervisor:
|
||||
max_concurrent_tasks: int | None = None,
|
||||
source_writes_permitted: Callable[[str], bool] | None = None,
|
||||
service_levels: ServiceLevelScheduler | None = None,
|
||||
sedimentation_enabled: Callable[[], bool] | None = None,
|
||||
) -> None:
|
||||
self.tasks = tasks
|
||||
self.profiles = profiles
|
||||
@@ -1303,12 +1230,6 @@ class MemoryProcessingSupervisor:
|
||||
#: absent every source is treated as writable, which is what the
|
||||
#: product did before the switch existed.
|
||||
self.source_writes_permitted = source_writes_permitted
|
||||
#: User-level "stop turning conversations into memories" switch. Optional
|
||||
#: so tests and a Core without a config file keep their current
|
||||
#: behaviour; when absent sedimentation is on, which is what the product
|
||||
#: did before the switch existed. Read per decision rather than cached,
|
||||
#: so stopping takes effect on the next dispatch without a restart.
|
||||
self.sedimentation_enabled = sedimentation_enabled
|
||||
self.service_levels = service_levels or ServiceLevelScheduler()
|
||||
self.extraction_settings = extraction_settings or ExtractionSettingsStore(
|
||||
data_dir / "one-memory.sqlite3"
|
||||
@@ -1362,26 +1283,12 @@ class MemoryProcessingSupervisor:
|
||||
self._orphan_reclaimer: asyncio.Task[None] | None = None
|
||||
self._diagnostic_idle_reason: str | None = None
|
||||
|
||||
def _sedimentation_stopped(self) -> bool:
|
||||
"""Whether the user stopped model-driven sedimentation.
|
||||
|
||||
Stopping must not look like a broken pipeline: nothing is sent to the
|
||||
model, so the pending count stays as it is and the scheduler reports
|
||||
why it is idle. Starting again picks the same work up.
|
||||
"""
|
||||
if self.sedimentation_enabled is None:
|
||||
return False
|
||||
if self.sedimentation_enabled():
|
||||
return False
|
||||
self._record_idle("sedimentation_stopped")
|
||||
return True
|
||||
|
||||
def _record_idle(self, reason: str) -> None:
|
||||
if self._diagnostic_idle_reason == reason:
|
||||
return
|
||||
self._diagnostic_idle_reason = reason
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.idle",
|
||||
outcome="idle",
|
||||
@@ -1403,8 +1310,7 @@ class MemoryProcessingSupervisor:
|
||||
) -> tuple[MemoryTask, ...]:
|
||||
"""Process a bounded, authorized snapshot through the existing HY runner."""
|
||||
async with self._explicit_processing_lock(records):
|
||||
if self._sedimentation_stopped():
|
||||
raise HyMemoryBridgeError("model_sedimentation_disabled")
|
||||
records = await in_the_background(self._original_records, records)
|
||||
active = await in_the_background(self.profiles.active, self.secrets)
|
||||
if active is None or (profile_id is not None and profile_id != active.profile_id):
|
||||
raise HyMemoryBridgeError("model_profile_not_active")
|
||||
@@ -1419,6 +1325,13 @@ class MemoryProcessingSupervisor:
|
||||
raise HyMemoryBridgeError("hy_memory_source_disabled")
|
||||
conversations: dict[tuple[str, str, str | None], list[EvidenceRecord]] = {}
|
||||
for record in records:
|
||||
stored = await in_the_background(self.evidence.get, record.evidence_id)
|
||||
if (
|
||||
stored is None
|
||||
or stored.revision != record.revision
|
||||
or stored.content_hash != record.content_hash
|
||||
):
|
||||
raise ValueError("hy_memory_evidence_changed")
|
||||
conversations.setdefault(
|
||||
(record.source_id, record.conversation_id, record.workspace), []
|
||||
).append(record)
|
||||
@@ -1435,7 +1348,7 @@ class MemoryProcessingSupervisor:
|
||||
for conversation in conversations.values()
|
||||
for chunk in _conversation_chunks(conversation, max_turns=12, completed_only=True)
|
||||
]
|
||||
prepared: list[tuple[MemoryTask, tuple[EvidenceRecord, ...]]] = []
|
||||
prepared = []
|
||||
for chunk in chunks:
|
||||
task = await in_the_background(
|
||||
self._enqueue_chunk,
|
||||
@@ -1443,12 +1356,12 @@ class MemoryProcessingSupervisor:
|
||||
active,
|
||||
freeze_inputs=True,
|
||||
)
|
||||
prepared.append((task, chunk))
|
||||
prepared.append(task)
|
||||
runner = self.runner_factory()
|
||||
completed = []
|
||||
if on_progress:
|
||||
on_progress(tuple(task for task, _chunk in prepared), 0)
|
||||
for task, chunk in prepared:
|
||||
on_progress(tuple(prepared), 0)
|
||||
for task in prepared:
|
||||
if task.status is not MemoryTaskStatus.COMPLETED:
|
||||
# Another importer may already own this task. Never race its claim.
|
||||
task = await in_the_background(self.tasks.require, task.task_id)
|
||||
@@ -1477,41 +1390,23 @@ class MemoryProcessingSupervisor:
|
||||
work_id=task.task_id,
|
||||
scope_key=chunk[0].source_id,
|
||||
):
|
||||
if isinstance(runner, MemoryTaskRunner):
|
||||
task = (
|
||||
await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
input_records=chunk,
|
||||
)
|
||||
if before_hy_invocation is None
|
||||
else await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
input_records=chunk,
|
||||
)
|
||||
)
|
||||
else:
|
||||
task = (
|
||||
await runner.run(task.task_id, timeout=timeout)
|
||||
if before_hy_invocation is None
|
||||
else await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
)
|
||||
task = (
|
||||
await runner.run(task.task_id, timeout=timeout)
|
||||
if before_hy_invocation is None
|
||||
else await runner.run(
|
||||
task.task_id,
|
||||
timeout=timeout,
|
||||
before_hy_invocation=before_hy_invocation,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
_unmark_direct_task(self.tasks.database_path, task.task_id)
|
||||
if task.status is not MemoryTaskStatus.COMPLETED:
|
||||
raise HyMemoryBridgeError(task.last_error_code or "hy_memory_incomplete")
|
||||
await in_the_background(runner._frozen_chunk, task)
|
||||
completed.append(task)
|
||||
if on_progress:
|
||||
on_progress(
|
||||
tuple(prepared_task for prepared_task, _chunk in prepared),
|
||||
len(completed),
|
||||
)
|
||||
on_progress(tuple(prepared), len(completed))
|
||||
expected = {record.evidence_id for record in records if _is_hy_turn_record(record)}
|
||||
actual = {
|
||||
row["evidence_id"] for task in completed for row in self.tasks.inputs(task.task_id)
|
||||
@@ -1772,8 +1667,6 @@ class MemoryProcessingSupervisor:
|
||||
def enqueue_evidence(self, record: EvidenceRecord) -> MemoryTask | None:
|
||||
if self._shutting_down:
|
||||
return None
|
||||
if self._sedimentation_stopped():
|
||||
return None
|
||||
if (
|
||||
record.role not in {"user", "assistant"}
|
||||
and record.content_kind is not EvidenceContentKind.MEMORY_CONTENT
|
||||
@@ -1806,8 +1699,6 @@ class MemoryProcessingSupervisor:
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
) -> list[QueuedWork]:
|
||||
"""Synchronous compatibility entry; preparation never starts a model."""
|
||||
if self._sedimentation_stopped():
|
||||
return []
|
||||
work = [item for batch in self._prepare_backfill_batches(should_stop) for item in batch]
|
||||
return [] if should_stop() or self._shutting_down else work
|
||||
|
||||
@@ -1816,7 +1707,7 @@ class MemoryProcessingSupervisor:
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
) -> Generator[list[QueuedWork], None, None]:
|
||||
"""Read bodies as needed; yield the first task before the next conversation."""
|
||||
if should_stop() or self._shutting_down or self._sedimentation_stopped():
|
||||
if should_stop() or self._shutting_down:
|
||||
return
|
||||
active = self.profiles.active(self.secrets)
|
||||
if active is None:
|
||||
@@ -1827,7 +1718,7 @@ class MemoryProcessingSupervisor:
|
||||
visited = 0
|
||||
seed_ids = self.evidence.list_conversation_seed_ids()
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_inventory",
|
||||
outcome="observed",
|
||||
@@ -1897,7 +1788,7 @@ class MemoryProcessingSupervisor:
|
||||
started = monotonic()
|
||||
prepared_count = 0
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_started",
|
||||
outcome="started",
|
||||
@@ -1909,7 +1800,6 @@ class MemoryProcessingSupervisor:
|
||||
stop = threading.Event()
|
||||
batches = self._prepare_backfill_batches(stop.is_set)
|
||||
exhausted = False
|
||||
stopped_saturated = False
|
||||
rounds = 0
|
||||
scan_error: BaseException | None = None
|
||||
try:
|
||||
@@ -1925,21 +1815,20 @@ class MemoryProcessingSupervisor:
|
||||
>= self._backfill_round_budget_seconds
|
||||
):
|
||||
break
|
||||
if (
|
||||
while (
|
||||
cohort_size is not None
|
||||
and len(self._queue) + len(self._in_flight) >= cohort_size
|
||||
):
|
||||
# A bounded scan stops at its ceiling instead of
|
||||
# waiting for the queue to drain. Waiting here slept
|
||||
# inside the scan lock, so whoever asked for the
|
||||
# scan — a switch toggle, over HTTP — waited with
|
||||
# it, and on a saturated backlog that is not a
|
||||
# moment but the whole backlog. The caller's
|
||||
# timeout then reported a failure for work that had
|
||||
# already been accepted.
|
||||
stopped_saturated = True
|
||||
exhausted = True
|
||||
break
|
||||
if self._shutting_down:
|
||||
return
|
||||
if (
|
||||
handled_batch
|
||||
and self._backfill_clock() - round_started
|
||||
>= self._backfill_round_budget_seconds
|
||||
):
|
||||
break
|
||||
self._record_idle("queue_saturated")
|
||||
await asyncio.sleep(0.1)
|
||||
if (
|
||||
handled_batch
|
||||
and self._backfill_clock() - round_started
|
||||
@@ -1958,7 +1847,7 @@ class MemoryProcessingSupervisor:
|
||||
on_first_scheduled()
|
||||
if not exhausted and not self._shutting_down:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_yielded",
|
||||
outcome="yielded",
|
||||
@@ -1988,18 +1877,14 @@ class MemoryProcessingSupervisor:
|
||||
stop.set()
|
||||
batches.close()
|
||||
if prepared_count == 0 and not self._shutting_down:
|
||||
if stopped_saturated:
|
||||
# Parked behind a full queue, not out of work: the status
|
||||
# must not read "no_tasks" while hundreds wait.
|
||||
self._record_idle("queue_saturated")
|
||||
elif scan_error is None:
|
||||
if scan_error is None:
|
||||
self._record_idle("no_tasks")
|
||||
else:
|
||||
self._record_idle("scan_failed")
|
||||
elif prepared_count:
|
||||
self._diagnostic_idle_reason = None
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.scan_completed",
|
||||
outcome=(
|
||||
@@ -2207,8 +2092,6 @@ class MemoryProcessingSupervisor:
|
||||
Work that falls outside the extraction window is marked deferred so
|
||||
the pending count stops pretending it is about to run.
|
||||
"""
|
||||
if self._sedimentation_stopped():
|
||||
return []
|
||||
now = datetime.now(UTC)
|
||||
window = self._extraction_window() if within_window else None
|
||||
exempt_all_apps_from_window = True
|
||||
@@ -2291,7 +2174,7 @@ class MemoryProcessingSupervisor:
|
||||
if work:
|
||||
self._diagnostic_idle_reason = None
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.scheduler",
|
||||
event="memory_scheduler.tasks_queued",
|
||||
outcome="succeeded",
|
||||
@@ -2356,7 +2239,7 @@ class MemoryProcessingSupervisor:
|
||||
return self.HARNESS_SERIAL
|
||||
|
||||
def _ensure_workers(self) -> None:
|
||||
if self._shutting_down or self._sedimentation_stopped():
|
||||
if self._shutting_down:
|
||||
return
|
||||
load = self._harness_load()
|
||||
background_capacity = (
|
||||
@@ -2434,8 +2317,6 @@ class MemoryProcessingSupervisor:
|
||||
"""
|
||||
looked_for_older = False
|
||||
while not self._shutting_down:
|
||||
if self._sedimentation_stopped():
|
||||
return
|
||||
# The harness cap is recomputed from what is running, so a worker
|
||||
# that just finished frees its harness's slot immediately.
|
||||
work = self._queue.take(
|
||||
@@ -2489,16 +2370,7 @@ class MemoryProcessingSupervisor:
|
||||
scope_key=scope_key,
|
||||
),
|
||||
):
|
||||
|
||||
async def require_sedimentation() -> None:
|
||||
if self._sedimentation_stopped():
|
||||
raise HyMemoryBridgeError("model_sedimentation_disabled")
|
||||
|
||||
finished = await runner.run(
|
||||
work.task_id,
|
||||
timeout=self._profile_timeout,
|
||||
before_hy_invocation=require_sedimentation,
|
||||
)
|
||||
finished = await runner.run(work.task_id, timeout=self._profile_timeout)
|
||||
if finished.status is MemoryTaskStatus.COMPLETED:
|
||||
self._request_read_bridge_refresh(finished.profile_snapshot)
|
||||
elif (
|
||||
@@ -3177,7 +3049,7 @@ class MemoryProcessingSupervisor:
|
||||
self._read_bridges_ready.add(key)
|
||||
self._read_bridge_generations[key] = generation
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="memory.semantic",
|
||||
event="semantic_read_bridge.refreshed",
|
||||
outcome="succeeded",
|
||||
@@ -3300,7 +3172,9 @@ class MemoryProcessingSupervisor:
|
||||
if not tail:
|
||||
return False
|
||||
last = tail[-1]
|
||||
return bool(last.role == "user" and _opens_turn(last))
|
||||
return bool(
|
||||
last.role == "user" and last.written_by is WrittenBy.PERSON and _opens_turn(last)
|
||||
)
|
||||
|
||||
def _admitted_chunks(
|
||||
self,
|
||||
@@ -3645,6 +3519,7 @@ def _is_memory_note(record: EvidenceRecord) -> bool:
|
||||
meta = record.origin_meta.get(EvidenceOrigin.MANUAL_IMPORT)
|
||||
return (
|
||||
record.source_id == "one-memory-manual"
|
||||
and record.written_by is WrittenBy.PERSON
|
||||
and meta is not None
|
||||
and meta.raw_meta.get("input_mode") == "direct_memory_note"
|
||||
)
|
||||
@@ -3720,7 +3595,7 @@ _HOST_ENVELOPE = re.compile(
|
||||
#: before HY chose a path cut long answers mid-way and sent tool-heavy turns as
|
||||
#: dozens of calls. ONE_MEMORY_SPLIT_TURNS=1 restores the 12,000 / 24 split.
|
||||
_WHOLE_TURN_CHARACTERS = 60_000
|
||||
_HY_TOOL_CHARACTERS = coding_tool_result_max_bytes()
|
||||
_HY_TOOL_CHARACTERS = 2_048
|
||||
#: Hard ceiling on the UTF-8 bytes one window may hold, whatever HY counts.
|
||||
#: HY reads the payload in-process, so a turn made of enormous raw tool
|
||||
#: records is held in memory even though HY will cut each one down. Cut by
|
||||
@@ -3939,12 +3814,14 @@ def _merge_historical_chunks(
|
||||
|
||||
|
||||
def _chunk_input_ref(chunk: tuple[EvidenceRecord, ...]) -> str:
|
||||
user_records = tuple(item for item in chunk if item.role == "user")
|
||||
return user_records[-1].evidence_id if user_records else chunk[-1].evidence_id
|
||||
person_records = tuple(
|
||||
item for item in chunk if item.role == "user" and item.written_by is WrittenBy.PERSON
|
||||
)
|
||||
return person_records[-1].evidence_id if person_records else chunk[-1].evidence_id
|
||||
|
||||
|
||||
def _opens_turn(record: EvidenceRecord) -> bool:
|
||||
return record.role == "user"
|
||||
return record.role == "user" and record.written_by is WrittenBy.PERSON
|
||||
|
||||
|
||||
def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
|
||||
@@ -3961,8 +3838,10 @@ def _trailing_user_turn_is_open(turn: list[EvidenceRecord]) -> bool:
|
||||
def _is_hy_turn_record(record: EvidenceRecord) -> bool:
|
||||
"""What HY receives of a turn: the input turns and the tool calls made in it."""
|
||||
if record.role == "tool":
|
||||
return record.content_kind is EvidenceContentKind.RAW_CONVERSATION and bool(
|
||||
record.verbatim.strip()
|
||||
return (
|
||||
record.written_by is WrittenBy.TOOL
|
||||
and record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and bool(record.verbatim.strip())
|
||||
)
|
||||
return _is_hy_input_turn(record)
|
||||
|
||||
@@ -3979,8 +3858,11 @@ def _is_hy_input_turn(record: EvidenceRecord) -> bool:
|
||||
# from a model.
|
||||
if record.role == "assistant":
|
||||
return True
|
||||
# A sub-agent question is still a protocol `user` message. Host-specific
|
||||
# author guesses do not decide whether it reaches HY.
|
||||
# D98: the author no longer decides. A sub agent's question arrives as a
|
||||
# `user` message marked written by the agent — it is ordinary
|
||||
# user-assistant data, and dropping it sent Claude Code transcripts to HY
|
||||
# with every question but the first missing. `written_by` labels the
|
||||
# display and decides owner; it cannot tell who actually wrote a message.
|
||||
if record.role != "user":
|
||||
return False
|
||||
if source.casefold().startswith(_HARNESS_PREFIXES):
|
||||
|
||||
@@ -30,12 +30,6 @@ class ConfigDocument(BaseModel):
|
||||
|
||||
schema_version: int = Field(default=1, ge=1, le=1)
|
||||
sources: dict[str, SourceConfig] = Field(default_factory=dict)
|
||||
#: Whether the configured model may turn imported conversations into
|
||||
#: memories. Defaulting to on keeps every existing install doing what it
|
||||
#: did before the switch existed; stopping it is the user's cost control
|
||||
#: and leaves the model configuration itself untouched, so starting again
|
||||
#: needs no re-entry of a key.
|
||||
sedimentation_enabled: StrictBool = True
|
||||
|
||||
@field_validator("sources")
|
||||
@classmethod
|
||||
@@ -69,19 +63,6 @@ class ConfigStore:
|
||||
source = self._document.sources.get(source_id)
|
||||
return bool(source and source.sync_enabled)
|
||||
|
||||
def sedimentation_enabled(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self._document.sedimentation_enabled)
|
||||
|
||||
def set_sedimentation_enabled(self, enabled: bool) -> ConfigDocument:
|
||||
if type(enabled) is not bool:
|
||||
raise ValueError("invalid_sedimentation_config")
|
||||
with self._lock:
|
||||
updated = self._document.model_copy(update={"sedimentation_enabled": enabled})
|
||||
self._write(updated)
|
||||
self._document = updated
|
||||
return updated
|
||||
|
||||
def set_sync(self, source_id: str, enabled: bool) -> ConfigDocument:
|
||||
return self.set_sync_many({source_id: enabled})
|
||||
|
||||
|
||||
@@ -23,13 +23,10 @@ import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
|
||||
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
#: Bumped when a derivation rule changes, so old and new ids never compare equal.
|
||||
SCHEMA_VERSION: Final[str] = "ms1"
|
||||
@@ -169,6 +166,7 @@ class CanonicalMessage(BaseModel):
|
||||
message_id: str | None = Field(default=None, min_length=1, max_length=512)
|
||||
ordinal: int = Field(ge=0)
|
||||
role: str = Field(min_length=1, max_length=128)
|
||||
written_by: str = Field(min_length=1, max_length=64)
|
||||
content: str = Field(min_length=1)
|
||||
workspace: str | None = Field(default=None, max_length=1024)
|
||||
|
||||
@@ -215,13 +213,8 @@ class WriteRequest(BaseModel):
|
||||
source_id: str | None = Field(default=None, min_length=1, max_length=128)
|
||||
batch_identity: BatchIdentity
|
||||
evidence_ids: list[str] = Field(min_length=1)
|
||||
# New jobs freeze the complete input in their own payload. The default is
|
||||
# solely for reading pre-upgrade rows whose payload only carried ids.
|
||||
input_records: tuple[EvidenceRecord, ...] = ()
|
||||
idempotency_key: str = Field(min_length=1, max_length=256)
|
||||
priority: Literal["live", "normal"] = "normal"
|
||||
event_at: datetime | None = None
|
||||
time_quality: Literal["exact", "anchored", "session", "approximate"] = "approximate"
|
||||
|
||||
@field_validator("evidence_ids")
|
||||
@classmethod
|
||||
@@ -234,16 +227,6 @@ class WriteRequest(BaseModel):
|
||||
raise ValueError("evidence_ids must be unique")
|
||||
return value
|
||||
|
||||
@field_validator("input_records")
|
||||
@classmethod
|
||||
def _input_records_match_ids(
|
||||
cls, value: tuple[EvidenceRecord, ...], info: ValidationInfo
|
||||
) -> tuple[EvidenceRecord, ...]:
|
||||
evidence_ids = info.data.get("evidence_ids")
|
||||
if value and evidence_ids != [record.evidence_id for record in value]:
|
||||
raise ValueError("input_records must match evidence_ids in order")
|
||||
return value
|
||||
|
||||
@property
|
||||
def request_key(self) -> str:
|
||||
"""Stable dedup key. Same batch + same intent must not enqueue twice."""
|
||||
|
||||
@@ -30,7 +30,7 @@ from uuid import uuid4
|
||||
from one_memory.background_work import in_the_background
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.diagnostic_log import diagnostic_event
|
||||
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.contracts import WriteRequest
|
||||
from one_memory.memory_server.data_safety import MIN_FREE_BYTES # noqa: F401 - re-exported
|
||||
@@ -249,7 +249,6 @@ MODEL_NOT_READY_CODES = frozenset(
|
||||
#: nothing was sent and retrying the same intent changes nothing — a terminal
|
||||
#: `failed`, not `unknown` (which would wrongly imply a possibly-applied
|
||||
#: write) and not a retry (which would fail identically).
|
||||
#:
|
||||
PRE_SEND_REFUSAL_CODES = frozenset(
|
||||
{
|
||||
"hy_memory_input_exceeds_window",
|
||||
@@ -283,6 +282,11 @@ _VALID_JOB_STATES = frozenset(
|
||||
#: A row in one of these states can never be moved back to a live one.
|
||||
_TERMINAL_WRITE_STATES = frozenset({COMPLETED, FAILED, DEAD, CANCELLED})
|
||||
|
||||
#: A job in one of these states can never run again, so its payload is dead
|
||||
#: weight. `unknown` is deliberately absent: it is non-terminal and still needs
|
||||
#: its intent for reconciliation. `running` is obviously still live.
|
||||
TERMINAL_PAYLOAD_STATES = (COMPLETED, FAILED, DEAD, CANCELLED)
|
||||
|
||||
#: Columns a current schema declares but an older `memory_write_jobs` may lack.
|
||||
#: `CREATE TABLE IF NOT EXISTS` is a no-op for an existing table, so without
|
||||
#: this an upgraded install would keep an older queue that every claim/lease
|
||||
@@ -300,12 +304,9 @@ LEGACY_MEMORY_WRITE_JOB_COLUMNS = (
|
||||
("lease_expires_at", "TEXT"),
|
||||
("hy_invoked", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("last_error", "TEXT"),
|
||||
("event_at", "TEXT"),
|
||||
("time_quality", "TEXT NOT NULL DEFAULT 'approximate'"),
|
||||
)
|
||||
|
||||
_FIFO_BLOCKING_STATES = (PENDING, RUNNING, WAITING_FOR_PERMISSION, WAITING_FOR_MODEL)
|
||||
EVENT_TIME_WINDOW = timedelta(minutes=10)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -392,42 +393,6 @@ def _matches_payload_except_priority(stored: str, payload: str) -> bool:
|
||||
return old == new
|
||||
|
||||
|
||||
def _matches_legacy_reference_replay(stored: str, payload: str) -> bool:
|
||||
"""Match an old id-only caller to the same job's frozen payload."""
|
||||
try:
|
||||
old = json.loads(stored)
|
||||
new = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if not isinstance(old, dict) or not isinstance(new, dict):
|
||||
return False
|
||||
old_inputs = old.get("input_records")
|
||||
new_inputs = new.get("input_records")
|
||||
if bool(old_inputs) == bool(new_inputs):
|
||||
return False
|
||||
old.pop("input_records", None)
|
||||
new.pop("input_records", None)
|
||||
return old == new
|
||||
|
||||
|
||||
def _matches_reaped_legacy_payload(stored: str, payload: str) -> bool:
|
||||
"""Compare a current frozen request with a pre-snapshot reaped payload."""
|
||||
try:
|
||||
old = json.loads(stored)
|
||||
current = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if (
|
||||
not isinstance(old, dict)
|
||||
or not isinstance(current, dict)
|
||||
or not isinstance(old.get(REAPED_PAYLOAD_MARKER), str)
|
||||
):
|
||||
return False
|
||||
current.pop("input_records", None)
|
||||
legacy_payload = json.dumps(current, sort_keys=True)
|
||||
return bool(old[REAPED_PAYLOAD_MARKER] == _payload_digest(legacy_payload))
|
||||
|
||||
|
||||
class DurableQueueSchemaUnavailableError(RuntimeError):
|
||||
"""The durable queue schema has not been created yet.
|
||||
|
||||
@@ -480,7 +445,6 @@ class DurableWriteQueue:
|
||||
if not initialize:
|
||||
return
|
||||
with main_database(database_path).write() as connection, connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
# First create the base table if absent. Then widen an older table
|
||||
# before replaying indexes that reference the new columns; a bare
|
||||
# CREATE INDEX would fail on a pre-priority queue.
|
||||
@@ -491,7 +455,6 @@ class DurableWriteQueue:
|
||||
for statement in memory_write_jobs_sqlite_schema()[1:]:
|
||||
connection.execute(statement)
|
||||
self._backfill_legacy_references(connection)
|
||||
self._backfill_legacy_event_times(connection)
|
||||
# A process that died while holding a lease cannot run its own
|
||||
# recovery. Opening the queue is the first safe point to recover any
|
||||
# rows it left behind, before any new worker claims them.
|
||||
@@ -516,7 +479,8 @@ class DurableWriteQueue:
|
||||
"UPDATE memory_write_jobs SET source_key=session_key WHERE source_key=''"
|
||||
)
|
||||
legacy_rows = connection.execute(
|
||||
"SELECT job_id FROM memory_write_jobs WHERE queue_seq=0 ORDER BY created_at, rowid"
|
||||
"SELECT job_id FROM memory_write_jobs WHERE queue_seq=0 "
|
||||
"ORDER BY created_at, rowid"
|
||||
).fetchall()
|
||||
next_seq = int(
|
||||
connection.execute(
|
||||
@@ -550,49 +514,6 @@ class DurableWriteQueue:
|
||||
if columns and columns != ["state", "priority", "created_at"]:
|
||||
connection.execute("DROP INDEX idx_write_jobs_claim")
|
||||
|
||||
@staticmethod
|
||||
def _backfill_legacy_event_times(connection: sqlite3.Connection) -> None:
|
||||
if (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='evidence'"
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET event_at=created_at WHERE event_at IS NULL"
|
||||
)
|
||||
return
|
||||
evidence_columns = {
|
||||
str(row[1]) for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
|
||||
}
|
||||
if not {"created_at", "imported_at", "role"} <= evidence_columns:
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET event_at=created_at WHERE event_at IS NULL"
|
||||
)
|
||||
return
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE memory_write_jobs
|
||||
SET event_at = COALESCE(
|
||||
(
|
||||
SELECT MAX(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
AND e.role = 'user'
|
||||
),
|
||||
(
|
||||
SELECT MIN(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
),
|
||||
created_at
|
||||
)
|
||||
WHERE event_at IS NULL
|
||||
"""
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _backfill_legacy_references(connection: sqlite3.Connection) -> None:
|
||||
"""Populate the reference table after its schema exists."""
|
||||
@@ -689,20 +610,8 @@ class DurableWriteQueue:
|
||||
replacing the earlier intent.
|
||||
"""
|
||||
self._refuse_if_below_reserve()
|
||||
request = self._freeze_request_inputs(request)
|
||||
payload = json.dumps(
|
||||
request.model_dump(
|
||||
mode="json",
|
||||
exclude={"event_at", "time_quality"},
|
||||
),
|
||||
sort_keys=True,
|
||||
)
|
||||
payload = json.dumps(request.model_dump(mode="json"), sort_keys=True)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
resolved_event, resolved_quality = self._event_time_for_request(request)
|
||||
event = request.event_at or resolved_event or datetime.fromisoformat(now)
|
||||
if event.tzinfo is None:
|
||||
event = event.replace(tzinfo=UTC)
|
||||
event_at = event.astimezone(UTC).isoformat()
|
||||
key = request.request_key
|
||||
with (
|
||||
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
|
||||
@@ -715,22 +624,16 @@ class DurableWriteQueue:
|
||||
(key,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
# A replay of work that already finished is a hit, not a
|
||||
# conflict. Terminal rows store a reaped digest instead of the
|
||||
# original payload, and the caller's key is derived from the
|
||||
# evidence's revision, which can move on between replays while
|
||||
# the payload's evidence ids stay the same. Refusing that pair
|
||||
# turned every startup replay of one settled job into an
|
||||
# exception: 236 of them in a single afternoon, all one key.
|
||||
already_settled = (
|
||||
str(existing["state"]) == COMPLETED and int(existing["hy_invoked"]) == 1
|
||||
)
|
||||
stored_payload = str(existing["payload_json"])
|
||||
priority_replay = _matches_payload_except_priority(stored_payload, payload)
|
||||
legacy_reaped_match = bool(
|
||||
already_settled
|
||||
and _matches_reaped_legacy_payload(stored_payload, payload)
|
||||
)
|
||||
if (
|
||||
not _matches_stored_payload(stored_payload, payload)
|
||||
and not priority_replay
|
||||
and not _matches_legacy_reference_replay(stored_payload, payload)
|
||||
and not legacy_reaped_match
|
||||
):
|
||||
raise ValueError(f"idempotency key {key} already exists with different content")
|
||||
if already_settled:
|
||||
return DurableWriteRecord(
|
||||
job_id=str(existing["job_id"]),
|
||||
@@ -741,6 +644,14 @@ class DurableWriteQueue:
|
||||
updated_at=str(existing["updated_at"]),
|
||||
detail="already completed; replay treated as a hit",
|
||||
)
|
||||
priority_replay = _matches_payload_except_priority(
|
||||
str(existing["payload_json"]), payload
|
||||
)
|
||||
if (
|
||||
not _matches_stored_payload(str(existing["payload_json"]), payload)
|
||||
and not priority_replay
|
||||
):
|
||||
raise ValueError(f"idempotency key {key} already exists with different content")
|
||||
if priority_replay and request.priority == "live":
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET priority='live', updated_at=? "
|
||||
@@ -800,13 +711,11 @@ class DurableWriteQueue:
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_write_jobs ("
|
||||
" job_id, queue_seq, session_key, source_key, batch_id,"
|
||||
" idempotency_key, payload_json,"
|
||||
" job_id, queue_seq, session_key, source_key, batch_id, idempotency_key, payload_json,"
|
||||
" evidence_ids_json,"
|
||||
" state, priority, attempts, max_attempts, lease_owner,"
|
||||
" lease_expires_at, hy_invoked, last_error, event_at, time_quality,"
|
||||
" created_at, updated_at"
|
||||
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
" lease_expires_at, hy_invoked, last_error, created_at, updated_at"
|
||||
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
job_id,
|
||||
queue_seq,
|
||||
@@ -824,8 +733,6 @@ class DurableWriteQueue:
|
||||
None,
|
||||
0,
|
||||
None,
|
||||
event_at,
|
||||
request.time_quality if request.event_at is not None else resolved_quality,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
@@ -834,18 +741,6 @@ class DurableWriteQueue:
|
||||
"INSERT INTO memory_write_job_evidence (job_id, evidence_id) VALUES (?,?)",
|
||||
((job_id, str(evidence_id)) for evidence_id in request.evidence_ids),
|
||||
)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.write_queue",
|
||||
event="memory_write_job.queued",
|
||||
outcome="queued",
|
||||
job_id=job_id,
|
||||
details={
|
||||
"source_id": request.source_id or request.internal_session_id,
|
||||
"mode": request.priority,
|
||||
"count": len(request.evidence_ids),
|
||||
},
|
||||
)
|
||||
return DurableWriteRecord(
|
||||
job_id=job_id,
|
||||
request_id=key,
|
||||
@@ -855,45 +750,6 @@ class DurableWriteQueue:
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
def _freeze_request_inputs(self, request: WriteRequest) -> WriteRequest:
|
||||
if request.input_records or self.evidence is None:
|
||||
return request
|
||||
records: list[EvidenceRecord] = []
|
||||
for evidence_id in request.evidence_ids:
|
||||
record = self.evidence.get(evidence_id)
|
||||
if record is None:
|
||||
return request
|
||||
records.append(record)
|
||||
return request.model_copy(update={"input_records": tuple(records)})
|
||||
|
||||
def _event_time_for_request(self, request: WriteRequest) -> tuple[datetime | None, str]:
|
||||
records = list(request.input_records)
|
||||
if not records and self.evidence is not None:
|
||||
records = [
|
||||
record
|
||||
for evidence_id in request.evidence_ids
|
||||
if (record := self.evidence.get(evidence_id)) is not None
|
||||
]
|
||||
if not records:
|
||||
return None, "approximate"
|
||||
opener = next(
|
||||
(record for record in records if record.role == "user"),
|
||||
records[0],
|
||||
)
|
||||
if opener.created_at is None:
|
||||
return opener.imported_at, "approximate"
|
||||
origin = opener.origin_meta.get(EvidenceOrigin.FILE)
|
||||
basis = origin.raw_meta.get("time_basis") if origin is not None else None
|
||||
turn_event_at = origin.raw_meta.get("turn_event_at") if origin is not None else None
|
||||
if isinstance(turn_event_at, str):
|
||||
try:
|
||||
return datetime.fromisoformat(turn_event_at), "exact"
|
||||
except ValueError:
|
||||
pass
|
||||
if basis == "file_mtime":
|
||||
return opener.created_at, "session"
|
||||
return opener.created_at, "exact"
|
||||
|
||||
def repair_pre_send_identity_failures(self, *, limit: int = 128) -> int:
|
||||
"""Refresh stale Evidence journals and requeue only provably unsent jobs."""
|
||||
if self.evidence is None:
|
||||
@@ -1003,55 +859,11 @@ class DurableWriteQueue:
|
||||
raise ReconcileSchemaUnavailableError(
|
||||
"evidence table is unavailable; run migrations before reconcile"
|
||||
)
|
||||
has_versions = (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM sqlite_master "
|
||||
"WHERE type='table' AND name='evidence_versions'"
|
||||
).fetchone()
|
||||
is not None
|
||||
where = (
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence ref "
|
||||
"WHERE ref.evidence_id=evidence.evidence_id)"
|
||||
)
|
||||
evidence_columns = {
|
||||
str(row[1])
|
||||
for row in connection.execute("PRAGMA table_info(evidence)").fetchall()
|
||||
}
|
||||
if {"revision", "content_hash", "updated_at"} <= evidence_columns:
|
||||
revision_time = (
|
||||
"COALESCE((SELECT version.changed_at FROM evidence_versions version "
|
||||
"WHERE version.evidence_id=evidence.evidence_id "
|
||||
"AND version.revision=evidence.revision),"
|
||||
"evidence.updated_at,evidence.imported_at)"
|
||||
if has_versions
|
||||
else "COALESCE(evidence.updated_at,evidence.imported_at)"
|
||||
)
|
||||
where = (
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_job_evidence linked "
|
||||
"JOIN memory_write_jobs job ON job.job_id=linked.job_id, "
|
||||
"json_each(job.payload_json,'$.input_records') frozen "
|
||||
"WHERE linked.evidence_id=evidence.evidence_id "
|
||||
"AND json_extract(frozen.value,'$.evidence_id')=evidence.evidence_id "
|
||||
"AND json_extract(frozen.value,'$.revision')=evidence.revision "
|
||||
"AND json_extract(frozen.value,'$.content_hash')=evidence.content_hash"
|
||||
") "
|
||||
# Pre-snapshot builds only recorded an Evidence id. Treat
|
||||
# that as coverage only when its Job was created after this
|
||||
# exact revision.
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_job_evidence legacy "
|
||||
"JOIN memory_write_jobs legacy_job ON legacy_job.job_id=legacy.job_id "
|
||||
"WHERE legacy.evidence_id=evidence.evidence_id "
|
||||
"AND COALESCE(json_array_length("
|
||||
"json_extract(legacy_job.payload_json,'$.input_records')),0)=0 "
|
||||
f"AND legacy_job.created_at>=({revision_time}))"
|
||||
)
|
||||
else:
|
||||
# A genuinely old/minimal schema has no revision identity.
|
||||
where = (
|
||||
"WHERE evidence.imported_at IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM memory_write_job_evidence legacy "
|
||||
"WHERE legacy.evidence_id=evidence.evidence_id)"
|
||||
)
|
||||
params: tuple[Any, ...] = ()
|
||||
cursor_clause = ""
|
||||
if after_cursor is not None:
|
||||
@@ -1283,7 +1095,6 @@ class DurableWriteQueue:
|
||||
with main_database(self.database_path).write() as connection, connection:
|
||||
updated = connection.execute(
|
||||
"UPDATE memory_write_jobs SET state=?,lease_owner=NULL,"
|
||||
"attempts=CASE WHEN attempts>0 THEN attempts-1 ELSE 0 END,"
|
||||
"lease_expires_at=NULL,updated_at=? WHERE " + where,
|
||||
(PENDING, now, *params),
|
||||
)
|
||||
@@ -1351,10 +1162,45 @@ class DurableWriteQueue:
|
||||
return state
|
||||
|
||||
def clear_terminal_payloads(self, *, limit: int = 512) -> int:
|
||||
"""Deprecated compatibility hook; durable inputs are immutable audit data."""
|
||||
"""Replace terminal rows' request bodies with a content digest.
|
||||
|
||||
A completed/failed/dead/cancelled row can never run again, so the full
|
||||
serialized `WriteRequest` is unbounded dead weight. A digest keeps the
|
||||
idempotency contract intact: a replay of the same intent still resolves
|
||||
to the same row, while the same key with different content still
|
||||
conflicts. `unknown` is left alone because it is non-terminal and its
|
||||
intent is needed for reconciliation.
|
||||
|
||||
Bounded by `limit` so a large backlog drains over several calls instead
|
||||
of one long write on the single writer.
|
||||
"""
|
||||
if type(limit) is not int or limit < 1:
|
||||
raise ValueError("limit must be positive")
|
||||
return 0
|
||||
placeholders = ",".join("?" for _ in TERMINAL_PAYLOAD_STATES)
|
||||
with (
|
||||
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
|
||||
connection,
|
||||
):
|
||||
rows = connection.execute(
|
||||
"SELECT job_id, payload_json FROM memory_write_jobs "
|
||||
f"WHERE state IN ({placeholders}) "
|
||||
"AND instr(payload_json, ?) = 0 "
|
||||
"LIMIT ?",
|
||||
(*TERMINAL_PAYLOAD_STATES, REAPED_PAYLOAD_MARKER, limit),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
for row in rows:
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET payload_json=? "
|
||||
f"WHERE job_id=? AND state IN ({placeholders})",
|
||||
(
|
||||
_reaped_payload(_payload_digest(str(row["payload_json"]))),
|
||||
str(row["job_id"]),
|
||||
*TERMINAL_PAYLOAD_STATES,
|
||||
),
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def readmit_parked(self, *, states: tuple[str, ...] = PARKED_STATES) -> int:
|
||||
"""Move parked jobs back to `pending` for a fresh claim.
|
||||
@@ -1556,13 +1402,7 @@ class DurableWriteQueue:
|
||||
cross back to pending, so this cannot duplicate a possibly-sent write.
|
||||
"""
|
||||
now = datetime.now(UTC).isoformat()
|
||||
payload = json.dumps(
|
||||
request.model_dump(
|
||||
mode="json",
|
||||
exclude={"event_at", "time_quality"},
|
||||
),
|
||||
sort_keys=True,
|
||||
)
|
||||
payload = json.dumps(request.model_dump(mode="json"), sort_keys=True)
|
||||
with (
|
||||
main_database(self.database_path).write(row_factory=sqlite3.Row) as connection,
|
||||
connection,
|
||||
@@ -1901,12 +1741,9 @@ class DurableWriteRegistry:
|
||||
def evidence_for_request(store: EvidenceStore, request: WriteRequest) -> tuple[EvidenceRecord, ...]:
|
||||
"""The Evidence a stored write points at, in the request's own order.
|
||||
|
||||
Current jobs are self-contained: their immutable records are the model
|
||||
input even if a Harness later rewinds or edits its live transcript. The
|
||||
store lookup remains only for payloads written by pre-upgrade builds.
|
||||
A write may only point at Evidence that still exists; a missing id is a
|
||||
rejection rather than a silent partial send.
|
||||
"""
|
||||
if request.input_records:
|
||||
return request.input_records
|
||||
records = []
|
||||
for evidence_id in request.evidence_ids:
|
||||
record = store.get(evidence_id)
|
||||
@@ -2013,14 +1850,7 @@ async def execute_job(
|
||||
# Never persist the raw message: it may be a provider response or the
|
||||
# user's own text, and `last_error` is shown over HTTP as `detail`.
|
||||
code = safe_error_code(error)
|
||||
if (
|
||||
code
|
||||
in {
|
||||
"model_content_source_disabled",
|
||||
"model_sedimentation_disabled",
|
||||
}
|
||||
and not hy_invoked
|
||||
):
|
||||
if code == "model_content_source_disabled" and not hy_invoked:
|
||||
await in_the_background(queue.release_unstarted, job_id, owner=owner)
|
||||
return
|
||||
# The task layer classifies these before any bridge is opened, so HY
|
||||
@@ -2031,11 +1861,7 @@ async def execute_job(
|
||||
# - transient model/provider trouble -> waiting_for_model (parked).
|
||||
# Everything else may have crossed into HY, and HY has no idempotency
|
||||
# key, so it is `unknown` rather than a blind retry.
|
||||
if code in PRE_SEND_REFUSAL_CODES:
|
||||
# The frozen input window is checked before the payload reaches
|
||||
# HY. This one refusal therefore has a proven pre-send boundary.
|
||||
state, invoked_marker = FAILED, False
|
||||
elif hy_invoked:
|
||||
if hy_invoked:
|
||||
# Error names do not prove execution phase: permission and
|
||||
# evidence rechecks also run after HY returns. Once the durable
|
||||
# marker is set, every failure is possibly-sent and must stay out
|
||||
@@ -2047,6 +1873,10 @@ async def execute_job(
|
||||
state, invoked_marker = WAITING_FOR_PERMISSION, False
|
||||
elif code in MODEL_NOT_READY_CODES:
|
||||
state, invoked_marker = WAITING_FOR_MODEL, False
|
||||
elif code in PRE_SEND_REFUSAL_CODES:
|
||||
# Provably before the bridge: the frozen input is stale or too
|
||||
# large. Nothing was sent, so this is terminal rather than unknown.
|
||||
state, invoked_marker = FAILED, False
|
||||
elif code == "model_content_transport_request_failed":
|
||||
# The generation or embedding HTTP exchange can fail after HY has
|
||||
# already persisted a node. Treating that as a pre-send failure
|
||||
@@ -2135,6 +1965,7 @@ class DurableWriteDrain:
|
||||
#: sustained backlog must not keep finished rows' request bodies in the
|
||||
#: business database indefinitely. Its own gate so neither interval can
|
||||
#: starve the other.
|
||||
self._last_terminal_reap: datetime | None = None
|
||||
#: `waiting_for_model` re-admission is also time-based recoverability:
|
||||
#: a model outage clears on its own, and a queue that never goes idle
|
||||
#: must still re-admit the write once the wait has elapsed.
|
||||
@@ -2185,7 +2016,7 @@ class DurableWriteDrain:
|
||||
now: datetime | None = None,
|
||||
budget_ms: int | None = None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Claim event-time heads, with one running job per enabled Harness."""
|
||||
"""Claim at most one FIFO head per enabled Harness."""
|
||||
if type(limit) is not int or limit < 1:
|
||||
raise ValueError("limit must be positive")
|
||||
now = now or datetime.now(UTC)
|
||||
@@ -2227,7 +2058,9 @@ class DurableWriteDrain:
|
||||
(PENDING,),
|
||||
).fetchall()
|
||||
enabled_sources = {
|
||||
str(row[0]) for row in source_rows if self.source_writes_permitted(str(row[0]))
|
||||
str(row[0])
|
||||
for row in source_rows
|
||||
if self.source_writes_permitted(str(row[0]))
|
||||
}
|
||||
if not enabled_sources:
|
||||
return ()
|
||||
@@ -2246,142 +2079,38 @@ class DurableWriteDrain:
|
||||
placeholders = ",".join("?" for _ in enabled_sources)
|
||||
source_clause = f"AND candidate.source_key IN ({placeholders}) "
|
||||
source_params = tuple(sorted(enabled_sources))
|
||||
window = connection.execute(
|
||||
"SELECT window_anchor,window_end FROM memory_write_schedule_state WHERE singleton=1"
|
||||
).fetchone()
|
||||
if window is not None:
|
||||
anchor_text, end_text = str(window[0]), str(window[1])
|
||||
active_source_clause = ""
|
||||
active_source_params: tuple[Any, ...] = ()
|
||||
if enabled_sources is not None:
|
||||
placeholders = ",".join("?" for _ in enabled_sources)
|
||||
active_source_clause = f"AND active.source_key IN ({placeholders}) "
|
||||
active_source_params = tuple(sorted(enabled_sources))
|
||||
# The contract retires a window when every *eligible*
|
||||
# pending/running job in it has settled or parked. A pending row
|
||||
# that scheduling already holds back (an unexpired lease, a
|
||||
# busy Harness, or an earlier event in its Session) is not eligible, so it
|
||||
# must not keep the anchor alive: those blockers usually sit
|
||||
# outside this ten-minute range and can never be claimed from
|
||||
# inside it, which pinned the anchor and starved the whole
|
||||
# queue. Reuse the claim's own predicate so "the window is
|
||||
# busy" and "the window can still be worked" agree.
|
||||
active = connection.execute(
|
||||
"SELECT 1 FROM memory_write_jobs AS active "
|
||||
"WHERE active.event_at>=? AND active.event_at<? AND ("
|
||||
"active.state=? OR (active.state=? "
|
||||
"AND (active.lease_expires_at IS NULL "
|
||||
"OR active.lease_expires_at < ?) " + active_source_clause + "AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS held "
|
||||
"WHERE held.source_key=active.source_key AND held.state=? "
|
||||
"AND held.lease_owner IS NOT NULL "
|
||||
"AND held.lease_expires_at>=?) "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS earlier "
|
||||
"WHERE earlier.session_key=active.session_key "
|
||||
f"AND earlier.state IN ({blocking}) "
|
||||
"AND (earlier.event_at<active.event_at OR "
|
||||
"(earlier.event_at=active.event_at "
|
||||
"AND earlier.queue_seq<active.queue_seq))) "
|
||||
")) LIMIT 1",
|
||||
(
|
||||
anchor_text,
|
||||
end_text,
|
||||
RUNNING,
|
||||
PENDING,
|
||||
now.isoformat(),
|
||||
*active_source_params,
|
||||
RUNNING,
|
||||
now.isoformat(),
|
||||
*_FIFO_BLOCKING_STATES,
|
||||
),
|
||||
).fetchone()
|
||||
if active is None:
|
||||
connection.execute("DELETE FROM memory_write_schedule_state WHERE singleton=1")
|
||||
window = None
|
||||
if window is None:
|
||||
anchor = connection.execute(
|
||||
"SELECT candidate.event_at "
|
||||
"FROM memory_write_jobs AS candidate "
|
||||
"WHERE candidate.state=? "
|
||||
"AND (candidate.lease_expires_at IS NULL "
|
||||
"OR candidate.lease_expires_at < ?) " + source_clause + "AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS held "
|
||||
"WHERE held.source_key=candidate.source_key AND held.state=? "
|
||||
"AND held.lease_owner IS NOT NULL "
|
||||
"AND held.lease_expires_at>=?) "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS earlier "
|
||||
"WHERE earlier.session_key=candidate.session_key "
|
||||
f"AND earlier.state IN ({blocking}) "
|
||||
"AND (earlier.event_at<candidate.event_at OR "
|
||||
"(earlier.event_at=candidate.event_at "
|
||||
"AND earlier.queue_seq<candidate.queue_seq))) "
|
||||
"ORDER BY candidate.event_at,candidate.queue_seq LIMIT 1",
|
||||
(
|
||||
PENDING,
|
||||
now.isoformat(),
|
||||
*source_params,
|
||||
RUNNING,
|
||||
now.isoformat(),
|
||||
*_FIFO_BLOCKING_STATES,
|
||||
),
|
||||
).fetchone()
|
||||
if anchor is None:
|
||||
return ()
|
||||
anchor_at = datetime.fromisoformat(str(anchor[0]))
|
||||
if anchor_at.tzinfo is None:
|
||||
anchor_at = anchor_at.replace(tzinfo=UTC)
|
||||
anchor_text = anchor_at.astimezone(UTC).isoformat()
|
||||
end_text = (anchor_at + EVENT_TIME_WINDOW).astimezone(UTC).isoformat()
|
||||
connection.execute(
|
||||
"INSERT INTO memory_write_schedule_state "
|
||||
"(singleton,window_anchor,window_end,updated_at) "
|
||||
"VALUES (1,?,?,?)",
|
||||
(anchor_text, end_text, now.isoformat()),
|
||||
)
|
||||
rows = connection.execute(
|
||||
"SELECT candidate.job_id,candidate.source_key "
|
||||
"FROM memory_write_jobs AS candidate "
|
||||
"WHERE candidate.state=? "
|
||||
"AND (candidate.lease_expires_at IS NULL OR candidate.lease_expires_at < ?) "
|
||||
+ source_clause
|
||||
+ "AND candidate.event_at>=? AND candidate.event_at<? "
|
||||
+ source_clause +
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS held "
|
||||
"WHERE held.source_key=candidate.source_key AND held.state=? "
|
||||
"AND held.lease_owner IS NOT NULL AND held.lease_expires_at>=?) "
|
||||
"AND NOT EXISTS ("
|
||||
"SELECT 1 FROM memory_write_jobs AS earlier "
|
||||
"WHERE earlier.session_key=candidate.session_key "
|
||||
"WHERE earlier.source_key=candidate.source_key "
|
||||
f"AND earlier.state IN ({blocking}) "
|
||||
"AND (earlier.event_at<candidate.event_at OR "
|
||||
"(earlier.event_at=candidate.event_at "
|
||||
"AND earlier.queue_seq<candidate.queue_seq))) "
|
||||
"ORDER BY candidate.event_at,candidate.queue_seq",
|
||||
"AND earlier.queue_seq<candidate.queue_seq) "
|
||||
"ORDER BY candidate.queue_seq",
|
||||
(
|
||||
PENDING,
|
||||
now.isoformat(),
|
||||
*source_params,
|
||||
anchor_text,
|
||||
end_text,
|
||||
RUNNING,
|
||||
now.isoformat(),
|
||||
*_FIFO_BLOCKING_STATES,
|
||||
),
|
||||
).fetchall()
|
||||
selected: list[str] = []
|
||||
selected_sources: set[str] = set()
|
||||
for row in rows:
|
||||
source_key = str(row["source_key"])
|
||||
if source_key in selected_sources:
|
||||
continue
|
||||
if self.source_claim_enabled is not None and not self.source_claim_enabled(
|
||||
source_key
|
||||
str(row["source_key"])
|
||||
):
|
||||
continue
|
||||
selected.append(str(row["job_id"]))
|
||||
selected_sources.add(source_key)
|
||||
if len(selected) == min(limit, self.max_concurrent_sources):
|
||||
break
|
||||
if not selected:
|
||||
@@ -2610,6 +2339,19 @@ class DurableWriteDrain:
|
||||
self.queue.reclaim_expired,
|
||||
operation_name="reclaim_expired",
|
||||
)
|
||||
# Terminal-payload reaping is a bounded retention write: a
|
||||
# finished row's full request body must not sit in the business
|
||||
# database forever just because the queue never went idle. Its
|
||||
# own interval keeps it off every poll.
|
||||
if (
|
||||
self._last_terminal_reap is None
|
||||
or tick_started - self._last_terminal_reap >= self._reap_interval
|
||||
):
|
||||
self._last_terminal_reap = tick_started
|
||||
await self._run_idle_maintenance(
|
||||
self.queue.clear_terminal_payloads,
|
||||
operation_name="clear_terminal_payloads",
|
||||
)
|
||||
# A model outage clears on its own, so `waiting_for_model` jobs
|
||||
# are re-admitted on a timer. That must not depend on idleness: a
|
||||
# queue busy for longer than the wait window would otherwise keep
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The result decides whether the `unknown` job state can ever be retired, so it
|
||||
is probed against the installed SDK rather than written down from memory.
|
||||
|
||||
What was verified against `hy-memory==2.0.4`:
|
||||
What was verified against `hy-memory==2.0.3`:
|
||||
|
||||
- `HyMemoryClient.add(..., request_id=...)` accepts a caller-supplied id and
|
||||
echoes it back. It is described as a request/receipt handle, and the SDK
|
||||
|
||||
@@ -151,8 +151,6 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
lease_expires_at TEXT,
|
||||
hy_invoked INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
event_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
time_quality TEXT NOT NULL DEFAULT 'approximate',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
@@ -166,8 +164,6 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
ON memory_write_jobs(source_key, state, queue_seq);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_jobs_fifo
|
||||
ON memory_write_jobs(queue_seq);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_jobs_event_claim
|
||||
ON memory_write_jobs(state, event_at, queue_seq);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_jobs_source_recent
|
||||
ON memory_write_jobs(source_key, updated_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS memory_write_job_evidence (
|
||||
@@ -176,13 +172,7 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
PRIMARY KEY (job_id, evidence_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_write_job_evidence_evidence
|
||||
ON memory_write_job_evidence(evidence_id);
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor TEXT NOT NULL,
|
||||
window_end TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
ON memory_write_job_evidence(evidence_id)
|
||||
"""
|
||||
return """
|
||||
CREATE TABLE IF NOT EXISTS memory_write_jobs (
|
||||
@@ -202,8 +192,6 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
lease_expires_at DATETIME(6) NULL,
|
||||
hy_invoked TINYINT(1) NOT NULL DEFAULT 0,
|
||||
last_error TEXT NULL,
|
||||
event_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
|
||||
time_quality VARCHAR(32) NOT NULL DEFAULT 'approximate',
|
||||
created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
|
||||
updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
|
||||
ON UPDATE CURRENT_TIMESTAMP(6),
|
||||
@@ -213,7 +201,6 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
KEY idx_write_jobs_source_claim (state, source_key, priority, created_at),
|
||||
KEY idx_write_jobs_source_fifo (source_key, state, queue_seq),
|
||||
KEY idx_write_jobs_fifo (queue_seq),
|
||||
KEY idx_write_jobs_event_claim (state, event_at, queue_seq),
|
||||
KEY idx_write_jobs_source_recent (source_key, updated_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
@@ -222,13 +209,6 @@ def memory_write_jobs_ddl(dialect: str = "mysql") -> str:
|
||||
evidence_id VARCHAR(128) NOT NULL,
|
||||
PRIMARY KEY (job_id, evidence_id),
|
||||
KEY idx_write_job_evidence_evidence (evidence_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INT PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor DATETIME(6) NOT NULL,
|
||||
window_end DATETIME(6) NOT NULL,
|
||||
updated_at DATETIME(6) NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
|
||||
@@ -1251,68 +1231,6 @@ CREATE TABLE IF NOT EXISTS model_request_budget_events (
|
||||
"""
|
||||
|
||||
|
||||
def model_request_audit_ddl(dialect: str = "mysql") -> str:
|
||||
"""Content-free accounting for physical precipitation model requests."""
|
||||
if dialect == "sqlite":
|
||||
return """
|
||||
CREATE TABLE IF NOT EXISTS model_request_audit (
|
||||
audit_id TEXT PRIMARY KEY,
|
||||
request_id TEXT NOT NULL,
|
||||
profile_id TEXT NOT NULL,
|
||||
profile_revision INTEGER NOT NULL,
|
||||
provider_kind TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
request_kind TEXT NOT NULL,
|
||||
prompt_stage TEXT NOT NULL DEFAULT 'other',
|
||||
started_at TEXT NOT NULL,
|
||||
completed_at TEXT,
|
||||
outcome TEXT NOT NULL CHECK(outcome IN ('in_flight','succeeded','failed')),
|
||||
http_status INTEGER,
|
||||
error_type TEXT,
|
||||
provider_request_id TEXT,
|
||||
prompt_tokens INTEGER,
|
||||
completion_tokens INTEGER,
|
||||
total_tokens INTEGER,
|
||||
cached_prompt_tokens INTEGER,
|
||||
reasoning_tokens INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS model_request_audit_request
|
||||
ON model_request_audit(request_id, started_at);
|
||||
CREATE INDEX IF NOT EXISTS model_request_audit_started
|
||||
ON model_request_audit(started_at);
|
||||
"""
|
||||
return """
|
||||
CREATE TABLE IF NOT EXISTS model_request_audit (
|
||||
audit_id VARCHAR(128) PRIMARY KEY,
|
||||
request_id VARCHAR(512) NOT NULL,
|
||||
profile_id VARCHAR(128) NOT NULL,
|
||||
profile_revision INT NOT NULL,
|
||||
provider_kind VARCHAR(64) NOT NULL,
|
||||
model VARCHAR(512) NOT NULL,
|
||||
request_kind VARCHAR(64) NOT NULL,
|
||||
prompt_stage VARCHAR(128) NOT NULL DEFAULT 'other',
|
||||
started_at DATETIME(6) NOT NULL,
|
||||
completed_at DATETIME(6) NULL,
|
||||
outcome VARCHAR(32) NOT NULL CHECK(outcome IN ('in_flight','succeeded','failed')),
|
||||
http_status INT NULL,
|
||||
error_type VARCHAR(128) NULL,
|
||||
provider_request_id VARCHAR(512) NULL,
|
||||
prompt_tokens BIGINT NULL,
|
||||
completion_tokens BIGINT NULL,
|
||||
total_tokens BIGINT NULL,
|
||||
cached_prompt_tokens BIGINT NULL,
|
||||
reasoning_tokens BIGINT NULL,
|
||||
KEY model_request_audit_request (request_id, started_at),
|
||||
KEY model_request_audit_started (started_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
|
||||
|
||||
async def _052_model_request_audit(cursor: Any) -> None:
|
||||
for statement in _split_statements(model_request_audit_ddl(_dialect(cursor))):
|
||||
await cursor.execute(statement)
|
||||
|
||||
|
||||
def model_probe_operations_ddl(dialect: str = "mysql") -> str:
|
||||
"""Probe operations; `state` is CHECKed to a known set.
|
||||
|
||||
@@ -1496,97 +1414,6 @@ async def _051_drop_hy_node_permissions(cursor: Any) -> None:
|
||||
await cursor.execute("DROP TABLE IF EXISTS hy_node_permissions")
|
||||
|
||||
|
||||
async def _053_memory_write_event_time(cursor: Any) -> None:
|
||||
"""Add event-time scheduling without re-keying any existing write."""
|
||||
|
||||
dialect = _dialect(cursor)
|
||||
columns = await cursor.columns("memory_write_jobs")
|
||||
added_event_at = "event_at" not in columns
|
||||
if added_event_at:
|
||||
await cursor.execute(
|
||||
"ALTER TABLE memory_write_jobs ADD COLUMN event_at "
|
||||
+ ("TEXT" if dialect == "sqlite" else "DATETIME(6) NULL")
|
||||
)
|
||||
if "time_quality" not in columns:
|
||||
await cursor.execute(
|
||||
"ALTER TABLE memory_write_jobs ADD COLUMN time_quality "
|
||||
+ (
|
||||
"TEXT NOT NULL DEFAULT 'approximate'"
|
||||
if dialect == "sqlite"
|
||||
else "VARCHAR(32) NOT NULL DEFAULT 'approximate'"
|
||||
)
|
||||
)
|
||||
await cursor.execute(
|
||||
"""
|
||||
UPDATE memory_write_jobs
|
||||
SET event_at = COALESCE(
|
||||
(
|
||||
SELECT MAX(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
AND e.role = 'user'
|
||||
),
|
||||
(
|
||||
SELECT MIN(COALESCE(e.created_at, e.imported_at))
|
||||
FROM memory_write_job_evidence AS linked
|
||||
JOIN evidence AS e ON e.evidence_id = linked.evidence_id
|
||||
WHERE linked.job_id = memory_write_jobs.job_id
|
||||
),
|
||||
created_at
|
||||
)
|
||||
WHERE event_at IS NULL
|
||||
"""
|
||||
)
|
||||
if dialect == "sqlite":
|
||||
await cursor.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_write_jobs_event_claim "
|
||||
"ON memory_write_jobs(state, event_at, queue_seq)"
|
||||
)
|
||||
await cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor TEXT NOT NULL,
|
||||
window_end TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
else:
|
||||
if added_event_at:
|
||||
await cursor.execute(
|
||||
"CREATE INDEX idx_write_jobs_event_claim "
|
||||
"ON memory_write_jobs(state, event_at, queue_seq)"
|
||||
)
|
||||
await cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS memory_write_schedule_state (
|
||||
singleton INT PRIMARY KEY CHECK (singleton = 1),
|
||||
window_anchor DATETIME(6) NOT NULL,
|
||||
window_end DATETIME(6) NOT NULL,
|
||||
updated_at DATETIME(6) NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
async def _054_legacy_possibly_sent_tasks(cursor: Any) -> None:
|
||||
"""Preserve the old sent marker before removing its retired permission table."""
|
||||
columns = await cursor.columns("memory_task_continuous_permissions")
|
||||
if "task_id" not in columns:
|
||||
return
|
||||
await cursor.execute(
|
||||
"""
|
||||
UPDATE memory_tasks
|
||||
SET hy_invocation_count=CASE
|
||||
WHEN hy_invocation_count<1 THEN 1 ELSE hy_invocation_count END
|
||||
WHERE task_id IN (SELECT task_id FROM memory_task_continuous_permissions)
|
||||
"""
|
||||
)
|
||||
await cursor.execute("DROP TABLE memory_task_continuous_permissions")
|
||||
|
||||
|
||||
def model_profiles_ddl(dialect: str = "mysql") -> str:
|
||||
"""Model profiles and their version history.
|
||||
|
||||
@@ -2853,11 +2680,7 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
2,
|
||||
"memory_write_jobs",
|
||||
_002_write_jobs,
|
||||
(
|
||||
"memory_write_jobs",
|
||||
"memory_write_job_evidence",
|
||||
"memory_write_schedule_state",
|
||||
),
|
||||
("memory_write_jobs", "memory_write_job_evidence"),
|
||||
),
|
||||
Migration(3, "source_file_cursors", _003_scanner_cursors, ("source_file_cursors",)),
|
||||
Migration(4, "change_outbox", _004_change_outbox, ("change_outbox",)),
|
||||
@@ -3026,21 +2849,4 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
_050_model_probe_operations_reserved_at,
|
||||
),
|
||||
Migration(51, "drop_hy_node_permissions", _051_drop_hy_node_permissions),
|
||||
Migration(
|
||||
52,
|
||||
"model_request_audit",
|
||||
_052_model_request_audit,
|
||||
("model_request_audit",),
|
||||
),
|
||||
Migration(
|
||||
53,
|
||||
"memory_write_event_time",
|
||||
_053_memory_write_event_time,
|
||||
("memory_write_schedule_state",),
|
||||
),
|
||||
Migration(
|
||||
54,
|
||||
"legacy_possibly_sent_tasks",
|
||||
_054_legacy_possibly_sent_tasks,
|
||||
),
|
||||
)
|
||||
@@ -107,8 +107,6 @@ def build_request(
|
||||
first_round: int,
|
||||
last_round: int,
|
||||
evidence_ids: list[str],
|
||||
input_records: tuple[EvidenceRecord, ...] = (),
|
||||
source_id: str | None = None,
|
||||
priority: Literal["live", "normal"] = "normal",
|
||||
idempotency_key: str | None = None,
|
||||
) -> WriteRequest:
|
||||
@@ -127,8 +125,6 @@ def build_request(
|
||||
internal_session_id=internal_session_id,
|
||||
batch_identity=batch,
|
||||
evidence_ids=evidence_ids,
|
||||
input_records=input_records,
|
||||
source_id=source_id,
|
||||
idempotency_key=idempotency_key or batch.batch_id,
|
||||
priority=priority,
|
||||
)
|
||||
@@ -275,8 +271,6 @@ def build_request_from_evidence(
|
||||
first_round=first_round,
|
||||
last_round=last_round,
|
||||
evidence_ids=list(ordered),
|
||||
input_records=tuple(ordered.values()),
|
||||
source_id=seed.source_id,
|
||||
priority=priority,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
@@ -18,11 +18,10 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from hashlib import sha256
|
||||
from typing import Literal
|
||||
|
||||
from one_memory.imports.models import EvidenceOrigin, EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.memory_server.contracts import (
|
||||
BatchIdentity,
|
||||
WriteRequest,
|
||||
@@ -125,12 +124,6 @@ class UnifiedWriteIntake:
|
||||
priority: WritePriority,
|
||||
) -> WriteRequest:
|
||||
session_id = internal_session_id_for(record.conversation_id, record.source_id)
|
||||
event_at, time_quality = _job_time((record,))
|
||||
request_key = (
|
||||
f"entry:{record.evidence_id}"
|
||||
if record.revision == 1
|
||||
else (f"entry:v2:{record.evidence_id}:{record.revision}:{record.content_hash}")
|
||||
)
|
||||
return WriteRequest(
|
||||
internal_session_id=session_id,
|
||||
source_id=record.source_id,
|
||||
@@ -140,11 +133,8 @@ class UnifiedWriteIntake:
|
||||
last_round=last_round,
|
||||
),
|
||||
evidence_ids=[record.evidence_id],
|
||||
input_records=(record,),
|
||||
idempotency_key=request_key,
|
||||
idempotency_key=f"entry:{record.evidence_id}",
|
||||
priority=priority,
|
||||
event_at=event_at,
|
||||
time_quality=time_quality,
|
||||
)
|
||||
|
||||
def submit_records(
|
||||
@@ -171,7 +161,6 @@ class UnifiedWriteIntake:
|
||||
)
|
||||
request_id = f"turn:v1:{sha256(material.encode()).hexdigest()}"
|
||||
session_id = internal_session_id_for(first.conversation_id, first.source_id)
|
||||
event_at, time_quality = _job_time(records)
|
||||
durable = self.queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id=session_id,
|
||||
@@ -182,11 +171,8 @@ class UnifiedWriteIntake:
|
||||
last_round=1,
|
||||
),
|
||||
evidence_ids=[record.evidence_id for record in records],
|
||||
input_records=records,
|
||||
idempotency_key=request_id,
|
||||
priority=priority,
|
||||
event_at=event_at,
|
||||
time_quality=time_quality,
|
||||
)
|
||||
)
|
||||
return UnifiedWriteReceipt(
|
||||
@@ -203,29 +189,3 @@ class SourceWriteDisabledError(RuntimeError):
|
||||
def __init__(self, evidence_id: str) -> None:
|
||||
super().__init__("model_content_source_disabled")
|
||||
self.evidence_id = evidence_id
|
||||
|
||||
|
||||
def _job_time(
|
||||
records: tuple[EvidenceRecord, ...],
|
||||
) -> tuple[datetime, Literal["exact", "anchored", "session", "approximate"]]:
|
||||
"""Use the turn opener's wall clock, with an explicit fallback quality."""
|
||||
|
||||
opener = next(
|
||||
(record for record in records if record.role == "user"),
|
||||
records[0],
|
||||
)
|
||||
if opener.created_at is None:
|
||||
return opener.imported_at, "approximate"
|
||||
file_origin = opener.origin_meta.get(EvidenceOrigin.FILE)
|
||||
basis = file_origin.raw_meta.get("time_basis") if file_origin is not None else None
|
||||
turn_event_at = file_origin.raw_meta.get("turn_event_at") if file_origin is not None else None
|
||||
if isinstance(turn_event_at, str):
|
||||
try:
|
||||
return datetime.fromisoformat(turn_event_at), "exact"
|
||||
except ValueError:
|
||||
pass
|
||||
if basis == "file_mtime":
|
||||
return opener.created_at, "session"
|
||||
if EvidenceOrigin.HOOK in opener.origins:
|
||||
return opener.created_at, "exact"
|
||||
return opener.created_at, "exact"
|
||||
@@ -17,7 +17,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from one_memory.database import take_the_write_lock_first
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.imports.models import EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.memory_server.migrations import memory_tasks_ddl
|
||||
from one_memory.model_profiles import ModelProfile, ModelProfileSnapshot
|
||||
|
||||
@@ -174,60 +174,29 @@ def evidence_ref(record: EvidenceRecord) -> TaskEvidenceRef:
|
||||
"segment_end": int(marker["end"]),
|
||||
"segment_hash": str(marker["hash"]),
|
||||
}
|
||||
identity = record.model_dump(mode="json", exclude={"origin_meta", "imported_at", "updated_at"})
|
||||
return {
|
||||
"evidence_id": record.evidence_id,
|
||||
"revision": record.revision,
|
||||
"content_hash": record.content_hash,
|
||||
"identity_hash": _evidence_identity_hash(record, include_written_by=False),
|
||||
"identity_hash": sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _evidence_identity_hash(record: EvidenceRecord, *, include_written_by: bool) -> str:
|
||||
excluded = {"origin_meta", "imported_at", "updated_at"}
|
||||
if not include_written_by:
|
||||
excluded.add("written_by")
|
||||
identity = record.model_dump(mode="json", exclude=excluded)
|
||||
return sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def _identity_hash_matches(record: EvidenceRecord, expected: object) -> bool:
|
||||
candidates = {_evidence_identity_hash(record, include_written_by=False)}
|
||||
# Tasks created by pre-role-only builds keep their original proof. The
|
||||
# legacy metadata could also have changed in place without a revision, so
|
||||
# accept the old hash under every historical author guess.
|
||||
for written_by in WrittenBy:
|
||||
candidates.add(
|
||||
_evidence_identity_hash(
|
||||
record.model_copy(update={"written_by": written_by}),
|
||||
include_written_by=True,
|
||||
)
|
||||
)
|
||||
return expected in candidates
|
||||
|
||||
|
||||
def matches_evidence_ref(record: EvidenceRecord, ref: TaskEvidenceRef) -> bool:
|
||||
if "segment_hash" in ref:
|
||||
base = evidence_ref(record)
|
||||
if any(
|
||||
base.get(key) != ref.get(key)
|
||||
for key in ("evidence_id", "revision", "content_hash")
|
||||
for key in ("evidence_id", "revision", "content_hash", "identity_hash")
|
||||
):
|
||||
return False
|
||||
if not _identity_hash_matches(record, ref.get("identity_hash")):
|
||||
return False
|
||||
start, end = ref.get("segment_start"), ref.get("segment_end")
|
||||
if not isinstance(start, int) or not isinstance(end, int) or not (0 <= start < end):
|
||||
return False
|
||||
return sha256(record.verbatim[start:end].encode()).hexdigest() == ref["segment_hash"]
|
||||
current = evidence_ref(record)
|
||||
return all(
|
||||
(
|
||||
_identity_hash_matches(record, value)
|
||||
if key == "identity_hash"
|
||||
else current.get(key) == value
|
||||
)
|
||||
for key, value in ref.items()
|
||||
)
|
||||
return all(current.get(key) == value for key, value in ref.items())
|
||||
|
||||
|
||||
def segment_evidence_record(
|
||||
@@ -1008,44 +977,6 @@ class MemoryTaskStore:
|
||||
error_code=error_code,
|
||||
)
|
||||
|
||||
def release_unstarted(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
runner_token: str,
|
||||
error_code: str,
|
||||
) -> MemoryTask:
|
||||
"""Return a claimed task to pending when a user pause wins before HY.
|
||||
|
||||
Claim increments ``attempts``. A pause is not an execution failure, so
|
||||
the attempt is returned as well; repeatedly pausing and resuming can
|
||||
never exhaust an otherwise healthy task.
|
||||
"""
|
||||
now = datetime.now(UTC).isoformat()
|
||||
with self._connect(writing=True) as connection:
|
||||
changed = connection.execute(
|
||||
"""
|
||||
UPDATE memory_tasks
|
||||
SET status=?, runner_token=NULL,
|
||||
attempts=CASE WHEN attempts>0 THEN attempts-1 ELSE 0 END,
|
||||
last_error_code=?, duration_seconds=NULL, updated_at=?
|
||||
WHERE task_id=? AND status=? AND runner_token=?
|
||||
""",
|
||||
(
|
||||
MemoryTaskStatus.PENDING,
|
||||
error_code,
|
||||
now,
|
||||
task_id,
|
||||
MemoryTaskStatus.RUNNING,
|
||||
runner_token,
|
||||
),
|
||||
)
|
||||
if changed.rowcount != 1:
|
||||
raise MemoryTaskOwnershipError(
|
||||
"Memory task runner no longer owns this task"
|
||||
)
|
||||
return self.require(task_id)
|
||||
|
||||
def park_for_permission(self, task_id: str, *, error_code: str) -> bool:
|
||||
"""Park unstarted work until a later permission snapshot admits it."""
|
||||
now = datetime.now(UTC).isoformat()
|
||||
|
||||
@@ -1,404 +0,0 @@
|
||||
"""Persistent, content-free accounting for precipitation model requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import TypedDict
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.memory_server.migrations import model_request_audit_ddl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelRequestAuditContext:
|
||||
request_id: str
|
||||
|
||||
|
||||
_audit_context: ContextVar[ModelRequestAuditContext | None] = ContextVar(
|
||||
"one_memory_model_request_audit_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def audit_precipitation_request(request_id: str) -> Iterator[None]:
|
||||
"""Enable physical-request accounting for one memory precipitation."""
|
||||
|
||||
token = _audit_context.set(ModelRequestAuditContext(request_id=request_id))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_audit_context.reset(token)
|
||||
|
||||
|
||||
def _current_prompt_stage() -> str:
|
||||
try:
|
||||
resolver = getattr( # noqa: B009 - optional SDK version boundary
|
||||
import_module("hy_memory.utils.model_call_context"),
|
||||
"current_model_call_stage",
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
return "other"
|
||||
return str(resolver())
|
||||
|
||||
|
||||
class ModelRequestAuditStore:
|
||||
def __init__(self, database: Path) -> None:
|
||||
self.database = database
|
||||
with main_database(database).write() as connection, connection:
|
||||
for statement in model_request_audit_ddl("sqlite").split(";"):
|
||||
if statement.strip():
|
||||
connection.execute(statement)
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
request_id: str,
|
||||
profile_id: str,
|
||||
profile_revision: int,
|
||||
provider_kind: str,
|
||||
model: str,
|
||||
request_kind: str,
|
||||
prompt_stage: str,
|
||||
) -> str:
|
||||
audit_id = str(uuid4())
|
||||
with main_database(self.database).write() as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO model_request_audit (
|
||||
audit_id,request_id,profile_id,profile_revision,provider_kind,
|
||||
model,request_kind,prompt_stage,started_at,outcome
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
audit_id,
|
||||
request_id,
|
||||
profile_id,
|
||||
profile_revision,
|
||||
provider_kind,
|
||||
model,
|
||||
request_kind,
|
||||
prompt_stage,
|
||||
datetime.now(UTC).isoformat(),
|
||||
"in_flight",
|
||||
),
|
||||
)
|
||||
return audit_id
|
||||
|
||||
def finish(
|
||||
self,
|
||||
audit_id: str,
|
||||
*,
|
||||
outcome: str,
|
||||
http_status: int | None,
|
||||
error_type: str | None = None,
|
||||
provider_request_id: str | None = None,
|
||||
prompt_tokens: int | None = None,
|
||||
completion_tokens: int | None = None,
|
||||
total_tokens: int | None = None,
|
||||
cached_prompt_tokens: int | None = None,
|
||||
reasoning_tokens: int | None = None,
|
||||
) -> None:
|
||||
with main_database(self.database).write() as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE model_request_audit
|
||||
SET completed_at=?,outcome=?,http_status=?,error_type=?,
|
||||
provider_request_id=?,prompt_tokens=?,completion_tokens=?,
|
||||
total_tokens=?,cached_prompt_tokens=?,reasoning_tokens=?
|
||||
WHERE audit_id=?
|
||||
""",
|
||||
(
|
||||
datetime.now(UTC).isoformat(),
|
||||
outcome,
|
||||
http_status,
|
||||
error_type,
|
||||
provider_request_id,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens,
|
||||
cached_prompt_tokens,
|
||||
reasoning_tokens,
|
||||
audit_id,
|
||||
),
|
||||
)
|
||||
|
||||
def report(
|
||||
self,
|
||||
*,
|
||||
since: str | None = None,
|
||||
request_id: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> dict[str, object]:
|
||||
clauses: list[str] = []
|
||||
params: list[object] = []
|
||||
if since is not None:
|
||||
clauses.append("started_at>=?")
|
||||
params.append(since)
|
||||
if request_id is not None:
|
||||
clauses.append("request_id=?")
|
||||
params.append(request_id)
|
||||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
with main_database(self.database).read() as connection:
|
||||
summary = connection.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS request_count,
|
||||
SUM(CASE WHEN outcome='succeeded' THEN 1 ELSE 0 END) AS succeeded,
|
||||
SUM(CASE WHEN outcome='failed' THEN 1 ELSE 0 END) AS failed,
|
||||
SUM(CASE WHEN outcome='in_flight' THEN 1 ELSE 0 END) AS in_flight,
|
||||
SUM(prompt_tokens) AS prompt_tokens,
|
||||
SUM(completion_tokens) AS completion_tokens,
|
||||
SUM(total_tokens) AS total_tokens,
|
||||
SUM(cached_prompt_tokens) AS cached_prompt_tokens,
|
||||
SUM(reasoning_tokens) AS reasoning_tokens,
|
||||
AVG(prompt_tokens) AS avg_prompt_tokens,
|
||||
AVG(completion_tokens) AS avg_completion_tokens
|
||||
FROM model_request_audit {where}
|
||||
""",
|
||||
tuple(params),
|
||||
).fetchone()
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT audit_id,request_id,profile_id,profile_revision,provider_kind,
|
||||
model,request_kind,prompt_stage,started_at,completed_at,outcome,http_status,
|
||||
error_type,provider_request_id,prompt_tokens,completion_tokens,
|
||||
total_tokens,cached_prompt_tokens,reasoning_tokens
|
||||
FROM model_request_audit {where}
|
||||
ORDER BY started_at DESC,audit_id DESC LIMIT ?
|
||||
""",
|
||||
(*params, limit),
|
||||
).fetchall()
|
||||
minute_rows = connection.execute(
|
||||
f"""
|
||||
SELECT substr(started_at,1,16) || ':00Z' AS minute,
|
||||
prompt_stage,
|
||||
COUNT(*) AS request_count,
|
||||
SUM(prompt_tokens) AS prompt_tokens,
|
||||
SUM(completion_tokens) AS completion_tokens,
|
||||
SUM(total_tokens) AS total_tokens,
|
||||
AVG(prompt_tokens) AS avg_prompt_tokens,
|
||||
AVG(completion_tokens) AS avg_completion_tokens
|
||||
FROM model_request_audit {where}
|
||||
GROUP BY substr(started_at,1,16),prompt_stage
|
||||
ORDER BY minute,prompt_stage
|
||||
""",
|
||||
tuple(params),
|
||||
).fetchall()
|
||||
keys = (
|
||||
"audit_id",
|
||||
"request_id",
|
||||
"profile_id",
|
||||
"profile_revision",
|
||||
"provider_kind",
|
||||
"model",
|
||||
"request_kind",
|
||||
"prompt_stage",
|
||||
"started_at",
|
||||
"completed_at",
|
||||
"outcome",
|
||||
"http_status",
|
||||
"error_type",
|
||||
"provider_request_id",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"cached_prompt_tokens",
|
||||
"reasoning_tokens",
|
||||
)
|
||||
summary_keys = (
|
||||
"request_count",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"in_flight",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"cached_prompt_tokens",
|
||||
"reasoning_tokens",
|
||||
"avg_prompt_tokens",
|
||||
"avg_completion_tokens",
|
||||
)
|
||||
return {
|
||||
"summary": {
|
||||
key: (
|
||||
round(float(value), 2)
|
||||
if key.startswith("avg_") and value is not None
|
||||
else int(value)
|
||||
if value is not None
|
||||
else 0
|
||||
)
|
||||
for key, value in zip(summary_keys, summary, strict=True)
|
||||
},
|
||||
"requests": [
|
||||
dict(zip(keys, tuple(row), strict=True))
|
||||
for row in rows
|
||||
],
|
||||
"by_minute": [
|
||||
{
|
||||
"minute": str(row[0]),
|
||||
"prompt_stage": str(row[1]),
|
||||
"request_count": int(row[2]),
|
||||
"prompt_tokens": int(row[3] or 0),
|
||||
"completion_tokens": int(row[4] or 0),
|
||||
"total_tokens": int(row[5] or 0),
|
||||
"avg_prompt_tokens": round(float(row[6] or 0), 2),
|
||||
"avg_completion_tokens": round(float(row[7] or 0), 2),
|
||||
}
|
||||
for row in minute_rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _non_negative_int(value: object) -> int | None:
|
||||
return value if type(value) is int and value >= 0 else None
|
||||
|
||||
|
||||
class _ResponseUsage(TypedDict):
|
||||
provider_request_id: str | None
|
||||
prompt_tokens: int | None
|
||||
completion_tokens: int | None
|
||||
total_tokens: int | None
|
||||
cached_prompt_tokens: int | None
|
||||
reasoning_tokens: int | None
|
||||
|
||||
|
||||
def _empty_usage() -> _ResponseUsage:
|
||||
return {
|
||||
"provider_request_id": None,
|
||||
"prompt_tokens": None,
|
||||
"completion_tokens": None,
|
||||
"total_tokens": None,
|
||||
"cached_prompt_tokens": None,
|
||||
"reasoning_tokens": None,
|
||||
}
|
||||
|
||||
|
||||
def _response_usage(content: bytes) -> _ResponseUsage:
|
||||
try:
|
||||
body = json.loads(content)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
return _empty_usage()
|
||||
if not isinstance(body, dict):
|
||||
return _empty_usage()
|
||||
usage = body.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
usage = {}
|
||||
prompt_details = usage.get("prompt_tokens_details")
|
||||
completion_details = usage.get("completion_tokens_details")
|
||||
cached = (
|
||||
prompt_details.get("cached_tokens")
|
||||
if isinstance(prompt_details, dict)
|
||||
else usage.get("prompt_cache_hit_tokens")
|
||||
)
|
||||
reasoning = (
|
||||
completion_details.get("reasoning_tokens")
|
||||
if isinstance(completion_details, dict)
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"provider_request_id": body.get("id") if isinstance(body.get("id"), str) else None,
|
||||
"prompt_tokens": _non_negative_int(usage.get("prompt_tokens")),
|
||||
"completion_tokens": _non_negative_int(usage.get("completion_tokens")),
|
||||
"total_tokens": _non_negative_int(usage.get("total_tokens")),
|
||||
"cached_prompt_tokens": _non_negative_int(cached),
|
||||
"reasoning_tokens": _non_negative_int(reasoning),
|
||||
}
|
||||
|
||||
|
||||
def _request_kind(request: httpx.Request) -> str:
|
||||
path = request.url.path.rstrip("/")
|
||||
if path.endswith("/chat/completions"):
|
||||
return "chat_completion"
|
||||
if path.endswith("/embeddings"):
|
||||
return "embedding"
|
||||
return "model_http"
|
||||
|
||||
|
||||
def _request_model(content: bytes, fallback: str) -> str:
|
||||
try:
|
||||
body = json.loads(content)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
return fallback
|
||||
model = body.get("model") if isinstance(body, dict) else None
|
||||
return model if isinstance(model, str) and model else fallback
|
||||
|
||||
|
||||
class AuditedModelTransport(httpx.AsyncBaseTransport):
|
||||
"""Record one row for each physical precipitation HTTP attempt."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: ModelRequestAuditStore,
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_revision: int,
|
||||
provider_kind: str,
|
||||
model: str,
|
||||
inner: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.profile_id = profile_id
|
||||
self.profile_revision = profile_revision
|
||||
self.provider_kind = provider_kind
|
||||
self.model = model
|
||||
self.inner = inner or httpx.AsyncHTTPTransport(retries=0, trust_env=False)
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
context = _audit_context.get()
|
||||
if context is None:
|
||||
return await self.inner.handle_async_request(request)
|
||||
content = await request.aread()
|
||||
audit_id = await asyncio.to_thread(
|
||||
self.store.start,
|
||||
request_id=context.request_id,
|
||||
profile_id=self.profile_id,
|
||||
profile_revision=self.profile_revision,
|
||||
provider_kind=self.provider_kind,
|
||||
model=_request_model(content, self.model),
|
||||
request_kind=_request_kind(request),
|
||||
prompt_stage=_current_prompt_stage(),
|
||||
)
|
||||
try:
|
||||
response = await self.inner.handle_async_request(request)
|
||||
except BaseException as error:
|
||||
await asyncio.to_thread(
|
||||
self.store.finish,
|
||||
audit_id,
|
||||
outcome="failed",
|
||||
http_status=None,
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise
|
||||
content = await response.aread()
|
||||
usage = _response_usage(content)
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
self.store.finish,
|
||||
audit_id,
|
||||
outcome="succeeded" if response.is_success else "failed",
|
||||
http_status=response.status_code,
|
||||
error_type=None if response.is_success else f"http_{response.status_code}",
|
||||
**usage,
|
||||
)
|
||||
except Exception:
|
||||
# The durable in_flight row is more truthful than hiding the model
|
||||
# response after it has already incurred cost.
|
||||
logger.exception("model_request_audit_finish_failed audit_id=%s", audit_id)
|
||||
return response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.inner.aclose()
|
||||
@@ -87,7 +87,7 @@ def build_acceptance_pack(
|
||||
"""
|
||||
SELECT DISTINCT e.evidence_id
|
||||
FROM evidence e
|
||||
WHERE e.role = 'user'
|
||||
WHERE e.role = 'user' AND e.written_by = 'person'
|
||||
AND LENGTH(e.verbatim) <= 300
|
||||
AND (
|
||||
e.verbatim LIKE '%我是一名%' OR e.verbatim LIKE '%我是一个%'
|
||||
@@ -118,7 +118,7 @@ def build_acceptance_pack(
|
||||
for row in connection.execute(
|
||||
"""
|
||||
SELECT evidence_id FROM evidence
|
||||
WHERE role != 'user'
|
||||
WHERE role = 'user' AND written_by != 'person'
|
||||
"""
|
||||
):
|
||||
strata["attribution_edge"].add(row["evidence_id"])
|
||||
@@ -138,7 +138,11 @@ def build_acceptance_pack(
|
||||
quota,
|
||||
)
|
||||
if len(selected) < sample_size:
|
||||
eligible = (evidence_id for evidence_id, row in evidence.items() if row["role"] == "user")
|
||||
eligible = (
|
||||
evidence_id
|
||||
for evidence_id, row in evidence.items()
|
||||
if row["role"] == "user" and row["written_by"] == "person"
|
||||
)
|
||||
_extend_unique(selected, _stable_order(eligible, seed=f"{seed}:fill"), sample_size)
|
||||
selected = selected[:sample_size]
|
||||
if len(selected) < sample_size:
|
||||
@@ -159,11 +163,7 @@ def build_acceptance_pack(
|
||||
row_id=f"{index:03d}",
|
||||
evidence_id=evidence_id,
|
||||
source_id=source["source_id"],
|
||||
written_by=(
|
||||
"person"
|
||||
if source["role"] == "user"
|
||||
else ("agent" if source["role"] == "assistant" else source["role"])
|
||||
),
|
||||
written_by=source["written_by"],
|
||||
character_count=len(source["verbatim"]),
|
||||
content_hash=source["content_hash"],
|
||||
strata=tuple(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceContentKind, EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.sources.parsing import is_host_generated_user_turn, strip_host_wrappers
|
||||
from one_memory.input_sanitization import sanitize_model_input
|
||||
from one_memory.memory_processing import _is_hy_input_turn
|
||||
@@ -15,7 +15,10 @@ def identity_content_ready(record: EvidenceRecord) -> bool:
|
||||
# cleaning requires a source map and remains explicitly uncovered for now.
|
||||
return (
|
||||
record.content_kind is EvidenceContentKind.RAW_CONVERSATION
|
||||
and record.role in {"user", "assistant"}
|
||||
and (
|
||||
(record.role == "user" and record.written_by is WrittenBy.PERSON)
|
||||
or (record.role == "assistant" and record.written_by is WrittenBy.AGENT)
|
||||
)
|
||||
and _is_hy_input_turn(record)
|
||||
and not is_host_generated_user_turn(record.verbatim)
|
||||
and strip_host_wrappers(record.verbatim) == record.verbatim.strip()
|
||||
|
||||
@@ -10,6 +10,7 @@ from one_memory.imports.models import (
|
||||
MAX_VERBATIM_CHARACTERS,
|
||||
EvidenceContentKind,
|
||||
EvidenceRecord,
|
||||
WrittenBy,
|
||||
)
|
||||
from one_memory.imports.sources.parsing import (
|
||||
is_host_generated_user_turn,
|
||||
@@ -163,10 +164,10 @@ def prepare_mapped_content(record: EvidenceRecord) -> PreparedMappedContent:
|
||||
The returned text is transient; only metadata is suitable for an outer manifest.
|
||||
That manifest must independently bind Evidence identity/revision/hash and grants.
|
||||
"""
|
||||
if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or record.role not in {
|
||||
"user",
|
||||
"assistant",
|
||||
}:
|
||||
if record.content_kind is not EvidenceContentKind.RAW_CONVERSATION or not (
|
||||
(record.role == "user" and record.written_by is WrittenBy.PERSON)
|
||||
or (record.role == "assistant" and record.written_by is WrittenBy.AGENT)
|
||||
):
|
||||
raise MappedContentError("mapped_content_ineligible")
|
||||
original = record.verbatim
|
||||
if len(original) > MAX_VERBATIM_CHARACTERS:
|
||||
|
||||
@@ -11,7 +11,7 @@ from datetime import UTC
|
||||
|
||||
from one_memory import ingest_journal as journal
|
||||
from one_memory.context_raw import _check, _connection
|
||||
from one_memory.imports.models import EvidenceRecord
|
||||
from one_memory.imports.models import EvidenceRecord, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.retrieval.index_store import RetrievalIndexStore
|
||||
from one_memory.retrieval.models import EvidenceSpan, RawEvidence
|
||||
@@ -166,7 +166,7 @@ def materialize_raw_candidates(
|
||||
source_id=record.source_id,
|
||||
conversation_id=record.conversation_id,
|
||||
role=record.role,
|
||||
written_by="person" if record.role == "user" else "agent",
|
||||
written_by="person" if record.written_by is WrittenBy.PERSON else "agent",
|
||||
event_time=event_time,
|
||||
spans=tuple(spans),
|
||||
matched_by=("semantic",),
|
||||
|
||||
@@ -264,7 +264,7 @@ class ServiceLevelScheduler:
|
||||
self._quiet_admission_logged_at[key] = now
|
||||
if last is not None:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="service_level.scheduler",
|
||||
event="service_level.summary",
|
||||
outcome="succeeded",
|
||||
@@ -276,7 +276,7 @@ class ServiceLevelScheduler:
|
||||
)
|
||||
return
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
level="info",
|
||||
component="service_level.scheduler",
|
||||
event="service_level.admitted",
|
||||
outcome="succeeded",
|
||||
|
||||
@@ -56,10 +56,10 @@ def build_source_import_receipt(
|
||||
"SELECT count(*) AS messages, "
|
||||
"count(DISTINCT conversation_id || char(0) || COALESCE(workspace, '')) "
|
||||
"AS conversations, "
|
||||
"coalesce(sum(role='user'),0) AS person_messages, "
|
||||
"coalesce(sum(role='assistant'),0) AS agent_messages, "
|
||||
"coalesce(sum(role='tool'),0) AS tool_messages, "
|
||||
"coalesce(sum(role NOT IN ('user','assistant','tool')),0) "
|
||||
"coalesce(sum(written_by='person'),0) AS person_messages, "
|
||||
"coalesce(sum(written_by='agent'),0) AS agent_messages, "
|
||||
"coalesce(sum(written_by='tool'),0) AS tool_messages, "
|
||||
"coalesce(sum(written_by NOT IN ('person','agent','tool')),0) "
|
||||
"AS unknown_author_messages, "
|
||||
"coalesce(sum(created_at IS NULL),0) AS unknown_time_messages "
|
||||
"FROM evidence WHERE user_id=? AND source_id=?",
|
||||
@@ -68,9 +68,7 @@ def build_source_import_receipt(
|
||||
# Bound text reads before materializing snippets. These are samples, not a
|
||||
# recency ranking; sorting every large original body is unnecessary here.
|
||||
examples = db.execute(
|
||||
"SELECT evidence_id, role, CASE role "
|
||||
"WHEN 'user' THEN 'person' WHEN 'assistant' THEN 'agent' "
|
||||
"WHEN 'tool' THEN 'tool' ELSE 'unknown' END AS written_by, created_at, "
|
||||
"SELECT evidence_id, role, written_by, created_at, "
|
||||
"substr(verbatim,1,600) AS excerpt, length(verbatim)>600 AS truncated "
|
||||
"FROM evidence WHERE evidence_id IN ("
|
||||
"SELECT evidence_id FROM evidence WHERE user_id=? AND source_id=? LIMIT 3)",
|
||||
@@ -82,13 +80,10 @@ def build_source_import_receipt(
|
||||
# Qualify before LIMIT: agent/system rows must not consume the three samples.
|
||||
query_filter = " AND evidence_id=?" if query_evidence_id is not None else ""
|
||||
queries = db.execute(
|
||||
f"""SELECT evidence_id, revision AS evidence_revision, role,
|
||||
CASE role WHEN 'user' THEN 'person' WHEN 'assistant' THEN 'agent'
|
||||
WHEN 'tool' THEN 'tool' ELSE 'unknown' END AS written_by,
|
||||
created_at,
|
||||
CASE WHEN user_id=? AND source_id=? AND role='user'
|
||||
f"""SELECT evidence_id, revision AS evidence_revision, role, written_by, created_at,
|
||||
CASE WHEN user_id=? AND source_id=? AND role='user' AND written_by='person'
|
||||
AND length(verbatim)<=65536
|
||||
THEN display_query_projection(verbatim,role,'person')
|
||||
THEN display_query_projection(verbatim,role,written_by)
|
||||
ELSE NULL END AS projection
|
||||
FROM evidence WHERE user_id=? AND source_id=?{query_filter}
|
||||
AND projection IS NOT NULL
|
||||
|
||||
@@ -250,7 +250,6 @@
|
||||
| `test_model_profile_identity.py` | `contract` | Stable model editing and configured-vs-available credentials; no Keychain/network. |
|
||||
| `test_model_profile_queue_handover.py` | `contract` | Queued work when the model changes or its configuration is deleted (D75). |
|
||||
| `test_model_profiles.py` | `contract` | 覆盖 profile uses one model and keeps key out of database、api key updates reuse one stable secret reference、startup cleanup removes only secrets not referenced by current profiles 等 15 个场景。 |
|
||||
| `test_model_request_audit.py` | `contract` | 覆盖 failure is recorded without error message、missing usage stays unknown in detail and zero in summary、records one physical request with provider usage and no content 等 5 个场景。 |
|
||||
| `test_model_request_budget.py` | `contract` | 覆盖 persists failed request reservations without content、parallel leases cannot exceed request limit、unknown or excessive output and body are denied 等 6 个场景。 |
|
||||
| `test_mysql_lifecycle.py` | `contract` | A1-07: the decisions Runtime makes around the bundled MySQL. |
|
||||
| `test_mysql_platform.py` | `contract` | A1-07: the machine-facing half of managing the bundled MySQL. |
|
||||
@@ -328,7 +327,6 @@
|
||||
| `test_secret_store_error_codes.py` | `integration` | 3925a kernel item 3: keychain failure classification and backoff that heals. |
|
||||
| `test_secure_storage.py` | `integration` | 覆盖 isolated keychain service namespace、custom keychain service is refused outside isolated tests、packaged keychain budget covers loaded process startup 等 20 个场景。 |
|
||||
| `test_sedimentation_concurrency.py` | `contract` | Sedimentation runs several harnesses at once, but one harness at a time. |
|
||||
| `test_sedimentation_switch.py` | `contract` | Stopping model-driven sedimentation without losing the configuration. |
|
||||
| `test_service_level_pressure_acceptance.py` | `contract` | 覆盖 workbuddy recall remains available during 100k import 场景。 |
|
||||
| `test_service_levels.py` | `contract` | 覆盖 background never consumes the realtime reservation、realtime is next for the same source after background yields、import history gets one slice after nine realtime admissions 等 8 个场景。 |
|
||||
| `test_session_inbox.py` | `contract` | A session inbox is not a shared mailbox and a pull is not adoption. |
|
||||
@@ -338,7 +336,6 @@
|
||||
| `test_snapshot_pool.py` | `contract` | A whole-directory snapshot must not hold a thread everything else is waiting for. |
|
||||
| `test_soak_probe_effectiveness.py` | `integration` | The soak probe must observe the load it reports on. |
|
||||
| `test_source_access.py` | `contract` | 覆盖 source root requires explicit user grant、sensitive home directory is rejected without requesting os access、authorized file cannot escape root through a symlink 等 24 个场景。 |
|
||||
| `test_source_authorization_state.py` | `unit` | Why a source cannot be read, named separately from whether it can. |
|
||||
| `test_source_candidate_api.py` | `integration` | Candidate location changes must not implicitly grant or import anything. |
|
||||
| `test_source_candidate_connections.py` | `contract` | 覆盖 empty candidate revisions preserve default fingerprint、persisted candidate requires exact revision、connection resolves candidate in transaction and prior replay never regrants 等 7 个场景。 |
|
||||
| `test_source_candidates.py` | `contract` | 覆盖 preview is metadata only and cancel changes nothing、select persists only candidate and restarts with revision、compare and swap and aba cannot reuse old preview 等 9 个场景。 |
|
||||
@@ -388,4 +385,4 @@
|
||||
| `test_workspace_recall.py` | `contract` | Workspace memories were unreachable from every host; these pin the way in. |
|
||||
| `test_write_queue_lease_renewal.py` | `contract` | Rule 8: a lease must outlive the work it covers, and be renewed while it runs. |
|
||||
|
||||
共 383 个测试文件。
|
||||
共 380 个测试文件。
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
PRIMARY_MARKERS = ("unit", "contract", "integration", "performance", "live")
|
||||
EXPECTED_TEST_FILE_COUNT = 383
|
||||
EXPECTED_TEST_FILES_SHA256 = "3c1a3d3febaf478e1ba005fbcd22ad23d76fc5925b32b8d8740e4e65c554b922"
|
||||
EXPECTED_TEST_FILE_COUNT = 380
|
||||
EXPECTED_TEST_FILES_SHA256 = "9f075d18403328882c25325d29ebe77d0e3365d675ba19d56134a6a0fb9b64b4"
|
||||
|
||||
# Pure transformations, validation models, parity checks and state machines.
|
||||
# This list is intentionally small: uncertain files default to contract.
|
||||
@@ -35,7 +35,6 @@ UNIT_FILES = frozenset(
|
||||
"test_memory_server_job_state.py",
|
||||
"test_scan_roots.py",
|
||||
"test_scanner_classifier.py",
|
||||
"test_source_authorization_state.py",
|
||||
"test_scanner_sessionizer.py",
|
||||
"test_source_envelope_positions.py",
|
||||
"test_source_lifecycle.py",
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
import asyncio
|
||||
import gc
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.background_work import (
|
||||
BULK_WORKERS,
|
||||
bulk_pool,
|
||||
in_interactive_read,
|
||||
in_read_pool,
|
||||
in_the_background,
|
||||
shutdown_bulk_pool,
|
||||
)
|
||||
@@ -115,73 +110,3 @@ def test_interactive_read_does_not_wait_behind_saturated_default_pool() -> None:
|
||||
shutdown_bulk_pool()
|
||||
|
||||
assert waited < 0.25
|
||||
|
||||
|
||||
def test_an_abandoned_read_does_not_report_an_unretrieved_failure() -> None:
|
||||
"""A read that outlives its caller must not leave its failure lying about.
|
||||
|
||||
`in_read_pool` shields the worker, so giving up on an answer does not yank
|
||||
a query out from under SQLite. The caller cancels; the read runs on. When
|
||||
such an abandoned read then failed, nothing awaited its future any more and
|
||||
asyncio reported an exception nobody retrieved -- which the loop handler
|
||||
files as `core.loop.task_exception`, a red error sitting next to a lane
|
||||
that had already degraded on its own budget and already said so.
|
||||
"""
|
||||
|
||||
def open_pool() -> ThreadPoolExecutor:
|
||||
return ThreadPoolExecutor(max_workers=1, thread_name_prefix="test-read-pool")
|
||||
|
||||
async def scenario() -> list[BaseException]:
|
||||
loop = asyncio.get_running_loop()
|
||||
reported: list[BaseException] = []
|
||||
previous = loop.get_exception_handler()
|
||||
loop.set_exception_handler(lambda _loop, context: reported.append(context.get("exception")))
|
||||
gate = threading.Event()
|
||||
|
||||
def work() -> None:
|
||||
gate.wait(5.0)
|
||||
raise TimeoutError("raw_context_timeout")
|
||||
|
||||
pool = open_pool()
|
||||
try:
|
||||
read = asyncio.create_task(in_read_pool(pool, work))
|
||||
await asyncio.sleep(0.05)
|
||||
read.cancel() # the lane ran out of budget; the read is still going
|
||||
await asyncio.sleep(0.10) # the cancellation settles first
|
||||
gate.set() # only now does the abandoned read fail
|
||||
await asyncio.gather(read, return_exceptions=True)
|
||||
# Let the shielded worker finish, then let asyncio notice anything
|
||||
# that nobody ever retrieved. The reference has to go: asyncio only
|
||||
# reports an unretrieved outcome when the future is collected.
|
||||
del read
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0.02)
|
||||
gc.collect()
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
loop.set_exception_handler(previous)
|
||||
return [error for error in reported if isinstance(error, TimeoutError)]
|
||||
|
||||
assert asyncio.run(scenario()) == []
|
||||
|
||||
|
||||
def test_an_awaited_read_still_raises_its_failure() -> None:
|
||||
"""Marking the outcome seen must not swallow it for a caller who waited."""
|
||||
|
||||
def open_pool() -> ThreadPoolExecutor:
|
||||
return ThreadPoolExecutor(max_workers=1, thread_name_prefix="test-read-pool")
|
||||
|
||||
async def scenario() -> str:
|
||||
def work() -> None:
|
||||
raise TimeoutError("raw_context_timeout")
|
||||
|
||||
pool = open_pool()
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
await in_read_pool(pool, work)
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
return "raised"
|
||||
|
||||
assert asyncio.run(scenario()) == "raised"
|
||||
@@ -1,5 +1,4 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -14,7 +13,6 @@ from one_memory.diagnostic_log import (
|
||||
configure_diagnostics,
|
||||
current_trace_id,
|
||||
database_error_code,
|
||||
diagnostic_event,
|
||||
diagnostic_exception,
|
||||
diagnostics_enabled,
|
||||
diagnostics_requested,
|
||||
@@ -25,32 +23,6 @@ from one_memory.diagnostic_log import (
|
||||
)
|
||||
|
||||
|
||||
def test_debug_events_are_collected_only_at_debug_level(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.INFO)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="debug",
|
||||
component="core.poll",
|
||||
event="poll.completed",
|
||||
outcome="succeeded",
|
||||
)
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.task",
|
||||
event="memory_task.finished",
|
||||
outcome="completed",
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["event"] for event in events] == ["memory_task.finished"]
|
||||
|
||||
|
||||
def test_high_frequency_success_events_are_summarized_but_warnings_are_not(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
@@ -125,53 +97,6 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) -> None:
|
||||
"""Rejection protects the log; it must not become the caller's outage.
|
||||
|
||||
Two of our own nodes shipped with unlisted `details` keys. Both sat on a
|
||||
request path, so the rejection surfaced as a 500 to the person using the
|
||||
app — including the sedimentation switch, whose every request failed
|
||||
because logging the toggle raised before the endpoint could answer.
|
||||
"""
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"prompt": "private"},
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
# Rejected, so nothing is written — but the call returned.
|
||||
assert not receipt.path.exists() or receipt.path.read_text() == ""
|
||||
|
||||
|
||||
def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
|
||||
"""`enabled` is a bounded boolean and has to be a legal detail key.
|
||||
|
||||
Omitting it from the allow-list is what made the switch's own endpoint
|
||||
answer 500 while still saving the new value.
|
||||
"""
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
diagnostic_event(
|
||||
level="info",
|
||||
component="memory.sedimentation",
|
||||
event="memory_sedimentation.toggled",
|
||||
outcome="succeeded",
|
||||
details={"enabled": False},
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["event"] for event in events] == ["memory_sedimentation.toggled"]
|
||||
assert events[0]["details"]["enabled"] is False
|
||||
|
||||
|
||||
def test_exception_diagnostic_keeps_code_location_not_message():
|
||||
secret = "private-provider-response-918273"
|
||||
log = DiagnosticLog(Path.cwd())
|
||||
|
||||
@@ -16,15 +16,16 @@ import pytest
|
||||
|
||||
from one_memory.diagnostic_log import _DETAIL_KEYS
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
|
||||
|
||||
#: Every module, not a hand-picked list. The two payloads that caused real
|
||||
#: outages were in files nobody thought to add here, so the only shape of this
|
||||
#: check that can actually hold the line is "every module, always".
|
||||
SOURCES = tuple(
|
||||
sorted(str(path.relative_to(ROOT)) for path in ROOT.rglob("*.py") if path.name != "__init__.py")
|
||||
SOURCES = (
|
||||
"api.py",
|
||||
"imports/async_bridge.py",
|
||||
"imports/live_sync.py",
|
||||
"imports/runner.py",
|
||||
"memory_processing.py",
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
|
||||
|
||||
|
||||
def _literal_calls(path: Path):
|
||||
tree = ast.parse(path.read_text())
|
||||
|
||||
@@ -162,6 +162,8 @@ def test_import_request_trace_reaches_the_background_job_without_original_text(t
|
||||
]
|
||||
names = [event["event"] for event in events]
|
||||
assert "import.request.started" in names
|
||||
assert "import.queued" in names
|
||||
assert "import.started" in names
|
||||
assert "import.finished" in names
|
||||
finished = next(event for event in events if event["event"] == "import.finished")
|
||||
assert {
|
||||
@@ -173,17 +175,18 @@ def test_import_request_trace_reaches_the_background_job_without_original_text(t
|
||||
assert secret not in written
|
||||
|
||||
|
||||
def test_default_info_omits_parser_and_resource_internals(tmp_path):
|
||||
def test_import_run_logs_parse_stages_and_core_resources_without_content(tmp_path):
|
||||
written, job_id = _run_codex_import(tmp_path)
|
||||
events = list(map(json.loads, written.splitlines()))
|
||||
job_events = [event for event in events if event.get("job_id") == job_id]
|
||||
names = [event["event"] for event in job_events]
|
||||
assert "import.parse.started" not in names
|
||||
assert "import.parse.completed" not in names
|
||||
assert "import.phase_changed" not in names
|
||||
assert "import.finished" in names
|
||||
assert names.index("import.parse.started") < names.index("import.parse.completed")
|
||||
assert "import.phase_changed" in names
|
||||
completed = next(event for event in job_events if event["event"] == "import.parse.completed")
|
||||
assert completed["outcome"] == "succeeded"
|
||||
assert {"stored_count", "skipped_count", "error_count"} <= completed["details"].keys()
|
||||
everything = [event["event"] for event in events]
|
||||
assert "core.resource_snapshot" not in everything
|
||||
assert "core.resource_snapshot" in everything
|
||||
assert everything.index("core.ready") < everything.index("core.stopped")
|
||||
assert IMPORT_SECRET not in written
|
||||
|
||||
@@ -227,5 +230,4 @@ def test_a_failing_import_finished_log_does_not_escape_the_supervisor_task(tmp_p
|
||||
|
||||
monkeypatch.setattr(supervisor, "diagnostic_event", rejecting)
|
||||
written, _job_id = _run_codex_import(tmp_path)
|
||||
assert "core.stopped" in written
|
||||
assert "import.finished" not in written
|
||||
assert "import.started" in written
|
||||
@@ -33,42 +33,6 @@ def test_trace_lookup_is_disabled_until_detailed_logging_is_enabled(tmp_path):
|
||||
assert response.json() == {"schema_version": 1, "state": "disabled", "events": []}
|
||||
|
||||
|
||||
def test_model_usage_route_returns_content_free_totals(tmp_path):
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path,
|
||||
source_home=tmp_path / "home",
|
||||
api_token="trace-token",
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
with TestClient(app, headers={"Authorization": "Bearer trace-token"}) as client:
|
||||
response = client.get("/api/v1/diagnostics/model-usage")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"request_count": 0,
|
||||
"succeeded": 0,
|
||||
"failed": 0,
|
||||
"in_flight": 0,
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"cached_prompt_tokens": 0,
|
||||
"reasoning_tokens": 0,
|
||||
"avg_prompt_tokens": 0,
|
||||
"avg_completion_tokens": 0,
|
||||
},
|
||||
"requests": [],
|
||||
"by_minute": [],
|
||||
}
|
||||
|
||||
|
||||
def test_trace_lookup_returns_only_safe_structured_steps(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
set_detailed_logging(tmp_path, True)
|
||||
|
||||
@@ -2,7 +2,6 @@ import json
|
||||
import sqlite3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
@@ -15,7 +14,6 @@ from one_memory.imports.models import (
|
||||
WrittenBy,
|
||||
)
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_tasks import evidence_ref, matches_evidence_ref
|
||||
from one_memory.workspace_resolution import KnownWorkspace
|
||||
|
||||
|
||||
@@ -27,7 +25,6 @@ def evidence(
|
||||
verbatim: str = "Remember the source, not the guess.",
|
||||
origin: EvidenceOrigin = EvidenceOrigin.FILE,
|
||||
written_by: WrittenBy = WrittenBy.PERSON,
|
||||
role: str = "user",
|
||||
source_ordinal: str | None = None,
|
||||
) -> EvidenceInput:
|
||||
return EvidenceInput(
|
||||
@@ -38,7 +35,7 @@ def evidence(
|
||||
source_ordinal=source_ordinal
|
||||
if source_ordinal is not None
|
||||
else ("line:1" if message_id is None and correlation_id is None else None),
|
||||
role=role,
|
||||
role="user",
|
||||
written_by=written_by,
|
||||
created_at=datetime(2026, 8, 31, 12, 0, tzinfo=UTC),
|
||||
verbatim=verbatim,
|
||||
@@ -90,7 +87,7 @@ def test_rescanning_an_unchanged_file_does_not_write_the_message_again(tmp_path)
|
||||
"""A rescan is idempotent because the identity is derived from the content.
|
||||
|
||||
`EvidenceInput.idempotency_key` is a hash of the source, conversation,
|
||||
protocol role and content hash, and `evidence_id` is a uuid5 over that
|
||||
author, ordinal and content hash, and `evidence_id` is a uuid5 over that
|
||||
key, so replaying an unchanged file reproduces the same primary key and
|
||||
`ON CONFLICT` absorbs it. Verified: removing the anchoring logic still
|
||||
leaves this green, so this pins the contract, not one implementation.
|
||||
@@ -105,87 +102,6 @@ def test_rescanning_an_unchanged_file_does_not_write_the_message_again(tmp_path)
|
||||
assert store.count() == 1
|
||||
|
||||
|
||||
def test_derived_identity_uses_role_not_host_author_guess(tmp_path) -> None:
|
||||
store = EvidenceStore(tmp_path / "one-memory.sqlite3")
|
||||
first = store.put(evidence(message_id=None, written_by=WrittenBy.UNKNOWN))
|
||||
replay = store.put(evidence(message_id=None, written_by=WrittenBy.PERSON))
|
||||
assistant = store.put(evidence(message_id=None, role="assistant", written_by=WrittenBy.AGENT))
|
||||
|
||||
assert replay.record.evidence_id == first.record.evidence_id
|
||||
assert replay.was_created is False
|
||||
assert assistant.record.evidence_id != first.record.evidence_id
|
||||
assert store.count() == 2
|
||||
|
||||
|
||||
def test_task_proof_ignores_legacy_written_by_metadata(tmp_path) -> None:
|
||||
store = EvidenceStore(tmp_path / "one-memory.sqlite3")
|
||||
record = store.put(evidence()).record
|
||||
proof = evidence_ref(record)
|
||||
|
||||
changed_author = record.model_copy(update={"written_by": WrittenBy.AGENT})
|
||||
legacy_identity = record.model_dump(
|
||||
mode="json",
|
||||
exclude={"origin_meta", "imported_at", "updated_at"},
|
||||
)
|
||||
legacy_proof = {
|
||||
**proof,
|
||||
"identity_hash": sha256(
|
||||
json.dumps(legacy_identity, sort_keys=True).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
|
||||
assert matches_evidence_ref(changed_author, proof) is True
|
||||
assert matches_evidence_ref(changed_author, legacy_proof) is True
|
||||
|
||||
|
||||
def test_rescan_reuses_a_row_stored_under_written_by_derived_v3(tmp_path) -> None:
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(database)
|
||||
item = evidence(message_id=None, written_by=WrittenBy.UNKNOWN)
|
||||
original = store.put(item).record
|
||||
with sqlite3.connect(database) as connection, connection:
|
||||
connection.execute(
|
||||
"UPDATE evidence SET idempotency_key=? WHERE evidence_id=?",
|
||||
(item.written_by_derived_v3_idempotency_key, original.evidence_id),
|
||||
)
|
||||
|
||||
replay = store.put(evidence(message_id=None, written_by=WrittenBy.PERSON))
|
||||
|
||||
assert replay.was_created is False
|
||||
assert replay.record.evidence_id == original.evidence_id
|
||||
assert store.count() == 1
|
||||
|
||||
|
||||
def test_legacy_identity_role_change_revises_the_existing_evidence(tmp_path) -> None:
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(database)
|
||||
old = evidence(
|
||||
message_id=None,
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
)
|
||||
original = store.put(old).record
|
||||
with sqlite3.connect(database) as connection, connection:
|
||||
connection.execute(
|
||||
"UPDATE evidence SET idempotency_key=? WHERE evidence_id=?",
|
||||
(old.written_by_derived_v3_idempotency_key, original.evidence_id),
|
||||
)
|
||||
|
||||
replay = store.put(
|
||||
evidence(
|
||||
message_id=None,
|
||||
role="user",
|
||||
written_by=WrittenBy.AGENT,
|
||||
)
|
||||
)
|
||||
|
||||
assert replay.record.evidence_id == original.evidence_id
|
||||
assert replay.was_revised is True
|
||||
assert replay.record.revision == original.revision + 1
|
||||
assert replay.record.role == "user"
|
||||
assert store.count() == 1
|
||||
|
||||
|
||||
def test_changed_text_at_one_position_is_a_new_message_not_a_silent_edit(tmp_path) -> None:
|
||||
"""An edited line becomes new evidence, never an overwrite of the old one.
|
||||
|
||||
@@ -230,7 +146,6 @@ def test_personal_candidate_prefilter_excludes_unrelated_user_turns(tmp_path) ->
|
||||
evidence(
|
||||
message_id="assistant",
|
||||
verbatim="我偏好简洁文案。",
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -310,7 +310,6 @@ def test_ollama_profile_ignores_ambient_remote_storage_and_models(
|
||||
monkeypatch.setenv("MEMORY_VECTOR_HOST", "remote.example.com")
|
||||
monkeypatch.setenv("MEMORY_COLLECTION_NAME", "hostile-collection")
|
||||
monkeypatch.setenv("MEMORY_EMBEDDING_DIMS", "99")
|
||||
monkeypatch.delenv("MEMORY_CODING_TOOL_RESULT_MAX_BYTES", raising=False)
|
||||
profile = snapshot(
|
||||
ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
@@ -334,20 +333,6 @@ def test_ollama_profile_ignores_ambient_remote_storage_and_models(
|
||||
assert config.vector_store.embedding_dims == 768
|
||||
assert config.graph_store.provider == "kuzu"
|
||||
assert str(tmp_path) in config.graph_store.db_path
|
||||
assert config.coding.tool_result_max_bytes == 512
|
||||
|
||||
|
||||
def test_memhub_coding_tool_result_ceiling_is_configurable(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("MEMORY_CODING_TOOL_RESULT_MAX_BYTES", "1024")
|
||||
profile = snapshot(
|
||||
ProviderKind.OLLAMA,
|
||||
base_url="http://127.0.0.1:11434",
|
||||
has_api_key=False,
|
||||
)
|
||||
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert config.coding.tool_result_max_bytes == 1024
|
||||
|
||||
|
||||
def _kuzu_graph_with_embedding_width(db_path: Path, width: int) -> None:
|
||||
@@ -1341,7 +1326,7 @@ def test_hy_runtime_matches_public_sdk_without_reranker_or_service_backends(
|
||||
config = build_hy_memory_config(profile, api_key=None, data_dir=tmp_path)
|
||||
|
||||
assert HY_MEMORY_DISTRIBUTION == "hy-memory"
|
||||
assert HY_MEMORY_VERSION == "2.0.4"
|
||||
assert HY_MEMORY_VERSION == "2.0.3"
|
||||
assert HY_MEMORY_MODE == "ultra"
|
||||
assert HY_MEMORY_READER == "hybrid"
|
||||
assert os.environ["MEMORY_MODE"] == "ultra"
|
||||
|
||||
@@ -5,7 +5,6 @@ Checking that contract automatically prevents a silently flipped rerank flag
|
||||
from going unnoticed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
@@ -17,7 +16,6 @@ from one_memory.hy_runtime_config import (
|
||||
HY_MEMORY_MODE,
|
||||
HY_MEMORY_READER,
|
||||
HY_MEMORY_VERSION,
|
||||
apply_diagnostic_log_level,
|
||||
hy_memory_version,
|
||||
)
|
||||
|
||||
@@ -71,19 +69,3 @@ def test_the_pin_matches_the_lock_file_and_its_wheel():
|
||||
)
|
||||
assert f"hy_memory-{HY_MEMORY_VERSION}-" in lock
|
||||
assert f'specifier = "=={HY_MEMORY_VERSION}"' in lock
|
||||
|
||||
|
||||
def test_info_level_suppresses_noisy_third_party_success_logs():
|
||||
names = ("", "one_memory", "httpx", "httpcore", "watchfiles", "jieba")
|
||||
previous = {name: logging.getLogger(name).level for name in names}
|
||||
try:
|
||||
apply_diagnostic_log_level("info")
|
||||
assert logging.getLogger("one_memory").level == logging.INFO
|
||||
for name in ("httpx", "httpcore", "watchfiles", "jieba"):
|
||||
assert logging.getLogger(name).level == logging.WARNING
|
||||
apply_diagnostic_log_level("debug")
|
||||
for name in ("one_memory", "httpx", "httpcore", "watchfiles", "jieba"):
|
||||
assert logging.getLogger(name).level == logging.DEBUG
|
||||
finally:
|
||||
for name, level in previous.items():
|
||||
logging.getLogger(name).setLevel(level)
|
||||
@@ -69,7 +69,7 @@ def test_no_idempotency_key_means_unknown_must_remain() -> None:
|
||||
|
||||
|
||||
def test_measured_sdk_verdict_is_recorded() -> None:
|
||||
"""Pin the measured answer for `hy-memory==2.0.4`.
|
||||
"""Pin the measured answer for `hy-memory==2.0.3`.
|
||||
|
||||
The finding: `add()` takes a `request_id` but nothing de-duplicates on it,
|
||||
while `MemoryInspector.get_pipeline_logs(request_id=...)` can look the
|
||||
|
||||
@@ -59,35 +59,6 @@ def test_bound_context_rejects_a_same_named_conversation_in_another_workspace(tm
|
||||
HyWriteBindings(HySourceContext(current=(current,), history=(other_workspace,)))
|
||||
|
||||
|
||||
def test_bound_context_accepts_a_user_turn_the_host_left_unattributed(tmp_path):
|
||||
"""A missing author is not a scope violation.
|
||||
|
||||
WorkBuddy records real user turns with `written_by=unknown` — 1078 of them
|
||||
across 1002 conversations on this machine. Refusing the write over that
|
||||
failed the whole conversation: the task ended `model_content_outcome_unknown`
|
||||
after two attempts, so a memory that may well have been written could never
|
||||
be confirmed, and the turn was never sedimented again.
|
||||
"""
|
||||
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
|
||||
|
||||
store = EvidenceStore(tmp_path / "evidence.db")
|
||||
current = store.put(
|
||||
EvidenceInput(
|
||||
source_id="workbuddy",
|
||||
conversation_id="atlas",
|
||||
message_id="now",
|
||||
role="user",
|
||||
written_by=WrittenBy.UNKNOWN,
|
||||
verbatim="帮我看下这个报错。",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
|
||||
bound = HyWriteBindings(HySourceContext(current=(current,)))
|
||||
|
||||
assert bound.sources[0]["role"] == "user"
|
||||
|
||||
|
||||
def test_bound_context_rejects_a_duplicated_evidence_id(tmp_path):
|
||||
"""The same Evidence must not be cited twice in one HY context.
|
||||
|
||||
@@ -248,8 +219,18 @@ def test_profile_rendering_keeps_per_field_evidence_without_dropping_native_prof
|
||||
assert bindings.profile_support("The user's name is 陈.", {"name": "陈"}) == ()
|
||||
|
||||
|
||||
def test_bound_context_uses_protocol_role_not_author_guess(tmp_path):
|
||||
"""Host author guesses never reject a valid user/assistant/tool message."""
|
||||
def test_bound_context_refuses_a_turn_with_a_wrong_author_pair(tmp_path):
|
||||
"""The author must match the role: only user/person, assistant/agent, tool/tool.
|
||||
|
||||
The objective requires 作者 to be truthful. The other scope tests only ever
|
||||
build well-formed pairs, so this pins the pair check itself: an assistant
|
||||
turn attributed to a person, or a tool turn attributed to one, must be
|
||||
refused with `context_scope` rather than silently trusted.
|
||||
|
||||
`user` + AGENT is deliberately accepted since D98: the host parser marks a
|
||||
sub agent's question that way, and that turn is sent like any other. What
|
||||
is refused is the pair that cannot occur, not the one we now allow.
|
||||
"""
|
||||
from one_memory.hy_source_context import HySourceContext, HyWriteBindings
|
||||
|
||||
store = EvidenceStore(tmp_path / "evidence.db")
|
||||
@@ -258,7 +239,8 @@ def test_bound_context_uses_protocol_role_not_author_guess(tmp_path):
|
||||
|
||||
assistant = source(store, "reply", "assistant", "好的,继续推进。")
|
||||
wrong_author = assistant.model_copy(update={"written_by": WrittenBy.PERSON})
|
||||
HyWriteBindings(HySourceContext(current=(wrong_author,)))
|
||||
with pytest.raises(ValueError, match="context_scope"):
|
||||
HyWriteBindings(HySourceContext(current=(wrong_author,)))
|
||||
|
||||
# `source()` maps non-user roles to AGENT, so a tool turn needs its own build.
|
||||
tool = store.put(
|
||||
@@ -273,10 +255,7 @@ def test_bound_context_uses_protocol_role_not_author_guess(tmp_path):
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
assert HyWriteBindings(HySourceContext(current=(tool,)))
|
||||
assert HyWriteBindings(HySourceContext(current=(tool,))) # valid tool/tool pair
|
||||
wrong_tool = tool.model_copy(update={"written_by": WrittenBy.PERSON})
|
||||
HyWriteBindings(HySourceContext(current=(wrong_tool,)))
|
||||
|
||||
unsupported = assistant.model_copy(update={"role": "system"})
|
||||
with pytest.raises(ValueError, match="context_scope"):
|
||||
HyWriteBindings(HySourceContext(current=(unsupported,)))
|
||||
HyWriteBindings(HySourceContext(current=(wrong_tool,)))
|
||||
@@ -111,7 +111,7 @@ def test_import_metadata_changes_invalidate_without_authorizing_content(setup, c
|
||||
@pytest.mark.parametrize("cause", ["import", "maintenance"])
|
||||
@pytest.mark.parametrize(
|
||||
"column,value",
|
||||
[("verbatim", "changed raw body"), ("role", "assistant")],
|
||||
[("verbatim", "changed raw body"), ("written_by", "agent"), ("role", "assistant")],
|
||||
)
|
||||
def test_material_change_eligibility_depends_on_trusted_cause(setup, cause, column, value):
|
||||
db, store, _ = setup
|
||||
@@ -126,17 +126,6 @@ def test_material_change_eligibility_depends_on_trusted_cause(setup, cause, colu
|
||||
assert new.kind == ("revise" if cause == "import" else "metadata")
|
||||
|
||||
|
||||
def test_legacy_written_by_change_is_not_a_material_event(setup):
|
||||
db, store, _ = setup
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
key = put(store, db)
|
||||
journal.initialize(db)
|
||||
old = journal.current(db, key)
|
||||
db.execute("UPDATE evidence SET written_by='agent' WHERE evidence_id=?", (key,))
|
||||
|
||||
assert journal.record(db, key, cause="import") == old
|
||||
|
||||
|
||||
def test_complete_origin_metadata_changes_and_observation_noise(setup):
|
||||
db, store, _ = setup
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
@@ -1256,9 +1255,6 @@ def test_triggering_a_sync_leaves_the_loop_free_to_answer(tmp_path) -> None:
|
||||
|
||||
def test_live_sync_diagnostics_only_emit_state_changes(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
supervisor = PollingLiveSyncSupervisor(
|
||||
@@ -1271,7 +1267,5 @@ def test_live_sync_diagnostics_only_emit_state_changes(tmp_path, monkeypatch):
|
||||
supervisor._set_health("codex", unavailable.model_copy(update={"source_id": "codex"}))
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [event["event"] for event in events] == ["live_sync.state_changed"]
|
||||
assert events[0]["level"] == "warn"
|
||||
@@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
@@ -88,9 +87,6 @@ def _segment_at_the_legacy_window(monkeypatch):
|
||||
|
||||
def test_stale_known_reason_diagnostics_never_fail_dispatch(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
_report_stale_known_reasons(
|
||||
@@ -99,7 +95,6 @@ def test_stale_known_reason_diagnostics_never_fail_dispatch(tmp_path, monkeypatc
|
||||
)
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
|
||||
records = [json.loads(line) for line in receipt.path.read_text().splitlines()]
|
||||
assert [(row["details"]["reason"], row["details"]["count"]) for row in records] == [
|
||||
@@ -311,23 +306,16 @@ def test_memory_task_logs_six_step_durations_without_content(tmp_path) -> None:
|
||||
content=marker,
|
||||
)
|
||||
task = _task(tasks, profile, record)
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path)
|
||||
try:
|
||||
asyncio.run(_runner(tmp_path, stores, FakeBridge()).run(task.task_id))
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
written = receipt.path.read_text()
|
||||
events = [json.loads(line) for line in written.splitlines()]
|
||||
started = next(event for event in events if event["event"] == "memory_task.started")
|
||||
assert started["level"] == "info"
|
||||
steps = [event for event in events if event["event"] == "memory_task.step_completed"]
|
||||
assert [event["details"]["phase"] for event in steps] == ["1", "2", "3", "4", "5", "6"]
|
||||
finished = next(event for event in events if event["event"] == "memory_task.finished")
|
||||
assert finished["level"] == "info"
|
||||
assert finished["details"] == {
|
||||
"attempt": 1,
|
||||
"pending_count": 0,
|
||||
@@ -1402,9 +1390,6 @@ def test_backfill_scan_logs_inventory_and_queue_without_content(tmp_path) -> Non
|
||||
conversation_id=f"conversation-{index}",
|
||||
content_kind=EvidenceContentKind.MEMORY_CONTENT,
|
||||
)
|
||||
root = logging.getLogger()
|
||||
previous = root.level
|
||||
root.setLevel(logging.DEBUG)
|
||||
receipt = configure_diagnostics(tmp_path / "logs")
|
||||
|
||||
async def exercise() -> None:
|
||||
@@ -1424,7 +1409,6 @@ def test_backfill_scan_logs_inventory_and_queue_without_content(tmp_path) -> Non
|
||||
asyncio.run(exercise())
|
||||
finally:
|
||||
reset_diagnostics()
|
||||
root.setLevel(previous)
|
||||
|
||||
written = receipt.path.read_text()
|
||||
events = [json.loads(line) for line in written.splitlines()]
|
||||
@@ -1780,7 +1764,7 @@ def test_process_records_bounds_a_hung_model_call_by_the_profile_timeout(
|
||||
with sqlite3.connect(tmp_path / "one-memory.sqlite3") as db:
|
||||
db.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS source_grants (
|
||||
CREATE TABLE source_grants (
|
||||
user_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL,
|
||||
@@ -2038,126 +2022,6 @@ def test_oversized_single_turn_segments_are_each_processed(tmp_path) -> None:
|
||||
assert len(governance.list(user_id="local")) == 2
|
||||
|
||||
|
||||
def test_direct_processing_uses_the_supplied_frozen_revision(tmp_path) -> None:
|
||||
stores = _setup(tmp_path)
|
||||
evidence, profiles, tasks, governance, secrets, _profile = stores
|
||||
old_user = _put(
|
||||
evidence,
|
||||
message_id="rewound-user",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="old branch",
|
||||
)
|
||||
reply = _put(
|
||||
evidence,
|
||||
message_id="rewound-reply",
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
content="old reply",
|
||||
)
|
||||
revised = _put(
|
||||
evidence,
|
||||
message_id="rewound-user",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="new branch",
|
||||
)
|
||||
assert revised.evidence_id == old_user.evidence_id
|
||||
assert revised.revision == old_user.revision + 1
|
||||
with sqlite3.connect(tasks.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE source_grants (
|
||||
user_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL,
|
||||
requested_path TEXT NOT NULL,
|
||||
root_identity TEXT NOT NULL,
|
||||
candidate_kind TEXT NOT NULL,
|
||||
granted_at TEXT NOT NULL,
|
||||
revoked_at TEXT,
|
||||
revision INTEGER NOT NULL,
|
||||
PRIMARY KEY (user_id, source_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO source_grants (
|
||||
user_id,source_id,root_path,requested_path,root_identity,
|
||||
candidate_kind,granted_at,revoked_at,revision
|
||||
) VALUES (?,?,?,?,?,?,?,NULL,1)
|
||||
""",
|
||||
(
|
||||
"local",
|
||||
"cursor",
|
||||
"/authorized",
|
||||
"/authorized",
|
||||
"test-root",
|
||||
"default",
|
||||
datetime.now(UTC).isoformat(),
|
||||
),
|
||||
)
|
||||
bridge = FakeBridge(outputs=())
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=tasks,
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=evidence,
|
||||
governance=governance,
|
||||
data_dir=tmp_path,
|
||||
runner_factory=lambda: _runner(tmp_path, stores, bridge),
|
||||
)
|
||||
|
||||
completed = asyncio.run(supervisor.process_records((old_user, reply)))
|
||||
|
||||
assert len(completed) == 1
|
||||
assert [turn.content for turn in bridge.requests[0].turns] == [
|
||||
"old branch",
|
||||
"old reply",
|
||||
]
|
||||
|
||||
|
||||
def test_frozen_current_still_revalidates_history_after_the_model_call(tmp_path) -> None:
|
||||
from one_memory.hy_source_context import HySourceContext
|
||||
|
||||
stores = _setup(tmp_path)
|
||||
evidence, _profiles, tasks, _governance, _secrets, _profile = stores
|
||||
current = _put(
|
||||
evidence,
|
||||
message_id="frozen-current",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="old current",
|
||||
)
|
||||
history = _put(
|
||||
evidence,
|
||||
message_id="frozen-history",
|
||||
role="assistant",
|
||||
written_by=WrittenBy.AGENT,
|
||||
content="stable history",
|
||||
)
|
||||
_put(
|
||||
evidence,
|
||||
message_id="frozen-current",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="new current",
|
||||
)
|
||||
runner = _runner(tmp_path, stores, FakeBridge(outputs=()))
|
||||
context = HySourceContext(current=(current,), history=(history,))
|
||||
|
||||
runner._check_source_context(context, skip_current=True)
|
||||
with sqlite3.connect(tasks.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"DELETE FROM evidence WHERE evidence_id=?",
|
||||
(history.evidence_id,),
|
||||
)
|
||||
|
||||
with pytest.raises(HyMemoryBridgeError, match="hy_memory_evidence_changed"):
|
||||
runner._check_source_context(context, skip_current=True)
|
||||
|
||||
|
||||
def test_oversized_assistant_segments_keep_the_question_as_context(tmp_path) -> None:
|
||||
evidence = EvidenceStore(tmp_path / "one-memory.sqlite3")
|
||||
question = _put(
|
||||
@@ -2514,105 +2378,6 @@ def test_unknown_task_is_reconciled_from_exact_hy_output_without_model_retry(tmp
|
||||
assert bridge.closed is True
|
||||
|
||||
|
||||
def test_an_unknown_task_without_inputs_does_not_end_the_reconcile_pass(tmp_path) -> None:
|
||||
"""One task with nothing to send must not strand the other unknown tasks.
|
||||
|
||||
An unknown task whose inputs are gone yields an empty chunk, and building a
|
||||
request from that reached `chunk[0]` and came out as IndexError. A
|
||||
reconcile pass skips a task that raises a domain error, so IndexError did
|
||||
not skip anything -- it ended the pass. On this machine 68 of 303 unknown
|
||||
tasks had no inputs, and while the retry backed off to 128 and gave up, the
|
||||
235 that could have been reconciled stayed unknown with them.
|
||||
"""
|
||||
stores = _setup(tmp_path)
|
||||
evidence, profiles, tasks, governance, secrets, profile = stores
|
||||
stranded = _put(
|
||||
evidence,
|
||||
message_id="reconcile-without-inputs",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="这条任务的输入已经不在了。",
|
||||
)
|
||||
record = _put(
|
||||
evidence,
|
||||
message_id="reconcile-with-inputs",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
content="用户偏好每周日上午攀岩。",
|
||||
)
|
||||
empty = tasks.create(
|
||||
request_id="reconcile-empty-request",
|
||||
user_id=stranded.user_id,
|
||||
kind=MemoryTaskKind.EXTRACT,
|
||||
input_ref=stranded.evidence_id,
|
||||
profile=profile,
|
||||
input_records=(stranded,),
|
||||
)
|
||||
task = tasks.create(
|
||||
request_id="reconcile-kept-request",
|
||||
user_id=record.user_id,
|
||||
kind=MemoryTaskKind.EXTRACT,
|
||||
input_ref=record.evidence_id,
|
||||
profile=profile,
|
||||
input_records=(record,),
|
||||
)
|
||||
with sqlite3.connect(tasks.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE source_grants (
|
||||
user_id TEXT NOT NULL, source_id TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL, candidate_kind TEXT NOT NULL,
|
||||
granted_at TEXT NOT NULL, revoked_at TEXT,
|
||||
PRIMARY KEY (user_id, source_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO source_grants VALUES (?,?,?,?,?,NULL)",
|
||||
(
|
||||
record.user_id,
|
||||
record.source_id,
|
||||
"/authorized",
|
||||
"default",
|
||||
datetime.now(UTC).isoformat(),
|
||||
),
|
||||
)
|
||||
for task_id in (empty.task_id, task.task_id):
|
||||
connection.execute(
|
||||
"UPDATE memory_tasks SET status='unknown', last_error_code=? WHERE task_id=?",
|
||||
("model_content_outcome_unknown", task_id),
|
||||
)
|
||||
# What the field data actually looks like: the task is there, its
|
||||
# inputs are not.
|
||||
connection.execute("DELETE FROM memory_task_inputs WHERE task_id=?", (empty.task_id,))
|
||||
bridge = FakeBridge(
|
||||
outputs=(
|
||||
HyMemoryOutput(
|
||||
memory_id="hy-recovered",
|
||||
content="用户偏好每周日上午攀岩。",
|
||||
layer="L2_FACT",
|
||||
owner="user",
|
||||
evidence_ids=(record.evidence_id,),
|
||||
),
|
||||
)
|
||||
)
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=tasks,
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=evidence,
|
||||
governance=governance,
|
||||
data_dir=tmp_path,
|
||||
runner_factory=lambda: _runner(tmp_path, stores, FakeBridge()),
|
||||
bridge_factory=lambda _profile, _secret, _data_dir: bridge,
|
||||
)
|
||||
|
||||
# The task with inputs is reconciled; the one without is passed over.
|
||||
assert asyncio.run(supervisor.reconcile_unknown_outputs()) == 1
|
||||
assert tasks.require(task.task_id).status is MemoryTaskStatus.COMPLETED
|
||||
assert [memory.memory_id for memory in governance.list(user_id="local")] == ["hy-recovered"]
|
||||
|
||||
|
||||
def test_two_workspaces_with_one_conversation_id_prepare_apart(tmp_path) -> None:
|
||||
"""Cursor can reuse a conversation id across projects.
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ def test_sync_config_defaults_off_and_survives_restart(tmp_path):
|
||||
saved = json.loads((tmp_path / "config.json").read_text())
|
||||
assert saved == {
|
||||
"schema_version": 1,
|
||||
"sedimentation_enabled": True,
|
||||
"sources": {"cursor": {"sync_enabled": True}},
|
||||
}
|
||||
assert ConfigStore(tmp_path).sync_enabled("cursor")
|
||||
@@ -47,7 +46,6 @@ def test_legacy_model_content_permission_is_ignored_on_upgrade(tmp_path):
|
||||
store.set_sync("codex", True)
|
||||
assert json.loads((tmp_path / "config.json").read_text()) == {
|
||||
"schema_version": 1,
|
||||
"sedimentation_enabled": True,
|
||||
"sources": {
|
||||
"codex": {"sync_enabled": True},
|
||||
"cursor": {"sync_enabled": True},
|
||||
@@ -59,35 +57,3 @@ def test_invalid_config_fails_loudly_instead_of_silently_disabling_sources(tmp_p
|
||||
(tmp_path / "config.json").write_text('{"schema_version":1,"sources":{"cursor":{}}')
|
||||
with pytest.raises(ConfigFileError, match="config_invalid"):
|
||||
ConfigStore(tmp_path)
|
||||
|
||||
|
||||
def test_sedimentation_defaults_on_and_survives_restart(tmp_path):
|
||||
assert ConfigStore(tmp_path).sedimentation_enabled()
|
||||
|
||||
store = ConfigStore(tmp_path)
|
||||
store.set_sedimentation_enabled(False)
|
||||
assert not store.sedimentation_enabled()
|
||||
|
||||
# The model configuration is a separate store, so stopping must not touch
|
||||
# it: starting again has to be a switch, not a re-entry of a key.
|
||||
reopened = ConfigStore(tmp_path)
|
||||
assert not reopened.sedimentation_enabled()
|
||||
reopened.set_sedimentation_enabled(True)
|
||||
assert ConfigStore(tmp_path).sedimentation_enabled()
|
||||
|
||||
|
||||
def test_stopping_sedimentation_keeps_source_sync_config(tmp_path):
|
||||
store = ConfigStore(tmp_path)
|
||||
store.set_sync("cursor", True)
|
||||
store.set_sedimentation_enabled(False)
|
||||
|
||||
reopened = ConfigStore(tmp_path)
|
||||
assert not reopened.sedimentation_enabled()
|
||||
assert reopened.sync_enabled("cursor")
|
||||
|
||||
|
||||
def test_sedimentation_rejects_a_non_boolean(tmp_path):
|
||||
store = ConfigStore(tmp_path)
|
||||
with pytest.raises(ValueError, match="invalid_sedimentation_config"):
|
||||
store.set_sedimentation_enabled("yes") # type: ignore[arg-type]
|
||||
assert store.sedimentation_enabled()
|
||||
@@ -125,7 +125,8 @@ def test_write_job_state_covers_unknown_and_terminal() -> None:
|
||||
} == {state for state in WriteJobState if not state.is_terminal}
|
||||
|
||||
|
||||
def test_canonical_message_requires_a_nonnegative_ordinal() -> None:
|
||||
def test_canonical_message_requires_author_and_ordinal() -> None:
|
||||
"""A message without a known author or position cannot be ordered later."""
|
||||
with pytest.raises(ValueError):
|
||||
CanonicalMessage(
|
||||
harness_id="cursor",
|
||||
@@ -133,5 +134,6 @@ def test_canonical_message_requires_a_nonnegative_ordinal() -> None:
|
||||
message_id=None,
|
||||
ordinal=-1,
|
||||
role="user",
|
||||
written_by="person",
|
||||
content="hello",
|
||||
)
|
||||
@@ -10,12 +10,10 @@ import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.database_access import main_database
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridgeError
|
||||
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
|
||||
from one_memory.memory_server.durable_queue import (
|
||||
CANCELLED,
|
||||
@@ -39,18 +37,14 @@ def _request(
|
||||
evidence: tuple[str, ...] = ("e1",),
|
||||
key: str = "idem-1",
|
||||
source: str | None = None,
|
||||
session: str | None = None,
|
||||
event_at: datetime | None = None,
|
||||
) -> WriteRequest:
|
||||
session_id = session or f"sess-{source or '1'}"
|
||||
batch = BatchIdentity(internal_session_id=session_id, first_round=1, last_round=5)
|
||||
batch = BatchIdentity(internal_session_id="sess-1", first_round=1, last_round=5)
|
||||
return WriteRequest(
|
||||
internal_session_id=session_id,
|
||||
internal_session_id="sess-1",
|
||||
batch_identity=batch,
|
||||
evidence_ids=list(evidence),
|
||||
idempotency_key=key,
|
||||
source_id=source,
|
||||
event_at=event_at,
|
||||
)
|
||||
|
||||
|
||||
@@ -60,19 +54,11 @@ def _rows(path):
|
||||
return [dict(row) for row in db.execute("SELECT * FROM memory_write_jobs")]
|
||||
|
||||
|
||||
def test_acceptance_is_written_to_the_business_database(tmp_path, monkeypatch) -> None:
|
||||
from one_memory.memory_server import durable_queue
|
||||
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
durable_queue,
|
||||
"diagnostic_event",
|
||||
lambda **event: events.append(event),
|
||||
)
|
||||
def test_acceptance_is_written_to_the_business_database(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
|
||||
record = queue.submit(_request(source="cursor"))
|
||||
record = queue.submit(_request())
|
||||
|
||||
rows = _rows(path)
|
||||
assert len(rows) == 1
|
||||
@@ -81,24 +67,15 @@ def test_acceptance_is_written_to_the_business_database(tmp_path, monkeypatch) -
|
||||
assert rows[0]["state"] == "pending"
|
||||
assert rows[0]["hy_invoked"] == 0
|
||||
assert record.enqueued is True
|
||||
assert events == [
|
||||
{
|
||||
"level": "info",
|
||||
"component": "memory.write_queue",
|
||||
"event": "memory_write_job.queued",
|
||||
"outcome": "queued",
|
||||
"job_id": record.job_id,
|
||||
"details": {
|
||||
"source_id": "cursor",
|
||||
"mode": "normal",
|
||||
"count": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_an_isolated_legacy_queue_keeps_the_empty_snapshot_compatibility_shape(tmp_path) -> None:
|
||||
"""Queues without an Evidence authority exist only for legacy/unit compatibility."""
|
||||
def test_the_stored_payload_is_references_only(tmp_path) -> None:
|
||||
"""The job payload must point at Evidence, never carry its text.
|
||||
|
||||
A queue row is durable and much longer-lived than the API call that made
|
||||
it, so storing verbatim content here would be a second copy of the user's
|
||||
private text. The payload is the request's identifiers only.
|
||||
"""
|
||||
import json
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
@@ -110,12 +87,10 @@ def test_an_isolated_legacy_queue_keeps_the_empty_snapshot_compatibility_shape(t
|
||||
"internal_session_id",
|
||||
"batch_identity",
|
||||
"evidence_ids",
|
||||
"input_records",
|
||||
"idempotency_key",
|
||||
"priority",
|
||||
"source_id",
|
||||
}
|
||||
assert payload["input_records"] == []
|
||||
assert payload["evidence_ids"] == ["ev-1", "ev-2"]
|
||||
|
||||
|
||||
@@ -152,7 +127,6 @@ def test_submit_accepts_a_write_whose_evidence_exists(tmp_path) -> None:
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
occurred_at = datetime(2026, 10, 17, 23, 45, tzinfo=UTC)
|
||||
created = store.put(
|
||||
EvidenceInput(
|
||||
source_id="test",
|
||||
@@ -160,7 +134,6 @@ def test_submit_accepts_a_write_whose_evidence_exists(tmp_path) -> None:
|
||||
message_id="m1",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=occurred_at,
|
||||
verbatim="present evidence",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
@@ -171,8 +144,6 @@ def test_submit_accepts_a_write_whose_evidence_exists(tmp_path) -> None:
|
||||
|
||||
assert record.enqueued is True
|
||||
assert len(_rows(path)) == 1
|
||||
assert datetime.fromisoformat(_rows(path)[0]["event_at"]) == occurred_at
|
||||
assert _rows(path)[0]["time_quality"] == "exact"
|
||||
|
||||
|
||||
def test_a_replay_is_idempotent_even_after_its_evidence_is_removed(tmp_path) -> None:
|
||||
@@ -216,17 +187,20 @@ def test_a_replay_is_idempotent_even_after_its_evidence_is_removed(tmp_path) ->
|
||||
assert replay.enqueued is False
|
||||
assert len(_rows(path)) == 1
|
||||
|
||||
|
||||
def test_a_settled_job_replays_as_a_hit_not_a_conflict(tmp_path) -> None:
|
||||
"""A historical reaped payload matches only its exact pre-snapshot intent."""
|
||||
"""Replaying a completed key must return the job, never raise.
|
||||
|
||||
Real trigger: 236 startup replays of one key, all
|
||||
`ValueError: idempotency key ... already exists with different content`.
|
||||
The stored payload had been reaped to a digest sentinel, and the caller's
|
||||
key is derived from the evidence's revision, which can move on between
|
||||
replays while the payload's evidence ids stay put. Refusing that pair
|
||||
turned a normal startup replay into an exception every time.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
request = _request(evidence=("e1",), key="idem-settled")
|
||||
first = queue.submit(request)
|
||||
legacy = request.model_dump(
|
||||
mode="json",
|
||||
exclude={"event_at", "time_quality", "input_records"},
|
||||
)
|
||||
digest = sha256(json.dumps(legacy, sort_keys=True).encode()).hexdigest()
|
||||
first = queue.submit(_request(evidence=("e1",), key="idem-settled"))
|
||||
|
||||
# Settle it the way the drain does: terminal state, HY invoked, and the
|
||||
# payload reaped down to its digest so no body is kept.
|
||||
@@ -235,40 +209,19 @@ def test_a_settled_job_replays_as_a_hit_not_a_conflict(tmp_path) -> None:
|
||||
"UPDATE memory_write_jobs SET state=?, hy_invoked=1, payload_json=? WHERE job_id=?",
|
||||
(
|
||||
COMPLETED,
|
||||
json.dumps({"_one_memory_reaped_payload": digest}),
|
||||
json.dumps({"_one_memory_reaped_payload": "0" * 64}),
|
||||
first.job_id,
|
||||
),
|
||||
)
|
||||
|
||||
replay = queue.submit(request)
|
||||
# A payload that cannot match the reaped digest, under the same key.
|
||||
replay = queue.submit(_request(evidence=("e1",), key="idem-settled"))
|
||||
|
||||
assert replay.job_id == first.job_id
|
||||
assert replay.state == COMPLETED
|
||||
assert replay.enqueued is False
|
||||
assert len(_rows(path)) == 1
|
||||
|
||||
with pytest.raises(ValueError, match="different content"):
|
||||
queue.submit(
|
||||
_request(
|
||||
evidence=("e1",),
|
||||
key="idem-settled",
|
||||
source="cursor",
|
||||
session="different-session",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_a_real_completed_job_still_rejects_the_same_key_with_different_input(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
first = queue.submit(_request(evidence=("e1",), key="completed-conflict"))
|
||||
queue.settle(first.job_id, state=COMPLETED, hy_invoked=True)
|
||||
|
||||
with pytest.raises(ValueError, match="different content"):
|
||||
queue.submit(_request(evidence=("e2",), key="completed-conflict"))
|
||||
|
||||
|
||||
def test_a_killed_worker_leaves_a_possibly_sent_job_unknown_after_restart(tmp_path) -> None:
|
||||
"""A real process death after HY was invoked must not become retryable.
|
||||
@@ -509,8 +462,6 @@ def test_an_older_queue_backfills_evidence_refs_from_the_intact_payload(tmp_path
|
||||
queue = DurableWriteQueue(path)
|
||||
|
||||
assert _rows(path)[0]["evidence_ids_json"] == '["legacy-evidence"]'
|
||||
assert _rows(path)[0]["event_at"] == "2026-09-01T00:00:00+00:00"
|
||||
assert _rows(path)[0]["time_quality"] == "approximate"
|
||||
assert queue.unsubmitted_evidence_ids() == ()
|
||||
|
||||
|
||||
@@ -1162,13 +1113,10 @@ def test_unresolved_unknown_is_automatically_retried_only_once(tmp_path) -> None
|
||||
first_retry = queue.get(record.request_id)
|
||||
assert first_retry.state == PENDING
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0]
|
||||
== 1
|
||||
)
|
||||
assert connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0] == 1
|
||||
|
||||
drain = DurableWriteDrain(queue, execute=_never_execute, owner="retry-owner")
|
||||
assert drain.claim(limit=1) == (record.job_id,)
|
||||
@@ -1182,13 +1130,10 @@ def test_unresolved_unknown_is_automatically_retried_only_once(tmp_path) -> None
|
||||
)
|
||||
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert (
|
||||
connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0]
|
||||
== 2
|
||||
)
|
||||
assert connection.execute(
|
||||
"SELECT attempts FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()[0] == 2
|
||||
assert (
|
||||
queue.retry_first_unknown_after_reconcile(
|
||||
older_than=datetime.now(UTC) + timedelta(minutes=1)
|
||||
@@ -2565,16 +2510,21 @@ def test_settle_with_an_owner_does_not_touch_an_ownerless_row(tmp_path, caplog)
|
||||
assert not any("durable_write_settle_missed_row" in message for message in messages), messages
|
||||
|
||||
|
||||
def test_terminal_payloads_are_retained_and_replay_still_detects_conflicts(tmp_path) -> None:
|
||||
"""Terminal input is immutable audit data and idempotency remains strict."""
|
||||
def test_terminal_payloads_are_reaped_but_replay_still_detects_conflicts(tmp_path) -> None:
|
||||
"""Payload cleanup must not weaken idempotency.
|
||||
|
||||
Once a job is terminal the serialized request is dead weight, so it is
|
||||
replaced by a digest. Replaying the same intent must still resolve to the
|
||||
same row, and the same key with different content must still be refused.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(evidence=("e1",)))
|
||||
queue.settle(record.job_id, state="completed")
|
||||
|
||||
assert queue.clear_terminal_payloads() == 0
|
||||
assert queue.clear_terminal_payloads() == 1
|
||||
stored = _rows(path)[0]["payload_json"]
|
||||
assert "e1" in stored and "_one_memory_reaped_payload" not in stored
|
||||
assert "e1" not in stored and "_one_memory_reaped_payload" in stored
|
||||
assert _rows(path)[0]["evidence_ids_json"] == '["e1"]'
|
||||
|
||||
# Same intent still resolves to the existing row, not a second write.
|
||||
@@ -2588,15 +2538,15 @@ def test_terminal_payloads_are_retained_and_replay_still_detects_conflicts(tmp_p
|
||||
queue.submit(_request(evidence=("e2",)))
|
||||
|
||||
|
||||
def test_terminal_payload_cleanup_is_a_compatibility_noop(tmp_path) -> None:
|
||||
def test_terminal_payload_reaping_is_bounded_and_idempotent(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
for index in range(3):
|
||||
record = queue.submit(_request(evidence=("e1",), key=f"idem-{index}"))
|
||||
queue.settle(record.job_id, state="dead")
|
||||
|
||||
assert queue.clear_terminal_payloads(limit=2) == 0
|
||||
assert queue.clear_terminal_payloads(limit=2) == 0
|
||||
assert queue.clear_terminal_payloads(limit=2) == 2
|
||||
assert queue.clear_terminal_payloads(limit=2) == 1
|
||||
assert queue.clear_terminal_payloads(limit=2) == 0
|
||||
|
||||
|
||||
@@ -2615,8 +2565,8 @@ def test_unknown_and_pending_payloads_are_never_reaped(tmp_path) -> None:
|
||||
assert "e2" in payloads[unknown.request_id]
|
||||
|
||||
|
||||
def test_retained_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_path) -> None:
|
||||
"""A completed Evidence remains queued while its full input is retained.
|
||||
def test_reaped_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tmp_path) -> None:
|
||||
"""Payload cleanup must not make a completed Evidence look unqueued.
|
||||
|
||||
The reconcile primitive scans the durable queue by Evidence id. If it
|
||||
derived that set from `payload_json`, reaping a terminal row would erase
|
||||
@@ -2650,7 +2600,7 @@ def test_retained_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tm
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(evidence=(evidence.evidence_id,), key="idem-reaped"))
|
||||
queue.settle(record.job_id, state="completed")
|
||||
assert queue.clear_terminal_payloads() == 0
|
||||
assert queue.clear_terminal_payloads() == 1
|
||||
assert queue.unsubmitted_evidence_ids() == ()
|
||||
|
||||
orphan = store.put(
|
||||
@@ -2668,17 +2618,19 @@ def test_retained_terminal_payload_does_not_reappear_as_an_unsubmitted_orphan(tm
|
||||
assert queue.unsubmitted_evidence_ids() == (orphan.evidence_id,)
|
||||
|
||||
|
||||
def test_payload_for_a_terminal_row_keeps_the_immutable_input(tmp_path) -> None:
|
||||
def test_payload_for_a_reaped_terminal_row_returns_none(tmp_path) -> None:
|
||||
"""A stale claim of a reaped row must not resurrect a dead intent."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request())
|
||||
queue.settle(record.job_id, state="completed")
|
||||
queue.clear_terminal_payloads()
|
||||
|
||||
assert queue.payload_for(record.job_id) == _request()
|
||||
assert queue.payload_for(record.job_id) is None
|
||||
|
||||
|
||||
def test_drain_retains_terminal_payloads_when_idle(tmp_path) -> None:
|
||||
def test_drain_reaps_terminal_payloads_when_idle(tmp_path) -> None:
|
||||
"""The reaper must actually run in the worker, not only in tests."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request())
|
||||
@@ -2703,11 +2655,19 @@ def test_drain_retains_terminal_payloads_when_idle(tmp_path) -> None:
|
||||
asyncio.run(scenario())
|
||||
|
||||
assert executed == []
|
||||
assert "_one_memory_reaped_payload" not in _rows(path)[0]["payload_json"]
|
||||
assert "_one_memory_reaped_payload" in _rows(path)[0]["payload_json"]
|
||||
|
||||
|
||||
def test_the_drain_retains_terminal_payloads_while_never_idle(tmp_path) -> None:
|
||||
"""A sustained backlog must not mutate an earlier job's input snapshot."""
|
||||
def test_the_drain_reaps_terminal_payloads_even_while_never_idle(tmp_path) -> None:
|
||||
"""Terminal payloads must be reaped even if the queue never goes idle.
|
||||
|
||||
Regression: `clear_terminal_payloads` ran only on the `executed == 0` idle
|
||||
branch, so a sustained backlog kept every finished row's full serialized
|
||||
request body (which holds the written evidence references) in the business
|
||||
database indefinitely — a disk/privacy-retention invariant starved exactly
|
||||
when the queue is busiest. Reaping is a bounded maintenance write and must
|
||||
be gated by its own interval, not by idleness.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
finished = queue.submit(_request(key="finished"))
|
||||
@@ -2735,8 +2695,9 @@ def test_the_drain_retains_terminal_payloads_while_never_idle(tmp_path) -> None:
|
||||
|
||||
payload = _rows(path)
|
||||
by_key = {row["idempotency_key"]: row["payload_json"] for row in payload}
|
||||
assert "_one_memory_reaped_payload" not in by_key["finished"]
|
||||
assert '"e1"' in by_key["finished"]
|
||||
assert "_one_memory_reaped_payload" in by_key["finished"], (
|
||||
"a completed row's payload must be reaped even under a standing backlog"
|
||||
)
|
||||
|
||||
|
||||
def test_the_drain_readmits_model_waiters_even_while_never_idle(tmp_path) -> None:
|
||||
@@ -3185,8 +3146,8 @@ def test_claim_budget_ab_against_a_held_writer(tmp_path) -> None:
|
||||
def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> None:
|
||||
"""An empty queue must not issue a write per idle poll.
|
||||
|
||||
`readmit_model_waiters` writes. On an empty queue the loop wakes every
|
||||
`idle_pause`; running it each time would
|
||||
`clear_terminal_payloads` and `readmit_model_waiters` both write. On an
|
||||
empty queue the loop wakes every `idle_pause`; running them each time would
|
||||
contend with imports for the single writer for no benefit. They share a
|
||||
coarse interval gate, so a 20-poll window performs at most a couple of
|
||||
writes.
|
||||
@@ -3195,14 +3156,22 @@ def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> No
|
||||
queue = DurableWriteQueue(path)
|
||||
drain = DurableWriteDrain(queue, execute=_never_execute)
|
||||
|
||||
reaps = 0
|
||||
readmits = 0
|
||||
real_reap = queue.clear_terminal_payloads
|
||||
real_readmit = queue.readmit_model_waiters
|
||||
|
||||
def counted_reap(*args, **kwargs):
|
||||
nonlocal reaps
|
||||
reaps += 1
|
||||
return real_reap(*args, **kwargs)
|
||||
|
||||
def counted_readmit(*args, **kwargs):
|
||||
nonlocal readmits
|
||||
readmits += 1
|
||||
return real_readmit(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(queue, "clear_terminal_payloads", counted_reap)
|
||||
monkeypatch.setattr(queue, "readmit_model_waiters", counted_readmit)
|
||||
monkeypatch.setattr(drain, "claim", lambda **_kw: ())
|
||||
|
||||
@@ -3216,6 +3185,7 @@ def test_idle_maintenance_does_not_write_every_poll(tmp_path, monkeypatch) -> No
|
||||
|
||||
# Well over 20 polls happened; maintenance ran at most once (the interval
|
||||
# gate), not once per poll.
|
||||
assert reaps <= 1
|
||||
assert readmits <= 1
|
||||
|
||||
|
||||
@@ -3346,7 +3316,7 @@ def test_idle_maintenance_database_failure_is_sampled_without_database_error(
|
||||
def locked(*_args, **_kwargs):
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
|
||||
monkeypatch.setattr(drain.queue, "readmit_model_waiters", locked)
|
||||
monkeypatch.setattr(drain.queue, "clear_terminal_payloads", locked)
|
||||
monkeypatch.setattr(drain, "claim", lambda **_kw: ())
|
||||
|
||||
async def scenario() -> None:
|
||||
@@ -3362,8 +3332,8 @@ def test_idle_maintenance_database_failure_is_sampled_without_database_error(
|
||||
def programming_error(*_args, **_kwargs):
|
||||
raise RuntimeError("maintenance bug")
|
||||
|
||||
monkeypatch.setattr(drain.queue, "readmit_model_waiters", programming_error)
|
||||
drain._last_model_readmit = None # noqa: SLF001 - force the next maintenance tick
|
||||
monkeypatch.setattr(drain.queue, "clear_terminal_payloads", programming_error)
|
||||
drain._last_terminal_reap = None # noqa: SLF001 - force the next maintenance tick
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
@@ -4217,42 +4187,6 @@ def test_disabled_source_is_not_claimed_until_reenabled(tmp_path) -> None:
|
||||
assert drain.claim(limit=1) == (record.job_id,)
|
||||
|
||||
|
||||
def test_a_source_admitted_only_by_grant_is_claimed(tmp_path) -> None:
|
||||
"""The claim gate must ask the question the write gate asks.
|
||||
|
||||
`one-memory-manual` is not a connected app: it never appears in the
|
||||
configuration document, so a per-app toggle answers False for it while the
|
||||
write gate admits it by its grant. Wired as the drain's claim gate, that
|
||||
second answer skipped the job on every tick — a saved manual note stayed
|
||||
pending with no task behind it and never reached the model. Admission is
|
||||
one decision, and it has one predicate.
|
||||
"""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="manual-note", source="one-memory-manual"))
|
||||
# What `sync_enabled` answers for this source: no entry, therefore not on.
|
||||
per_app_toggle: dict[str, bool] = {}
|
||||
granted = {"one-memory-manual"}
|
||||
|
||||
drained = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
source_writes_permitted=lambda source_id: source_id in granted,
|
||||
source_claim_enabled=lambda source_id: source_id in granted,
|
||||
)
|
||||
assert drained.claim(limit=1) == (record.job_id,)
|
||||
|
||||
# The wiring that shipped: admitted for writes, refused at claim.
|
||||
queue.settle(record.job_id, state=PENDING, owner=None)
|
||||
stranded = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
source_writes_permitted=lambda source_id: source_id in granted,
|
||||
source_claim_enabled=lambda source_id: per_app_toggle.get(source_id, False),
|
||||
)
|
||||
assert stranded.claim(limit=1) == ()
|
||||
|
||||
|
||||
def test_pause_before_hy_releases_an_already_claimed_job(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
@@ -4276,130 +4210,9 @@ def test_pause_before_hy_releases_an_already_claimed_job(tmp_path) -> None:
|
||||
assert current.state == PENDING
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT attempts,hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
"SELECT hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone() == (0, 0)
|
||||
|
||||
|
||||
def test_event_time_window_claims_cross_harness_jobs_within_ten_minutes(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
first = queue.submit(_request(key="window-a", source="cursor", event_at=anchor))
|
||||
inside = queue.submit(
|
||||
_request(
|
||||
key="window-b",
|
||||
source="codex",
|
||||
event_at=anchor + timedelta(minutes=9, seconds=59),
|
||||
)
|
||||
)
|
||||
boundary = queue.submit(
|
||||
_request(
|
||||
key="window-c",
|
||||
source="claude-code",
|
||||
event_at=anchor + timedelta(minutes=10),
|
||||
)
|
||||
)
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="window-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (first.job_id, inside.job_id)
|
||||
queue.settle(
|
||||
first.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="window-owner",
|
||||
)
|
||||
queue.settle(
|
||||
inside.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="window-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (boundary.job_id,)
|
||||
|
||||
|
||||
def test_event_time_window_anchor_does_not_slide_while_a_job_is_running(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
first = queue.submit(_request(key="fixed-a", source="cursor", event_at=anchor))
|
||||
inside = queue.submit(
|
||||
_request(
|
||||
key="fixed-b",
|
||||
source="codex",
|
||||
event_at=anchor + timedelta(minutes=9),
|
||||
)
|
||||
)
|
||||
chained = queue.submit(
|
||||
_request(
|
||||
key="fixed-c",
|
||||
source="claude-code",
|
||||
event_at=anchor + timedelta(minutes=18),
|
||||
)
|
||||
)
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="fixed-window-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=2) == (first.job_id, inside.job_id)
|
||||
queue.settle(
|
||||
first.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="fixed-window-owner",
|
||||
)
|
||||
assert drain.claim(limit=1) == ()
|
||||
assert queue.get(chained.request_id).state == PENDING
|
||||
queue.settle(
|
||||
inside.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="fixed-window-owner",
|
||||
)
|
||||
assert drain.claim(limit=1) == (chained.job_id,)
|
||||
|
||||
|
||||
def test_sedimentation_pause_releases_claim_without_spending_an_attempt(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="sedimentation-pause", source="cursor"))
|
||||
drain = DurableWriteDrain(queue, execute=_never_execute, owner="pause-owner")
|
||||
assert drain.claim(limit=1) == (record.job_id,)
|
||||
|
||||
async def refuse_before_hy(_source_id, _mark_invoked):
|
||||
raise RuntimeError("model_sedimentation_disabled")
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
record.job_id,
|
||||
queue=queue,
|
||||
supervisor=_Recorder(invoke_hy=True),
|
||||
evidence=_EvidenceStore({"e1"}),
|
||||
owner="pause-owner",
|
||||
source_writes_permitted=lambda _source_id: True,
|
||||
admit_hy_invocation=refuse_before_hy,
|
||||
)
|
||||
)
|
||||
|
||||
current = queue.get(record.request_id)
|
||||
assert current is not None and current.state == PENDING
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT attempts,hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone() == (0, 0)
|
||||
).fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_pause_after_first_hy_call_does_not_interrupt_the_current_job(tmp_path) -> None:
|
||||
@@ -4808,170 +4621,3 @@ def test_readmit_paths_never_requeue_a_possibly_sent_row(tmp_path) -> None:
|
||||
assert rows["parked"]["state"] == WAITING_FOR_PERMISSION
|
||||
assert rows["model"]["state"] == WAITING_FOR_MODEL
|
||||
assert DurableWriteDrain(queue, execute=_never_execute).claim(limit=5) == ()
|
||||
|
||||
|
||||
def test_a_refused_source_context_is_nonblocking_unknown(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""An ambiguous call-boundary failure is retained but never blocks FIFO."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="refused-scope", source="opencode"))
|
||||
claimer = DurableWriteDrain(queue, execute=_never_execute, owner="refused-owner")
|
||||
assert claimer.claim(limit=1) == (record.job_id,)
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
record.job_id,
|
||||
queue=queue,
|
||||
supervisor=_Recorder(
|
||||
error=HyMemoryBridgeError("hy_memory_write_verification_context_scope"),
|
||||
invoke_hy=True,
|
||||
),
|
||||
evidence=_EvidenceStore({"e1"}),
|
||||
owner="refused-owner",
|
||||
)
|
||||
)
|
||||
|
||||
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
|
||||
state, invoked, last_error = db.execute(
|
||||
"SELECT state, hy_invoked, last_error FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()
|
||||
|
||||
assert state == UNKNOWN
|
||||
assert invoked == 1
|
||||
assert last_error == "hy_memory_write_verification_context_scope"
|
||||
assert DurableWriteDrain(queue, execute=_never_execute).claim(limit=1) == ()
|
||||
|
||||
|
||||
def test_a_failure_after_the_call_still_parks_as_unknown(tmp_path) -> None:
|
||||
"""The fix must not widen: a post-call refusal is still possibly-sent."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
record = queue.submit(_request(key="after-call", source="opencode"))
|
||||
claimer = DurableWriteDrain(queue, execute=_never_execute, owner="after-call-owner")
|
||||
assert claimer.claim(limit=1) == (record.job_id,)
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
record.job_id,
|
||||
queue=queue,
|
||||
supervisor=_Recorder(
|
||||
error=HyMemoryBridgeError("hy_memory_write_verification_authority_invalid"),
|
||||
invoke_hy=True,
|
||||
),
|
||||
evidence=_EvidenceStore({"e1"}),
|
||||
owner="after-call-owner",
|
||||
)
|
||||
)
|
||||
|
||||
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
|
||||
state, invoked = db.execute(
|
||||
"SELECT state, hy_invoked FROM memory_write_jobs WHERE job_id=?",
|
||||
(record.job_id,),
|
||||
).fetchone()
|
||||
|
||||
assert state == UNKNOWN
|
||||
assert invoked == 1
|
||||
|
||||
|
||||
def test_event_time_not_enqueue_order_drives_the_window(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""A later-enqueued old turn runs before an earlier-enqueued new turn."""
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
anchor = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
|
||||
enqueued_first = queue.submit(
|
||||
_request(
|
||||
key="newer-event",
|
||||
source="workbuddy",
|
||||
session="newer-session",
|
||||
event_at=anchor + timedelta(hours=1),
|
||||
)
|
||||
)
|
||||
enqueued_second = queue.submit(
|
||||
_request(
|
||||
key="older-event",
|
||||
source="workbuddy",
|
||||
session="older-session",
|
||||
event_at=anchor + timedelta(minutes=5),
|
||||
)
|
||||
)
|
||||
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="starve-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (enqueued_second.job_id,)
|
||||
queue.settle(
|
||||
enqueued_second.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="starve-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (enqueued_first.job_id,)
|
||||
|
||||
with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) as db:
|
||||
window = db.execute(
|
||||
"SELECT window_anchor FROM memory_write_schedule_state WHERE singleton=1"
|
||||
).fetchone()
|
||||
assert window is not None
|
||||
assert str(window[0]) == (anchor + timedelta(hours=1)).astimezone(UTC).isoformat()
|
||||
|
||||
|
||||
def test_same_session_uses_event_time_then_queue_sequence(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
base = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
later = queue.submit(
|
||||
_request(key="same-session-later", source="cursor", event_at=base + timedelta(hours=1))
|
||||
)
|
||||
earlier = queue.submit(_request(key="same-session-earlier", source="cursor", event_at=base))
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="event-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (earlier.job_id,)
|
||||
queue.settle(
|
||||
earlier.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="event-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (later.job_id,)
|
||||
|
||||
|
||||
def test_equal_event_time_uses_queue_sequence_inside_one_session(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
queue = DurableWriteQueue(path)
|
||||
event_at = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
first = queue.submit(
|
||||
_request(key="equal-first", source="cursor", event_at=event_at)
|
||||
)
|
||||
second = queue.submit(
|
||||
_request(key="equal-second", source="cursor", event_at=event_at)
|
||||
)
|
||||
drain = DurableWriteDrain(
|
||||
queue,
|
||||
execute=_never_execute,
|
||||
owner="equal-owner",
|
||||
max_concurrent_sources=10,
|
||||
)
|
||||
|
||||
assert drain.claim(limit=10) == (first.job_id,)
|
||||
queue.settle(
|
||||
first.job_id,
|
||||
state=COMPLETED,
|
||||
hy_invoked=True,
|
||||
owner="equal-owner",
|
||||
)
|
||||
assert drain.claim(limit=10) == (second.job_id,)
|
||||
@@ -2103,7 +2103,7 @@ def test_migration_49_resets_only_cursor_history_when_conversation_time_is_missi
|
||||
|
||||
result = asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
assert result.applied == (49, 50, 51, 52, 53, 54)
|
||||
assert result.applied == (49, 50, 51)
|
||||
with main_database(path).read() as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM source_cursors").fetchone()[0] == 0
|
||||
assert connection.execute("SELECT COUNT(*) FROM live_source_cursors").fetchone()[0] == 1
|
||||
@@ -2136,7 +2136,7 @@ def test_upgrade_drops_obsolete_hy_node_permissions_without_touching_memories(
|
||||
|
||||
result = asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
assert result.applied == (51, 52, 53, 54)
|
||||
assert result.applied == (51,)
|
||||
with sqlite3.connect(path) as connection:
|
||||
tables = {
|
||||
row[0]
|
||||
@@ -2147,125 +2147,3 @@ def test_upgrade_drops_obsolete_hy_node_permissions_without_touching_memories(
|
||||
assert "hy_node_permissions" not in tables
|
||||
assert "governed_memories" in tables
|
||||
assert "evidence" in tables
|
||||
|
||||
|
||||
def test_migration_53_backfills_write_job_event_time_from_person_turn(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
import sqlite3
|
||||
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore
|
||||
|
||||
path = tmp_path / "legacy-write-jobs.sqlite3"
|
||||
with sqlite3.connect(path) as connection:
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE memory_write_jobs (
|
||||
job_id TEXT PRIMARY KEY,
|
||||
queue_seq INTEGER NOT NULL,
|
||||
session_key TEXT NOT NULL,
|
||||
source_key TEXT NOT NULL,
|
||||
batch_id TEXT NOT NULL,
|
||||
idempotency_key TEXT NOT NULL UNIQUE,
|
||||
payload_json TEXT NOT NULL,
|
||||
evidence_ids_json TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
priority TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL,
|
||||
max_attempts INTEGER NOT NULL,
|
||||
lease_owner TEXT,
|
||||
lease_expires_at TEXT,
|
||||
hy_invoked INTEGER NOT NULL,
|
||||
last_error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE evidence (
|
||||
evidence_id TEXT PRIMARY KEY,
|
||||
role TEXT NOT NULL,
|
||||
written_by TEXT NOT NULL,
|
||||
created_at TEXT,
|
||||
imported_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE memory_write_job_evidence (
|
||||
job_id TEXT NOT NULL,
|
||||
evidence_id TEXT NOT NULL,
|
||||
PRIMARY KEY (job_id, evidence_id)
|
||||
);
|
||||
INSERT INTO memory_write_jobs VALUES (
|
||||
'job-1',1,'session','cursor','batch','idem','{}','[]',
|
||||
'pending','normal',0,3,NULL,NULL,0,NULL,
|
||||
'2026-10-18T08:00:00+00:00','2026-10-18T08:00:00+00:00'
|
||||
);
|
||||
INSERT INTO evidence VALUES (
|
||||
'user-1','user','person','2026-10-16T01:02:03+00:00',
|
||||
'2026-10-18T08:00:00+00:00'
|
||||
);
|
||||
INSERT INTO memory_write_job_evidence VALUES ('job-1','user-1');
|
||||
"""
|
||||
)
|
||||
|
||||
migration = next(item for item in MIGRATIONS if item.version == 53)
|
||||
result = asyncio.run(migrate(SqliteMigrationStore(path), (migration,)))
|
||||
|
||||
assert result.applied == (53,)
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT event_at,time_quality FROM memory_write_jobs"
|
||||
).fetchone() == ("2026-10-16T01:02:03+00:00", "approximate")
|
||||
assert connection.execute(
|
||||
"SELECT 1 FROM sqlite_master "
|
||||
"WHERE type='table' AND name='memory_write_schedule_state'"
|
||||
).fetchone() == (1,)
|
||||
|
||||
|
||||
def test_migration_54_preserves_legacy_possibly_sent_tasks(tmp_path) -> None:
|
||||
import sqlite3
|
||||
|
||||
from one_memory.memory_server.migration_runner import SqliteMigrationStore
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
through_53 = tuple(item for item in MIGRATIONS if item.version <= 53)
|
||||
asyncio.run(migrate(SqliteMigrationStore(path), through_53))
|
||||
with sqlite3.connect(path) as connection, connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO memory_tasks (
|
||||
task_id,request_id,user_id,kind,input_ref,status,
|
||||
profile_id,profile_revision,profile_snapshot_json,
|
||||
attempts,created_at,updated_at,hy_invocation_count
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,0,?,?,0)
|
||||
""",
|
||||
(
|
||||
"legacy-sent",
|
||||
"legacy-sent-request",
|
||||
"local",
|
||||
"extract",
|
||||
"evidence",
|
||||
"running",
|
||||
"profile",
|
||||
1,
|
||||
"{}",
|
||||
"2026-10-18T00:00:00+00:00",
|
||||
"2026-10-18T00:00:00+00:00",
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE memory_task_continuous_permissions "
|
||||
"(task_id TEXT PRIMARY KEY)"
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_task_continuous_permissions VALUES (?)",
|
||||
("legacy-sent",),
|
||||
)
|
||||
|
||||
asyncio.run(migrate(SqliteMigrationStore(path), MIGRATIONS))
|
||||
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute(
|
||||
"SELECT hy_invocation_count FROM memory_tasks WHERE task_id='legacy-sent'"
|
||||
).fetchone() == (1,)
|
||||
assert connection.execute(
|
||||
"SELECT 1 FROM sqlite_master "
|
||||
"WHERE type='table' AND name='memory_task_continuous_permissions'"
|
||||
).fetchone() is None
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
|
||||
@@ -11,23 +9,13 @@ import pytest
|
||||
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.durable_queue import (
|
||||
DurableWriteDrain,
|
||||
DurableWriteQueue,
|
||||
execute_job,
|
||||
)
|
||||
from one_memory.memory_server.durable_queue import DurableWriteQueue
|
||||
from one_memory.memory_server.unified_entry import (
|
||||
UnifiedWriteIntake,
|
||||
)
|
||||
|
||||
|
||||
def _record(
|
||||
store: EvidenceStore,
|
||||
*,
|
||||
conversation_id: str = "c1",
|
||||
message_id: str = "m1",
|
||||
verbatim: str = "durable intake",
|
||||
):
|
||||
def _record(store: EvidenceStore, *, conversation_id: str = "c1", message_id: str = "m1"):
|
||||
return store.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
@@ -36,7 +24,7 @@ def _record(
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=datetime.now(UTC),
|
||||
verbatim=verbatim,
|
||||
verbatim="durable intake",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
@@ -54,51 +42,6 @@ def test_evidence_submission_creates_one_durable_row(tmp_path) -> None:
|
||||
assert receipt.evidence_id == record.evidence_id
|
||||
assert receipt.request_id == f"entry:{record.evidence_id}"
|
||||
assert intake.queue.get(receipt.request_id) is not None
|
||||
with sqlite3.connect(path) as connection:
|
||||
event_at, quality = connection.execute(
|
||||
"SELECT event_at,time_quality FROM memory_write_jobs WHERE job_id=?",
|
||||
(receipt.record.job_id,),
|
||||
).fetchone()
|
||||
assert datetime.fromisoformat(event_at) == record.created_at
|
||||
assert quality == "exact"
|
||||
|
||||
|
||||
def test_cursor_turn_timestamp_orders_job_without_reordering_evidence(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
file_mtime = datetime(2026, 10, 18, 8, 0, tzinfo=UTC)
|
||||
turn_time = datetime(2026, 10, 16, 0, 24, tzinfo=UTC)
|
||||
record = store.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
conversation_id="cursor-session",
|
||||
message_id="cursor-user",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=file_mtime,
|
||||
verbatim="timestamped Cursor turn",
|
||||
origin_meta={
|
||||
EvidenceOrigin.FILE: OriginMetadata(
|
||||
raw_meta={
|
||||
"time_basis": "file_mtime",
|
||||
"turn_event_at": turn_time.isoformat(),
|
||||
}
|
||||
)
|
||||
},
|
||||
)
|
||||
).record
|
||||
intake = UnifiedWriteIntake(DurableWriteQueue(path, evidence=store))
|
||||
|
||||
receipt = intake.submit_evidence(record, first_round=1, last_round=1)
|
||||
|
||||
with sqlite3.connect(path) as connection:
|
||||
event_at, quality = connection.execute(
|
||||
"SELECT event_at,time_quality FROM memory_write_jobs WHERE job_id=?",
|
||||
(receipt.record.job_id,),
|
||||
).fetchone()
|
||||
assert record.created_at == file_mtime
|
||||
assert datetime.fromisoformat(event_at) == turn_time
|
||||
assert quality == "exact"
|
||||
|
||||
|
||||
def test_a_long_branch_conversation_id_still_enqueues(tmp_path) -> None:
|
||||
@@ -184,136 +127,6 @@ def test_replaying_the_same_evidence_does_not_create_a_second_row(tmp_path) -> N
|
||||
assert replay.enqueued is False
|
||||
|
||||
|
||||
def test_rewound_harness_revision_appends_a_job_with_immutable_input(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
intake = UnifiedWriteIntake(DurableWriteQueue(path, evidence=store))
|
||||
old = _record(store, verbatim="old branch")
|
||||
old_job = intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
|
||||
revised = _record(store, verbatim="new branch")
|
||||
new_job = intake.submit_evidence(revised, first_round=1, last_round=1)
|
||||
|
||||
assert revised.evidence_id == old.evidence_id
|
||||
assert revised.revision == old.revision + 1
|
||||
assert new_job.record.job_id != old_job.record.job_id
|
||||
assert intake.queue.payload_for(old_job.record.job_id).input_records[0].verbatim == "old branch"
|
||||
assert intake.queue.payload_for(new_job.record.job_id).input_records[0].verbatim == "new branch"
|
||||
assert intake.queue.unsubmitted_evidence_ids() == ()
|
||||
with sqlite3.connect(path) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM memory_write_jobs").fetchone()[0] == 2
|
||||
|
||||
|
||||
def test_revision_committed_before_job_crash_is_discovered_for_reconcile(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
intake = UnifiedWriteIntake(queue)
|
||||
old = _record(store, verbatim="old branch")
|
||||
intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
|
||||
revised = _record(store, verbatim="new branch")
|
||||
|
||||
assert revised.evidence_id == old.evidence_id
|
||||
assert revised.revision == old.revision + 1
|
||||
assert queue.unsubmitted_evidence_ids() == (revised.evidence_id,)
|
||||
|
||||
|
||||
def test_late_old_revision_snapshot_does_not_cover_the_current_revision(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
intake = UnifiedWriteIntake(queue)
|
||||
old = _record(store, verbatim="old branch")
|
||||
revised = _record(store, verbatim="new branch")
|
||||
|
||||
intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
|
||||
assert revised.evidence_id == old.evidence_id
|
||||
assert revised.revision == old.revision + 1
|
||||
assert queue.unsubmitted_evidence_ids() == (revised.evidence_id,)
|
||||
|
||||
|
||||
def test_old_branch_job_executes_its_frozen_revision_after_harness_rewind(tmp_path) -> None:
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
intake = UnifiedWriteIntake(queue)
|
||||
old = _record(store, verbatim="old branch")
|
||||
old_job = intake.submit_evidence(old, first_round=1, last_round=1)
|
||||
revised = _record(store, verbatim="new branch")
|
||||
intake.submit_evidence(revised, first_round=1, last_round=1)
|
||||
drain = DurableWriteDrain(queue, execute=lambda _job_id: asyncio.sleep(0), owner="frozen")
|
||||
assert drain.claim(limit=1) == (old_job.record.job_id,)
|
||||
|
||||
captured = []
|
||||
|
||||
class Supervisor:
|
||||
async def process_records(self, records, *, before_hy_invocation):
|
||||
captured.extend(records)
|
||||
await before_hy_invocation()
|
||||
|
||||
asyncio.run(
|
||||
execute_job(
|
||||
old_job.record.job_id,
|
||||
queue=queue,
|
||||
supervisor=Supervisor(),
|
||||
evidence=store,
|
||||
owner="frozen",
|
||||
)
|
||||
)
|
||||
|
||||
assert [record.verbatim for record in captured] == ["old branch"]
|
||||
assert queue.get(old_job.request_id).state == "completed"
|
||||
|
||||
|
||||
def test_reopening_reads_an_old_id_only_job_without_rewriting_it(tmp_path) -> None:
|
||||
from one_memory.memory_server.contracts import BatchIdentity, WriteRequest
|
||||
|
||||
path = tmp_path / "one-memory.sqlite3"
|
||||
store = EvidenceStore(path)
|
||||
record = _record(store)
|
||||
queue = DurableWriteQueue(path, evidence=store)
|
||||
legacy = queue.submit(
|
||||
WriteRequest(
|
||||
internal_session_id=record.conversation_id,
|
||||
source_id=record.source_id,
|
||||
batch_identity=BatchIdentity(
|
||||
internal_session_id=record.conversation_id,
|
||||
first_round=1,
|
||||
last_round=1,
|
||||
),
|
||||
evidence_ids=[record.evidence_id],
|
||||
idempotency_key="legacy-id-only",
|
||||
)
|
||||
)
|
||||
with sqlite3.connect(path) as connection, connection:
|
||||
payload = json.loads(
|
||||
connection.execute(
|
||||
"SELECT payload_json FROM memory_write_jobs WHERE job_id=?",
|
||||
(legacy.job_id,),
|
||||
).fetchone()[0]
|
||||
)
|
||||
payload.pop("input_records", None)
|
||||
connection.execute(
|
||||
"UPDATE memory_write_jobs SET payload_json=? WHERE job_id=?",
|
||||
(json.dumps(payload, sort_keys=True), legacy.job_id),
|
||||
)
|
||||
assert queue.payload_for(legacy.job_id).input_records == ()
|
||||
|
||||
reopened = DurableWriteQueue(path, evidence=store)
|
||||
|
||||
assert reopened.payload_for(legacy.job_id).input_records == ()
|
||||
with sqlite3.connect(path) as connection:
|
||||
persisted = json.loads(
|
||||
connection.execute(
|
||||
"SELECT payload_json FROM memory_write_jobs WHERE job_id=?",
|
||||
(legacy.job_id,),
|
||||
).fetchone()[0]
|
||||
)
|
||||
assert "input_records" not in persisted
|
||||
|
||||
|
||||
def test_committed_evidence_without_submit_is_the_red_state(tmp_path) -> None:
|
||||
"""The exact crash window the C0 wiring has to close.
|
||||
|
||||
|
||||
@@ -1500,8 +1500,14 @@ def test_restart_splits_a_sent_continuous_task_from_an_unsent_one(tmp_path) -> N
|
||||
input_ref="evidence-unsent",
|
||||
profile=profile,
|
||||
)
|
||||
_sent_task, sent_token = store.claim(sent.task_id)
|
||||
store.record_hy_invocation(sent.task_id, runner_token=sent_token)
|
||||
with sqlite3.connect(store.database_path) as connection, connection:
|
||||
connection.execute(
|
||||
"CREATE TABLE memory_task_continuous_permissions (task_id TEXT PRIMARY KEY)"
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO memory_task_continuous_permissions VALUES (?)", (sent.task_id,)
|
||||
)
|
||||
store.claim(sent.task_id)
|
||||
store.claim(unsent.task_id)
|
||||
|
||||
recovered = store.recover_running()
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from one_memory.model_request_audit import (
|
||||
AuditedModelTransport,
|
||||
ModelRequestAuditStore,
|
||||
audit_precipitation_request,
|
||||
)
|
||||
|
||||
|
||||
def _transport(tmp_path, response):
|
||||
store = ModelRequestAuditStore(tmp_path / "one-memory.sqlite3")
|
||||
return store, AuditedModelTransport(
|
||||
store,
|
||||
profile_id="profile-1",
|
||||
profile_revision=2,
|
||||
provider_kind="openai_compatible",
|
||||
model="deepseek-flash",
|
||||
inner=httpx.MockTransport(response),
|
||||
)
|
||||
|
||||
|
||||
def test_records_one_physical_request_with_provider_usage_and_no_content(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
private = "private conversation text"
|
||||
monkeypatch.setattr(
|
||||
"one_memory.model_request_audit._current_prompt_stage",
|
||||
lambda: "coding_judge",
|
||||
)
|
||||
|
||||
def response(_request):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "provider-response-1",
|
||||
"usage": {
|
||||
"prompt_tokens": 120,
|
||||
"completion_tokens": 30,
|
||||
"total_tokens": 150,
|
||||
"prompt_tokens_details": {"cached_tokens": 80},
|
||||
"completion_tokens_details": {"reasoning_tokens": 10},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
store, transport = _transport(tmp_path, response)
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
with audit_precipitation_request("memory-request-1"):
|
||||
result = await client.post(
|
||||
"https://example.test/chat/completions",
|
||||
json={"model": "deepseek-flash", "messages": [{"content": private}]},
|
||||
)
|
||||
assert result.status_code == 200
|
||||
assert result.json()["id"] == "provider-response-1"
|
||||
|
||||
asyncio.run(scenario())
|
||||
report = store.report(request_id="memory-request-1")
|
||||
assert report["summary"] == {
|
||||
"request_count": 1,
|
||||
"succeeded": 1,
|
||||
"failed": 0,
|
||||
"in_flight": 0,
|
||||
"prompt_tokens": 120,
|
||||
"completion_tokens": 30,
|
||||
"total_tokens": 150,
|
||||
"cached_prompt_tokens": 80,
|
||||
"reasoning_tokens": 10,
|
||||
"avg_prompt_tokens": 120.0,
|
||||
"avg_completion_tokens": 30.0,
|
||||
}
|
||||
row = report["requests"][0]
|
||||
assert row["provider_request_id"] == "provider-response-1"
|
||||
assert row["request_kind"] == "chat_completion"
|
||||
assert row["prompt_stage"] == "coding_judge"
|
||||
assert row["profile_revision"] == 2
|
||||
assert report["by_minute"][0]["prompt_stage"] == "coding_judge"
|
||||
assert report["by_minute"][0]["total_tokens"] == 150
|
||||
assert report["by_minute"][0]["avg_prompt_tokens"] == 120.0
|
||||
assert private.encode() not in store.database.read_bytes()
|
||||
|
||||
|
||||
def test_requests_outside_precipitation_are_not_recorded(tmp_path):
|
||||
store, transport = _transport(tmp_path, lambda _request: httpx.Response(200, json={}))
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
await client.post("https://example.test/chat/completions", json={})
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert store.report()["summary"]["request_count"] == 0
|
||||
|
||||
|
||||
def test_precipitation_context_reaches_the_sdk_worker_thread(tmp_path):
|
||||
store, transport = _transport(
|
||||
tmp_path,
|
||||
lambda _request: httpx.Response(
|
||||
200,
|
||||
json={"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}},
|
||||
),
|
||||
)
|
||||
|
||||
def worker():
|
||||
async def request():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
await client.post("https://example.test/chat/completions", json={})
|
||||
|
||||
asyncio.run(request())
|
||||
|
||||
async def scenario():
|
||||
with audit_precipitation_request("threaded-memory-request"):
|
||||
await asyncio.to_thread(worker)
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert store.report(request_id="threaded-memory-request")["summary"]["total_tokens"] == 3
|
||||
|
||||
|
||||
def test_failure_is_recorded_without_error_message(tmp_path):
|
||||
secret = "provider leaked private detail"
|
||||
|
||||
def response(_request):
|
||||
raise httpx.ConnectError(secret)
|
||||
|
||||
store, transport = _transport(tmp_path, response)
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
with audit_precipitation_request("memory-request-failed"):
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
await client.post("https://example.test/embeddings", json={})
|
||||
|
||||
asyncio.run(scenario())
|
||||
report = store.report(request_id="memory-request-failed")
|
||||
row = report["requests"][0]
|
||||
assert row["outcome"] == "failed"
|
||||
assert row["error_type"] == "ConnectError"
|
||||
assert row["request_kind"] == "embedding"
|
||||
assert secret.encode() not in store.database.read_bytes()
|
||||
|
||||
|
||||
def test_missing_usage_stays_unknown_in_detail_and_zero_in_summary(tmp_path):
|
||||
store, transport = _transport(
|
||||
tmp_path,
|
||||
lambda _request: httpx.Response(200, content=json.dumps({"id": "r"}).encode()),
|
||||
)
|
||||
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
with audit_precipitation_request("memory-request-no-usage"):
|
||||
await client.post("https://example.test/chat/completions", json={})
|
||||
|
||||
asyncio.run(scenario())
|
||||
report = store.report(request_id="memory-request-no-usage")
|
||||
assert report["requests"][0]["total_tokens"] is None
|
||||
assert report["summary"]["total_tokens"] == 0
|
||||
|
||||
|
||||
def test_store_schema_matches_migration_and_supports_parallel_starts(tmp_path):
|
||||
store = ModelRequestAuditStore(tmp_path / "one-memory.sqlite3")
|
||||
|
||||
async def start(index):
|
||||
return await asyncio.to_thread(
|
||||
store.start,
|
||||
request_id=f"request-{index}",
|
||||
profile_id="profile",
|
||||
profile_revision=1,
|
||||
provider_kind="openai_compatible",
|
||||
model="model",
|
||||
request_kind="chat_completion",
|
||||
prompt_stage="other",
|
||||
)
|
||||
|
||||
async def scenario():
|
||||
return await asyncio.gather(*(start(index) for index in range(8)))
|
||||
|
||||
ids = asyncio.run(scenario())
|
||||
assert len(set(ids)) == 8
|
||||
with sqlite3.connect(store.database) as connection:
|
||||
assert connection.execute("SELECT COUNT(*) FROM model_request_audit").fetchone() == (8,)
|
||||
@@ -3,12 +3,7 @@ import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
from one_memory.api import (
|
||||
_history_import_gate_released,
|
||||
_run_backfill,
|
||||
_sedimentation_claim_enabled,
|
||||
_source_writes_on,
|
||||
)
|
||||
from one_memory.api import _run_backfill
|
||||
from one_memory.imports.models import EvidenceInput, EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceStore
|
||||
from one_memory.memory_server.config_state import ConfigStore
|
||||
@@ -97,61 +92,3 @@ def test_source_pause_skips_history_until_the_source_is_reenabled(tmp_path) -> N
|
||||
asyncio.run(_run_backfill(app))
|
||||
|
||||
assert app.state.memory_write_queue.unsubmitted_evidence_ids() == ()
|
||||
|
||||
|
||||
def test_the_manual_source_is_admitted_by_grant_without_a_toggle(tmp_path) -> None:
|
||||
"""A source that is not a connected app has no toggle to read.
|
||||
|
||||
`one-memory-manual` is admitted by its grant and never appears in the
|
||||
configuration document, so any gate that reads the per-app toggle answers
|
||||
False for it while the write gate answers True. The drain's claim gate used
|
||||
to be that gate, and every manual note's job was skipped on every tick.
|
||||
"""
|
||||
config = ConfigStore(tmp_path)
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
config_store=config,
|
||||
source_grant_store=SimpleNamespace(
|
||||
receipt=lambda **_kwargs: SimpleNamespace(revoked_at=None)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert _source_writes_on(app, "one-memory-manual") is True
|
||||
assert config.sync_enabled("one-memory-manual") is False
|
||||
|
||||
assert _sedimentation_claim_enabled(app, "one-memory-manual") is True
|
||||
config.set_sedimentation_enabled(False)
|
||||
assert _sedimentation_claim_enabled(app, "one-memory-manual") is False
|
||||
|
||||
|
||||
def test_first_import_gate_waits_for_every_selected_history_job() -> None:
|
||||
state = SimpleNamespace(
|
||||
status="importing",
|
||||
selected_source_ids=("cursor", "codex"),
|
||||
started_at=datetime(2026, 10, 18, tzinfo=UTC),
|
||||
)
|
||||
jobs = [
|
||||
SimpleNamespace(
|
||||
operation="history",
|
||||
source_id="cursor",
|
||||
created_at=datetime(2026, 10, 18, tzinfo=UTC),
|
||||
status="completed",
|
||||
),
|
||||
SimpleNamespace(
|
||||
operation="history",
|
||||
source_id="codex",
|
||||
created_at=datetime(2026, 10, 18, tzinfo=UTC),
|
||||
status="running",
|
||||
),
|
||||
]
|
||||
app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
onboarding_store=SimpleNamespace(get=lambda: state),
|
||||
import_job_store=SimpleNamespace(list_jobs=lambda: jobs),
|
||||
)
|
||||
)
|
||||
|
||||
assert _history_import_gate_released(app) is False
|
||||
jobs[1].status = "failed"
|
||||
assert _history_import_gate_released(app) is True
|
||||
@@ -19,12 +19,7 @@ def test_production_serve_passes_only_supported_app_options(monkeypatch, tmp_pat
|
||||
|
||||
def run(application, **kwargs):
|
||||
assert application is marker
|
||||
assert kwargs == {
|
||||
"host": "127.0.0.1",
|
||||
"port": 12345,
|
||||
"reload": False,
|
||||
"access_log": False,
|
||||
}
|
||||
assert kwargs == {"host": "127.0.0.1", "port": 12345, "reload": False}
|
||||
|
||||
monkeypatch.setattr(cli, "load_settings", lambda: settings)
|
||||
monkeypatch.setattr(log_file, "start_logging_to_file", lambda _: None)
|
||||
@@ -51,12 +46,7 @@ def test_maintenance_serve_does_not_own_the_runtime(monkeypatch, tmp_path):
|
||||
|
||||
def run(application, **kwargs):
|
||||
assert application is marker
|
||||
assert kwargs == {
|
||||
"host": "127.0.0.1",
|
||||
"port": 12346,
|
||||
"reload": False,
|
||||
"access_log": False,
|
||||
}
|
||||
assert kwargs == {"host": "127.0.0.1", "port": 12346, "reload": False}
|
||||
|
||||
monkeypatch.setattr(cli, "load_settings", lambda: settings)
|
||||
monkeypatch.setattr(log_file, "start_logging_to_file", lambda _: None)
|
||||
|
||||
@@ -13,7 +13,7 @@ from one_memory.imports.adapters import (
|
||||
SourceCursor,
|
||||
)
|
||||
from one_memory.imports.jobs import ImportJobStore, ImportStatus
|
||||
from one_memory.imports.models import EvidenceOrigin, WrittenBy
|
||||
from one_memory.imports.models import WrittenBy
|
||||
from one_memory.imports.runner import ImportRunner
|
||||
from one_memory.imports.sources.claude_code import create_claude_code_adapter
|
||||
from one_memory.imports.sources.codex import create_codex_adapter
|
||||
@@ -131,9 +131,6 @@ def test_cursor_store_timestamp_envelope_keeps_only_the_user_query_text(tmp_path
|
||||
|
||||
assert [item.evidence.verbatim for item in records] == [query]
|
||||
assert records[0].evidence.written_by is WrittenBy.PERSON
|
||||
meta = records[0].evidence.origin_meta[EvidenceOrigin.FILE].raw_meta
|
||||
assert meta["time_basis"] == "file_mtime"
|
||||
assert meta["turn_event_at"] == "2026-09-22T23:34:00+00:00"
|
||||
|
||||
|
||||
def test_cursor_transcript_timestamp_envelope_keeps_only_the_user_query_text(tmp_path) -> None:
|
||||
@@ -166,9 +163,6 @@ def test_cursor_transcript_timestamp_envelope_keeps_only_the_user_query_text(tmp
|
||||
|
||||
assert [item.evidence.verbatim for item in records] == [query]
|
||||
assert records[0].evidence.written_by is WrittenBy.PERSON
|
||||
meta = records[0].evidence.origin_meta[EvidenceOrigin.FILE].raw_meta
|
||||
assert meta["time_basis"] == "file_mtime"
|
||||
assert meta["turn_event_at"] == "2026-09-22T23:34:00+00:00"
|
||||
|
||||
|
||||
def test_cursor_transcript_uses_file_mtime_with_stable_record_order(tmp_path) -> None:
|
||||
|
||||
@@ -191,12 +191,10 @@ from one_memory import cli
|
||||
from one_memory.config import Settings
|
||||
|
||||
served_on = []
|
||||
served_with = []
|
||||
|
||||
|
||||
def fake_run(application, *, host=None, port=None, **_kwargs):
|
||||
served_on.append(port)
|
||||
served_with.append(_kwargs)
|
||||
|
||||
|
||||
uvicorn.run = fake_run
|
||||
@@ -221,7 +219,6 @@ json.dump(
|
||||
{
|
||||
"printed": printed,
|
||||
"served_on": served_on,
|
||||
"served_with": served_with,
|
||||
"port": int(match.group(1)) if match else None,
|
||||
},
|
||||
sys.stdout,
|
||||
@@ -252,4 +249,3 @@ json.dump(
|
||||
# Port 0 means "any free port", so the reported value must be a real one.
|
||||
assert payload["port"] > 0
|
||||
assert payload["served_on"] == [payload["port"]]
|
||||
assert payload["served_with"] == [{"reload": False, "access_log": False}]
|
||||
@@ -132,22 +132,19 @@ def test_raw_authors_keep_exact_body_and_do_not_mutate_evidence(role, author):
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"role": "user", "written_by": WrittenBy.AGENT},
|
||||
{"role": "assistant", "written_by": WrittenBy.PERSON},
|
||||
{"role": "tool", "written_by": WrittenBy.TOOL},
|
||||
{"role": "system"},
|
||||
{"written_by": WrittenBy.UNKNOWN},
|
||||
{"content_kind": EvidenceContentKind.MEMORY_CONTENT},
|
||||
],
|
||||
)
|
||||
def test_raw_role_qualification_rejects_without_tokenizing(change):
|
||||
def test_raw_and_author_qualification_rejects_without_tokenizing(change):
|
||||
with pytest.raises(MappedContentError, match="mapped_content_ineligible"):
|
||||
prepare_mapped_content(record("ordinary body", **change))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("author", list(WrittenBy))
|
||||
def test_host_author_guess_does_not_change_protocol_role_qualification(author):
|
||||
content = prepare_mapped_content(record("ordinary body", role="user", written_by=author))
|
||||
assert content.state == "ready"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,reason",
|
||||
[
|
||||
|
||||
@@ -25,6 +25,7 @@ def _message(
|
||||
message_id=message_id or f"m{ordinal}",
|
||||
ordinal=ordinal,
|
||||
role=role,
|
||||
written_by="person" if role == "user" else "model",
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ Dev history: when the model extracted no fact, the coding extractor stored the
|
||||
raw dialogue as a working fact, and with no words of the person's it took the
|
||||
first 600 characters of the rendered trace. MemHub showed that as the memory
|
||||
content, which is where `[0] assistant: ... [1] tool(?): ...` came from
|
||||
(D87). The SDK fix remains present in 2.0.4: it does not store that fallback by
|
||||
(D87). The SDK fix remains present in 2.0.3: it does not store that fallback by
|
||||
default.
|
||||
|
||||
This pins the default here, so a future bump that turns it back on, or our own
|
||||
|
||||
@@ -1,375 +0,0 @@
|
||||
"""Stopping model-driven sedimentation without losing the configuration.
|
||||
|
||||
The switch is the user's cost control: while it is off nothing may reach the
|
||||
model, but neither the model configuration nor the already imported
|
||||
conversations may be touched, and work that was already queued has to survive
|
||||
so starting again resumes it instead of beginning from nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import SecretStr
|
||||
|
||||
from one_memory.api import create_app
|
||||
from one_memory.config import Settings
|
||||
from one_memory.extraction_settings import ExtractionSettingsStore
|
||||
from one_memory.hy_memory_bridge import HyMemoryBridgeError
|
||||
from one_memory.imports.models import EvidenceOrigin, OriginMetadata, WrittenBy
|
||||
from one_memory.imports.storage import EvidenceInput, EvidenceStore
|
||||
from one_memory.memory_processing import MemoryProcessingSupervisor
|
||||
from one_memory.memory_tasks import MemoryTaskKind, MemoryTaskStatus, MemoryTaskStore
|
||||
from one_memory.model_profiles import (
|
||||
ModelCapabilities,
|
||||
ModelProfileInput,
|
||||
ModelProfileStore,
|
||||
ProviderKind,
|
||||
)
|
||||
from one_memory.secrets import InMemorySecretStore
|
||||
|
||||
NOW = datetime(2026, 9, 9, tzinfo=UTC)
|
||||
|
||||
|
||||
class _Flag:
|
||||
"""The user's switch, standing in for `ConfigStore.sedimentation_enabled`."""
|
||||
|
||||
def __init__(self, value: bool = True) -> None:
|
||||
self.value = value
|
||||
|
||||
def __call__(self) -> bool:
|
||||
return self.value
|
||||
|
||||
|
||||
def test_sedimentation_route_persists_and_resumes_without_restart(tmp_path) -> None:
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=tmp_path / "home",
|
||||
api_token=SecretStr("test"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
with TestClient(app, headers={"Authorization": "Bearer test"}) as client:
|
||||
assert client.get("/api/v1/sedimentation").json()["enabled"] is True
|
||||
stopped = client.put(
|
||||
"/api/v1/sedimentation", json={"enabled": False}
|
||||
)
|
||||
assert stopped.status_code == 200
|
||||
assert stopped.json()["enabled"] is False
|
||||
assert client.get("/api/v1/sedimentation").json()["enabled"] is False
|
||||
resumed = client.put("/api/v1/sedimentation", json={"enabled": True})
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json()["enabled"] is True
|
||||
|
||||
|
||||
def test_switch_answers_before_the_resume_finishes(tmp_path, monkeypatch) -> None:
|
||||
"""The PUT is a decision; the walk over the backlog happens after it.
|
||||
|
||||
Awaiting that walk is what turned "turn sedimentation back on" into a
|
||||
request that never returned. The renderer gave up at its thirty-second
|
||||
timeout and reported a failure for a switch that had in fact been saved,
|
||||
which is the worst of both: the user believes the model is still stopped
|
||||
while it is already running.
|
||||
"""
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=tmp_path / "home",
|
||||
api_token=SecretStr("test"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
released = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
with TestClient(app, headers={"Authorization": "Bearer test"}) as client:
|
||||
supervisor = app.state.memory_processing_supervisor
|
||||
|
||||
async def slow_resume() -> None:
|
||||
await asyncio.get_running_loop().run_in_executor(None, released.wait, 10.0)
|
||||
finished.set()
|
||||
|
||||
monkeypatch.setattr(supervisor, "resume_allowed", slow_resume)
|
||||
assert client.put("/api/v1/sedimentation", json={"enabled": False}).status_code == 200
|
||||
resumed = client.put("/api/v1/sedimentation", json={"enabled": True})
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json()["enabled"] is True
|
||||
# Answered while the resume is still walking the backlog.
|
||||
assert finished.is_set() is False
|
||||
released.set()
|
||||
|
||||
|
||||
def test_a_failing_resume_does_not_fail_the_switch(tmp_path, monkeypatch) -> None:
|
||||
"""Resuming is opportunistic; the user's choice is already stored."""
|
||||
app = create_app(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
data_dir=tmp_path / "data",
|
||||
source_home=tmp_path / "home",
|
||||
api_token=SecretStr("test"),
|
||||
imports_enabled=False,
|
||||
automatic_memory_processing=False,
|
||||
live_sync_enabled=False,
|
||||
),
|
||||
source_catalog={},
|
||||
)
|
||||
with TestClient(app, headers={"Authorization": "Bearer test"}) as client:
|
||||
supervisor = app.state.memory_processing_supervisor
|
||||
|
||||
async def broken_resume() -> None:
|
||||
raise RuntimeError("hy_memory_processing_unavailable")
|
||||
|
||||
monkeypatch.setattr(supervisor, "resume_allowed", broken_resume)
|
||||
assert client.put("/api/v1/sedimentation", json={"enabled": False}).status_code == 200
|
||||
resumed = client.put("/api/v1/sedimentation", json={"enabled": True})
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json()["enabled"] is True
|
||||
|
||||
|
||||
def _stores(tmp_path, flag: _Flag):
|
||||
database_path = tmp_path / "one-memory.sqlite3"
|
||||
evidence = EvidenceStore(database_path)
|
||||
tasks = MemoryTaskStore(database_path)
|
||||
profiles = ModelProfileStore(database_path)
|
||||
secrets = InMemorySecretStore()
|
||||
profile = profiles.create(
|
||||
ModelProfileInput(
|
||||
name="Local",
|
||||
provider_kind=ProviderKind.OPENAI_COMPATIBLE,
|
||||
base_url="https://example.com/v1",
|
||||
model="test-model",
|
||||
embedding_provider_kind=ProviderKind.LOCAL_ONNX,
|
||||
embedding_model="builtin",
|
||||
embedding_dimensions=512,
|
||||
),
|
||||
secrets,
|
||||
)
|
||||
profiles.record_capabilities(
|
||||
profile.profile_id,
|
||||
expected_revision=profile.revision,
|
||||
capabilities=ModelCapabilities(
|
||||
endpoint_reachable=True,
|
||||
model_available=True,
|
||||
structured_output=True,
|
||||
embedding_available=True,
|
||||
),
|
||||
secrets=secrets,
|
||||
)
|
||||
active = profiles.activate(profile.profile_id, secrets)
|
||||
supervisor = MemoryProcessingSupervisor(
|
||||
tasks=tasks,
|
||||
profiles=profiles,
|
||||
secrets=secrets,
|
||||
evidence=evidence,
|
||||
governance=None, # type: ignore[arg-type]
|
||||
data_dir=tmp_path,
|
||||
extraction_settings=ExtractionSettingsStore(database_path),
|
||||
sedimentation_enabled=flag,
|
||||
)
|
||||
return evidence, tasks, active, supervisor, profiles, secrets
|
||||
|
||||
|
||||
def _task(evidence, tasks, active, *, index: int, age_days: int = 1):
|
||||
record = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
workspace="/Users/me/project",
|
||||
conversation_id=f"conv-{index}",
|
||||
message_id=f"m{index}",
|
||||
correlation_id=f"m{index}",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=NOW - timedelta(days=age_days),
|
||||
verbatim="I prefer concise answers.",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
return tasks.create(
|
||||
request_id=f"req-{index}",
|
||||
user_id="local",
|
||||
kind=MemoryTaskKind.EXTRACT,
|
||||
input_ref=record.evidence_id,
|
||||
profile=active,
|
||||
)
|
||||
|
||||
|
||||
def test_stopped_switch_classifies_no_work_and_keeps_it_pending(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
|
||||
assert supervisor._classify([task]) == []
|
||||
# Parked, not failed: nothing was sent to the model, so a stopped switch
|
||||
# must not read as a broken pipeline.
|
||||
assert tasks.require(task.task_id).status is MemoryTaskStatus.PENDING
|
||||
|
||||
flag.value = True
|
||||
assert len(supervisor._classify([task])) == 1
|
||||
|
||||
|
||||
def test_stopped_switch_refuses_direct_durable_processing(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, flag
|
||||
)
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
record = evidence.get(task.input_ref)
|
||||
assert record is not None
|
||||
|
||||
with pytest.raises(HyMemoryBridgeError, match="model_sedimentation_disabled"):
|
||||
asyncio.run(supervisor.process_records((record,)))
|
||||
|
||||
assert tasks.require(task.task_id).status is MemoryTaskStatus.PENDING
|
||||
|
||||
|
||||
def test_pause_returns_a_claimed_task_attempt(tmp_path) -> None:
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, _supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, flag
|
||||
)
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
claimed, token = tasks.claim(task.task_id)
|
||||
assert claimed.attempts == 1
|
||||
|
||||
released = tasks.release_unstarted(
|
||||
task.task_id,
|
||||
runner_token=token,
|
||||
error_code="model_sedimentation_disabled",
|
||||
)
|
||||
|
||||
assert released.status is MemoryTaskStatus.PENDING
|
||||
assert released.attempts == 0
|
||||
|
||||
|
||||
def test_stopping_after_scheduler_classification_starts_no_worker(tmp_path) -> None:
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, flag
|
||||
)
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
work = supervisor._classify([task])
|
||||
assert len(work) == 1
|
||||
supervisor._queue.add(work[0])
|
||||
|
||||
flag.value = False
|
||||
asyncio.run(supervisor._work())
|
||||
|
||||
current = tasks.require(task.task_id)
|
||||
assert current.status is MemoryTaskStatus.PENDING
|
||||
assert current.attempts == 0
|
||||
|
||||
|
||||
def test_stopped_switch_enqueues_nothing(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, _tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
record = evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
workspace="/Users/me/project",
|
||||
conversation_id="conv-live",
|
||||
message_id="m-live",
|
||||
correlation_id="m-live",
|
||||
role="user",
|
||||
written_by=WrittenBy.PERSON,
|
||||
created_at=NOW,
|
||||
verbatim="Remember this.",
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
).record
|
||||
|
||||
assert supervisor.enqueue_evidence(record) is None
|
||||
assert supervisor._debounces == {}
|
||||
|
||||
|
||||
def test_stopped_switch_prepares_no_backfill(tmp_path) -> None:
|
||||
flag = _Flag(False)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
# No task is queued yet: backfill is what would create one for this
|
||||
# conversation, so the stopped switch must keep it from creating any.
|
||||
# A settled turn (question plus answer): an unanswered user tail is not
|
||||
# backfill material, so the conversation needs both records.
|
||||
for message_id, role, written_by, verbatim in (
|
||||
("m-old", "user", WrittenBy.PERSON, "Remember this too."),
|
||||
("m-old-reply", "assistant", WrittenBy.AGENT, "Saved."),
|
||||
):
|
||||
evidence.put(
|
||||
EvidenceInput(
|
||||
source_id="cursor",
|
||||
workspace="/Users/me/project",
|
||||
conversation_id="conv-old",
|
||||
message_id=message_id,
|
||||
correlation_id=message_id,
|
||||
role=role,
|
||||
written_by=written_by,
|
||||
created_at=NOW - timedelta(days=1),
|
||||
verbatim=verbatim,
|
||||
origin_meta={EvidenceOrigin.FILE: OriginMetadata()},
|
||||
)
|
||||
)
|
||||
|
||||
assert supervisor.prepare_backfill() == []
|
||||
|
||||
flag.value = True
|
||||
assert len(supervisor.prepare_backfill()) == 1
|
||||
|
||||
|
||||
def test_absent_switch_keeps_sedimenting(tmp_path) -> None:
|
||||
"""A Core without the setting behaves exactly as it did before it existed."""
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(
|
||||
tmp_path, _Flag(True)
|
||||
)
|
||||
supervisor.sedimentation_enabled = None
|
||||
task = _task(evidence, tasks, active, index=1)
|
||||
|
||||
assert len(supervisor._classify([task])) == 1
|
||||
|
||||
|
||||
def test_stopping_does_not_disturb_the_model_configuration(tmp_path) -> None:
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, supervisor, profiles, secrets = _stores(tmp_path, flag)
|
||||
_task(evidence, tasks, active, index=1)
|
||||
|
||||
flag.value = False
|
||||
assert supervisor.prepare_backfill() == []
|
||||
# Still configured: starting again is a switch, not a re-entry of a key.
|
||||
assert profiles.active(secrets) is not None
|
||||
|
||||
|
||||
def test_bounded_backfill_returns_when_the_queue_is_already_full(tmp_path) -> None:
|
||||
"""A full queue means nothing to add, not "wait until it drains".
|
||||
|
||||
The wait used to sleep inside the scan lock, so whoever asked for the scan
|
||||
— the resume behind a switch toggle, and through it an HTTP request —
|
||||
waited for the whole backlog to clear. A saturated queue never clears by
|
||||
waiting, so that request never came back.
|
||||
"""
|
||||
flag = _Flag(True)
|
||||
evidence, tasks, active, supervisor, _profiles, _secrets = _stores(tmp_path, flag)
|
||||
for index in range(8):
|
||||
task = _task(evidence, tasks, active, index=index)
|
||||
supervisor._queue.add(supervisor._classify([task])[0])
|
||||
|
||||
started = time.monotonic()
|
||||
asyncio.run(
|
||||
asyncio.wait_for(supervisor.backfill_off_thread(cohort_size=8), timeout=3)
|
||||
)
|
||||
assert time.monotonic() - started < 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__]))
|
||||
@@ -1,194 +0,0 @@
|
||||
"""Why a source cannot be read, named separately from whether it can.
|
||||
|
||||
`authorized=false` was one answer to six different situations, so the UI could
|
||||
only offer every recovery at once and people reinstalled plugins for a problem
|
||||
in the source grant (issue #29). These cover the naming, and the capture-side
|
||||
refusal that used to be silent and unrecorded.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from one_memory.imports.access import SourceAccessError, SourceAccessErrorCode
|
||||
from one_memory.imports.grants import SourceAuthorizationState, SourceGrantStore
|
||||
from one_memory.imports.sources.registry import RootCandidateKind
|
||||
from one_memory.live_capture import (
|
||||
CaptureRejection,
|
||||
capture_rejection,
|
||||
record_capture_rejection,
|
||||
)
|
||||
from one_memory.memory_server.config_state import ConfigStore
|
||||
|
||||
|
||||
def build_store(tmp_path: Path) -> tuple[SourceGrantStore, Path, Path]:
|
||||
home = tmp_path / "home"
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
database = tmp_path / "one-memory.sqlite3"
|
||||
return SourceGrantStore(database, home=home), database, home
|
||||
|
||||
|
||||
def _grant(store: SourceGrantStore, home: Path, name: str) -> Path:
|
||||
root = home / name
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
store.grant(
|
||||
user_id="local",
|
||||
source_id="workbuddy",
|
||||
root=root,
|
||||
candidate_kind=RootCandidateKind.DEFAULT,
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
def test_no_grant_is_named_missing(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=home / "workbuddy"
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.MISSING_GRANT
|
||||
|
||||
|
||||
def test_a_live_grant_on_the_current_candidate_is_ok(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.OK
|
||||
|
||||
|
||||
def test_a_revoked_grant_is_not_a_missing_one(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
store.revoke(user_id="local", source_id="workbuddy")
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.REVOKED_GRANT
|
||||
|
||||
|
||||
def test_a_rebuilt_directory_is_not_a_moved_candidate(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
root.rmdir()
|
||||
root.mkdir()
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.SOURCE_IDENTITY_CHANGED
|
||||
|
||||
|
||||
def test_a_missing_directory_is_named_unavailable(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
root.rmdir()
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=root
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.SOURCE_ROOT_UNAVAILABLE
|
||||
|
||||
|
||||
def test_a_grant_pointing_somewhere_else_is_a_moved_candidate(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
elsewhere = home / "workbuddy-elsewhere"
|
||||
elsewhere.mkdir()
|
||||
|
||||
state = store.authorization_state(
|
||||
user_id="local", source_id="workbuddy", candidate=elsewhere
|
||||
)
|
||||
|
||||
assert state is SourceAuthorizationState.CANDIDATE_CHANGED
|
||||
|
||||
|
||||
def test_a_grant_still_fails_closed_when_a_state_is_named(tmp_path) -> None:
|
||||
store, _database, home = build_store(tmp_path)
|
||||
root = _grant(store, home, "workbuddy")
|
||||
root.rmdir()
|
||||
root.mkdir()
|
||||
|
||||
# Naming the reason is not granting access: the read path still refuses,
|
||||
# and says so through the same code it always did.
|
||||
with pytest.raises(SourceAccessError) as captured:
|
||||
store.authorized_root(user_id="local", source_id="workbuddy")
|
||||
assert captured.value.code is SourceAccessErrorCode.SOURCE_IDENTITY_CHANGED
|
||||
|
||||
|
||||
def test_capture_rejection_names_sync_disabled(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
|
||||
assert (
|
||||
capture_rejection(database, "workbuddy") is CaptureRejection.SYNC_DISABLED
|
||||
)
|
||||
|
||||
|
||||
def test_capture_rejection_names_a_missing_then_a_revoked_grant(tmp_path) -> None:
|
||||
_store, database, _home = build_store(tmp_path)
|
||||
ConfigStore(database.parent).set_sync("workbuddy", True)
|
||||
|
||||
assert capture_rejection(database, "workbuddy") is CaptureRejection.GRANT_MISSING
|
||||
|
||||
store, _database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
store.revoke(user_id="local", source_id="workbuddy")
|
||||
|
||||
assert capture_rejection(database, "workbuddy") is CaptureRejection.GRANT_REVOKED
|
||||
|
||||
|
||||
def test_capture_rejection_is_none_when_a_turn_may_be_captured(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
ConfigStore(database.parent).set_sync("workbuddy", True)
|
||||
|
||||
assert capture_rejection(database, "workbuddy") is None
|
||||
|
||||
|
||||
def test_the_recorded_rejection_carries_no_turn_and_no_path(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
secret = "深绿色的按钮,不要告诉别人"
|
||||
marker = home / "secret.txt"
|
||||
marker.write_text(secret, encoding="utf8")
|
||||
|
||||
record_capture_rejection(database, "workbuddy", CaptureRejection.SYNC_DISABLED)
|
||||
record_capture_rejection(database, "workbuddy", CaptureRejection.SYNC_DISABLED)
|
||||
|
||||
record = json.loads(
|
||||
(
|
||||
database.parent
|
||||
/ "logs"
|
||||
/ "capture-rejection-workbuddy-sync_disabled.json"
|
||||
).read_text(encoding="utf8")
|
||||
)
|
||||
assert record["count"] == 2
|
||||
assert record["source_id"] == "workbuddy"
|
||||
assert record["reason"] == "sync_disabled"
|
||||
assert secret not in json.dumps(record, ensure_ascii=False)
|
||||
assert str(home) not in json.dumps(record, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_recording_a_rejection_never_breaks_the_host_turn(tmp_path) -> None:
|
||||
store, database, home = build_store(tmp_path)
|
||||
_grant(store, home, "workbuddy")
|
||||
# A data directory that cannot be written must stay silent: this runs
|
||||
# inside somebody's turn, where raising would print into the conversation.
|
||||
blocked = tmp_path / "blocked"
|
||||
blocked.write_text("not a directory", encoding="utf8")
|
||||
|
||||
record_capture_rejection(
|
||||
blocked / "one-memory.sqlite3",
|
||||
"workbuddy",
|
||||
CaptureRejection.GRANT_REVOKED,
|
||||
)
|
||||
@@ -195,19 +195,11 @@ def test_connection_retry_reuses_completed_job_and_checks_root_snapshot(tmp_path
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("turn_count", "age_days", "limited"),
|
||||
[
|
||||
(200, 4, False),
|
||||
(201, 1, False),
|
||||
(201, 4, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(("session_count", "limited"), [(200, False), (201, True)])
|
||||
def test_onboarding_applies_large_history_window_to_all_sources(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
turn_count,
|
||||
age_days,
|
||||
session_count,
|
||||
limited,
|
||||
):
|
||||
monkeypatch.setattr("one_memory.model_presets._preset_data", lambda: None)
|
||||
@@ -218,22 +210,8 @@ def test_onboarding_applies_large_history_window_to_all_sources(
|
||||
codex = home / ".codex/sessions"
|
||||
claude.mkdir(parents=True)
|
||||
codex.mkdir(parents=True)
|
||||
timestamp = (datetime.now(UTC) - timedelta(days=age_days)).isoformat()
|
||||
(claude / "large-session.jsonl").write_text(
|
||||
"".join(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "user",
|
||||
"sessionId": "large-session",
|
||||
"uuid": f"user-{index}",
|
||||
"timestamp": timestamp,
|
||||
"message": {"content": f"turn {index}"},
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
for index in range(turn_count)
|
||||
)
|
||||
)
|
||||
for index in range(session_count):
|
||||
(claude / f"{index:03}.jsonl").write_text("")
|
||||
(codex / "one.jsonl").write_text("")
|
||||
app = create_app(
|
||||
Settings(
|
||||
@@ -262,7 +240,7 @@ def test_onboarding_applies_large_history_window_to_all_sources(
|
||||
"/api/v1/sources/connect",
|
||||
json={
|
||||
"source_ids": ["claude-code", "codex"],
|
||||
"request_id": f"history-window-{turn_count}-{age_days}",
|
||||
"request_id": f"history-window-{session_count}",
|
||||
"source_roots": {
|
||||
"claude-code": claude_root,
|
||||
"codex": codex_root,
|
||||
|
||||
@@ -39,11 +39,7 @@ def record(
|
||||
source_id=source,
|
||||
conversation_id=f"c{index % 2}",
|
||||
message_id=str(index),
|
||||
role=(
|
||||
"assistant"
|
||||
if author == WrittenBy.AGENT
|
||||
else ("tool" if author == WrittenBy.TOOL else "user")
|
||||
),
|
||||
role="assistant" if author == WrittenBy.AGENT else "user",
|
||||
written_by=author,
|
||||
created_at=time,
|
||||
verbatim=text or f"original {index}",
|
||||
@@ -87,8 +83,8 @@ def test_receipt_counts_saved_records_without_inventing_dates_or_other_sources(t
|
||||
data = response.json()
|
||||
assert data["source_id"] == "claude-code"
|
||||
assert data["messages"] == 5 and data["conversations"] == 2
|
||||
assert data["person_messages"] == 3 and data["agent_messages"] == 1
|
||||
assert data["unknown_author_messages"] == 0 and data["tool_messages"] == 1
|
||||
assert data["person_messages"] == 2 and data["agent_messages"] == 1
|
||||
assert data["unknown_author_messages"] == 1 and data["tool_messages"] == 1
|
||||
assert data["unknown_time_messages"] == 4
|
||||
assert len(data["examples"]) == 3
|
||||
assert "excluded-source" not in response.text and "excluded-user" not in response.text
|
||||
|
||||
@@ -34,8 +34,8 @@ def test_a_heartbeat_is_not_a_question():
|
||||
assert last_row_opens_an_unanswered_turn(("user", "person", "<heartbeat>ping")) is False
|
||||
|
||||
|
||||
def test_protocol_user_role_opens_a_turn_regardless_of_legacy_author_guess():
|
||||
assert last_row_opens_an_unanswered_turn(("user", "agent", "继续")) is True
|
||||
def test_a_message_written_by_the_agent_as_user_is_not_a_question():
|
||||
assert last_row_opens_an_unanswered_turn(("user", "agent", "继续")) is False
|
||||
|
||||
|
||||
def test_a_conversation_with_no_rows_is_not_unanswered():
|
||||
|
||||
@@ -124,7 +124,7 @@ def test_the_sdk_strips_tool_messages_on_the_chat_path():
|
||||
|
||||
|
||||
def test_the_sdk_still_cuts_tool_results_the_way_this_assumes():
|
||||
"""MemHub's window estimate follows its configured SDK tool ceiling.
|
||||
"""D74 counts 2 KiB per tool record because the SDK cuts there.
|
||||
|
||||
If an SDK upgrade moves that boundary or stops dropping tool records on
|
||||
the chat path, `_HY_TOOL_CHARACTERS` no longer describes what HY reads and
|
||||
@@ -141,14 +141,14 @@ def test_the_sdk_still_cuts_tool_results_the_way_this_assumes():
|
||||
from one_memory.memory_processing import _HY_TOOL_CHARACTERS
|
||||
|
||||
assert DEFAULT_MAX_TOOL_RESULT_BYTES == 2048
|
||||
assert _HY_TOOL_CHARACTERS == 512
|
||||
assert _HY_TOOL_CHARACTERS == 2048
|
||||
long = "z" * 20_000
|
||||
cut = truncate_tool_result_text(long, max_bytes=_HY_TOOL_CHARACTERS)
|
||||
assert len(cut.encode()) <= _HY_TOOL_CHARACTERS + 128
|
||||
cut = truncate_tool_result_text(long)
|
||||
assert len(cut) <= DEFAULT_MAX_TOOL_RESULT_BYTES + 128
|
||||
|
||||
from hy_memory.coding.preproc import truncate_messages
|
||||
|
||||
messages = [ChatMessage(role="tool", content=long)]
|
||||
truncated = truncate_messages(messages, max_bytes=_HY_TOOL_CHARACTERS)
|
||||
truncated = truncate_messages(messages)
|
||||
assert sum(len(item.content or "") for item in truncated) < len(long)
|
||||
assert _HY_TOOL_CHARACTERS < DEFAULT_HEAD_BYTES + DEFAULT_TAIL_BYTES
|
||||
assert _HY_TOOL_CHARACTERS >= DEFAULT_HEAD_BYTES + DEFAULT_TAIL_BYTES
|
||||
Generated
+5
-5
@@ -991,7 +991,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "hy-memory"
|
||||
version = "2.0.4"
|
||||
version = "2.0.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "chromadb" },
|
||||
@@ -1008,9 +1008,9 @@ dependencies = [
|
||||
{ name = "scikit-learn" },
|
||||
{ name = "tiktoken" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1f/d0/44af626b3991a49f787074ebf7474ab68d7adf93a048555ffcdd3092d215/hy_memory-2.0.4.tar.gz", hash = "sha256:ae68837a55afdd50143003608e586350da36c5bd90f4dafcf460c34377179a11", size = 824206, upload-time = "2026-10-17T13:47:29.219Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/12/10e91468d1331eff8380e6b32c1e8784413660a299005ce07ac3e60e90bd/hy_memory-2.0.3.tar.gz", hash = "sha256:cdebd46b16ebe3cb04c89475f9044d30c6cc9065796be369f5d9607410a5c96b", size = 823596, upload-time = "2026-10-17T11:06:23.093Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/68/1c6512a7c3c41cad2731df40416a5331cbc95763feb4983fb59a91869cc7/hy_memory-2.0.4-py3-none-any.whl", hash = "sha256:1d6fd4120185cc6cbcf04f9d81d5bf47e33c467f52787e5d175a412d4e687cd1", size = 888026, upload-time = "2026-10-17T13:47:27.062Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/ed/bac8c15eb91baeaeb15a9061d7937ffd02ebe3384757addf0ca09660e592/hy_memory-2.0.3-py3-none-any.whl", hash = "sha256:763d6388ea83762e06c51aecd8e5d8e76211000f67355f65a760eb0a17ede4c5", size = 887157, upload-time = "2026-10-17T11:06:20.994Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1801,7 +1801,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "one-memory"
|
||||
version = "1.0.4"
|
||||
version = "1.0.3"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiomysql" },
|
||||
@@ -1837,7 +1837,7 @@ requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.141.1" },
|
||||
{ name = "filelock", specifier = ">=3.32.4" },
|
||||
{ name = "httpx", specifier = ">=0.28.1" },
|
||||
{ name = "hy-memory", specifier = "==2.0.4" },
|
||||
{ name = "hy-memory", specifier = "==2.0.3" },
|
||||
{ name = "keyring", specifier = ">=25.7.0" },
|
||||
{ name = "numpy", specifier = "==2.5.2" },
|
||||
{ name = "onnxruntime", specifier = "==1.23.2" },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@one-memory/desktop",
|
||||
"version": "1.0.4",
|
||||
"version": "1.0.3",
|
||||
"private": true,
|
||||
"main": "dist/main.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,12 +1,7 @@
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
isStartupScreenRequest,
|
||||
resolveWebResource,
|
||||
STARTUP_SCREEN_PATH,
|
||||
} from "./app-protocol";
|
||||
import { STARTUP_SCREEN_URL } from "./startup-screen";
|
||||
import { resolveWebResource } from "./app-protocol";
|
||||
|
||||
describe("packaged web protocol", () => {
|
||||
it("serves only files inside the packaged web root", () => {
|
||||
@@ -25,14 +20,4 @@ describe("packaged web protocol", () => {
|
||||
resolveWebResource(root, "one-memory://app/%2F..%2Fsecret"),
|
||||
).toBeNull();
|
||||
});
|
||||
it("recognizes the startup route and never maps it onto a file", () => {
|
||||
const root = path.resolve("/bundle/web");
|
||||
expect(isStartupScreenRequest(STARTUP_SCREEN_URL)).toBe(true);
|
||||
expect(isStartupScreenRequest("one-memory://app/index.html")).toBe(false);
|
||||
expect(isStartupScreenRequest("one-memory://other/__startup")).toBe(false);
|
||||
// The placeholder is served from memory, so a file of that name inside the
|
||||
// web bundle must never be reachable through it.
|
||||
expect(resolveWebResource(root, STARTUP_SCREEN_URL)).toBeNull();
|
||||
expect(STARTUP_SCREEN_PATH).toBe("/__startup");
|
||||
});
|
||||
});
|
||||
@@ -5,28 +5,12 @@ import { net, protocol } from "electron";
|
||||
|
||||
export const APP_ORIGIN = "one-memory://app";
|
||||
|
||||
/** The route the startup placeholder is served from. It is never a file: the
|
||||
* placeholder has to load before anything in the web bundle is trusted. */
|
||||
export const STARTUP_SCREEN_PATH = "/__startup";
|
||||
|
||||
export function isStartupScreenRequest(requestUrl: string): boolean {
|
||||
const url = new URL(requestUrl);
|
||||
return (
|
||||
url.protocol === "one-memory:" &&
|
||||
url.host === "app" &&
|
||||
url.pathname === STARTUP_SCREEN_PATH
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveWebResource(
|
||||
webRoot: string,
|
||||
requestUrl: string,
|
||||
): string | null {
|
||||
const url = new URL(requestUrl);
|
||||
if (url.protocol !== "one-memory:" || url.host !== "app") return null;
|
||||
// Reserved for the in-memory startup placeholder: a file of that name in the
|
||||
// web bundle is not the placeholder and must not be reachable as one.
|
||||
if (isStartupScreenRequest(requestUrl)) return null;
|
||||
const pathname = decodeURIComponent(url.pathname);
|
||||
const requested = pathname === "/" ? "index.html" : pathname.slice(1);
|
||||
const resolvedRoot = path.resolve(webRoot);
|
||||
@@ -38,17 +22,8 @@ export function resolveWebResource(
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export function installAppProtocol(
|
||||
webRoot: string,
|
||||
options: { startupScreen?: () => string } = {},
|
||||
): void {
|
||||
export function installAppProtocol(webRoot: string): void {
|
||||
protocol.handle("one-memory", (request) => {
|
||||
if (options.startupScreen && isStartupScreenRequest(request.url)) {
|
||||
return new Response(options.startupScreen(), {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html; charset=utf-8" },
|
||||
});
|
||||
}
|
||||
const resource = resolveWebResource(webRoot, request.url);
|
||||
if (!resource) {
|
||||
return new Response("Not found", { status: 404 });
|
||||
|
||||
Loaded 100 of 168 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user