mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-03 20:27:56 +00:00
Keep timeout and custody provenance aligned
Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
This commit is contained in:
parent
b87f5f9fee
commit
f702439f62
6 changed files with 79 additions and 10 deletions
|
|
@ -1990,7 +1990,10 @@ apply. A dead socket or unterminated stream after dispatch is instead
|
|||
not infer pre-dispatch provenance from Python's implicit `__context__`, because a
|
||||
fallback raised inside a prior provider handler can inherit that earlier attempt;
|
||||
only an explicit `__cause__` or typed transport metadata can release a row. A
|
||||
spent owner window yields a typed `$0
|
||||
low-level main-call helper with no explicit reserve uses the raw owner deadline;
|
||||
the normal round dispatcher passes the finalization reserve explicitly, keeping
|
||||
dispatch admission and the transport bound on the same window. A spent owner
|
||||
window yields a typed `$0
|
||||
not_dispatched` row before fan-out; under blocking enforcement an in-flight
|
||||
triad row remains pending instead of becoming a final quorum verdict. A
|
||||
reviewed commit has no independent outer tool cutoff: the foreground caller
|
||||
|
|
|
|||
|
|
@ -1625,6 +1625,7 @@ Before every commit, verify the following:
|
|||
- [ ] Preserve exact direct-Anthropic native assistant content only as private unfinished-turn custody: same-route replay must include the complete original block list/order and every opaque member; cross-route send, summarizer input, and public observability must scrub values. The active assistant/tool-result unit cannot compact until a later successful assistant response consumes it. Owner `none` is `thinking.type=disabled`; do not guess legacy manual-thinking budgets.
|
||||
- [ ] Every core-mediated physical provider send goes through `usage_accounting.execute_physical_attempt[_async]`: reserve, mark dispatched, then settle/unresolve. A marked dispatch may be released only through a typed pre-dispatch connection/pool failure that proves no request bytes were sent; an ordinary timeout or unknown error remains unresolved. A transport retry is a new attempt. `llm_usage`, state, and UI counters are projections carrying attempt ids, never a second monetary authority. Provider tier pricing and any empirical tokenizer margin affect only a known reservation; settlement prefers actual provider usage/cost. Unknown price reserves `None`, remains nullable in usage events, and never blocks a model merely because its tariff is unavailable. An external skill with granted model-provider credentials is explicitly unknown/unmetered when it bypasses core transport—not `$0`; an ordinary spawned process must not be mislabeled as monetary work.
|
||||
- [ ] Custody classifiers must not treat Python's implicit exception `__context__` as transport provenance: a fallback raised inside a prior provider's `except` block inherits that prior attempt even after its own request was dispatched. Use the explicit `__cause__` chain or typed transport metadata only; an ambiguous timeout remains unresolved.
|
||||
- [ ] The low-level `call_llm_with_retry` seam treats an omitted transport reserve as the raw owner-deadline window; callers that own a finalization reserve must pass it explicitly so admission and the transport bound cannot disagree.
|
||||
- [ ] Hold the usage-ledger cross-process lock only for budget check, validated append, and fsync. Never hold it over network I/O. Preserve a paid response if settlement persistence fails and leave an honest dispatched/unresolved bound.
|
||||
- [ ] **Tree-spend visibility.** Under a root cap, pacing and stop text use root-subtree ledger spend including in-flight holds; own cost is diagnostic, and unavailable remains unknown rather than `$0`. Reuse `usage_accounting.last_root_accounting` and refresh only at rare cache-breaking/explicitly stale decision surfaces, never by an unconditional per-round ledger scan or inside a stable cached prefix. `task_pacing.resolve_cost_ceiling` returns `disabled|active|exhausted_soft_land|unknown` from the independent global-percentage and root-cap-minus-absolute-margin axes; graceful finalization precedes, but cannot bypass, the ledger fence. `resolve_deciding_spend` is the sole fallback seam and must label own-cost-under-root-cap as a lower bound.
|
||||
- [ ] Before dispatching any post-task consolidation or synthesis worker, read `usage_breakdown` once for the whole root subtree and pass the same loop-local snapshot to summary and reflection. It is explicitly non-final (`cost_final=false`, `cost_with_children_partial=true`) and carries child-inclusive accounted cost, reservations, unresolved upper bound, unknown/unmetered count, ledger integrity, and capture time. A read failure is unavailable/null, never `$0`. Consolidation, summary, and reflection model spend belongs only to the existing terminal checkpoint; do not add another ledger or reconciliation LLM call.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from ouroboros.request_wire_recovery import (
|
|||
)
|
||||
from ouroboros.usage_accounting import (
|
||||
AttemptRequest,
|
||||
PhysicalAttemptCapture,
|
||||
PhysicalAttemptPreconditionFailed,
|
||||
PhysicalAttemptPreparationFailed,
|
||||
UsageAccountingError,
|
||||
|
|
@ -46,7 +47,6 @@ from ouroboros.usage_accounting import (
|
|||
execute_physical_attempt,
|
||||
execute_physical_attempt_async,
|
||||
last_physical_attempt_capture,
|
||||
physical_attempt_capture_from_exception,
|
||||
usage_scope,
|
||||
)
|
||||
from ouroboros.utils import in_worker_process, sanitize_tool_result_for_log
|
||||
|
|
@ -2516,8 +2516,14 @@ class LLMClient:
|
|||
if (_is_structured_context_overflow_exception(exc)
|
||||
or context_overflow_message(err)):
|
||||
raise LocalContextTooLargeError(err) from exc
|
||||
capture = physical_attempt_capture_from_exception(exc)
|
||||
if capture is not None and capture.state in {"dispatched", "unresolved"}:
|
||||
# Only an exception-owned capture can prove that THIS local
|
||||
# attempt reached the provider. The process-local "last"
|
||||
# capture may belong to an unrelated earlier operation (or a
|
||||
# compatibility executor that never entered custody), and
|
||||
# must not turn an ordinary retryable error into a no-resend
|
||||
# tombstone.
|
||||
capture = getattr(exc, "physical_attempt_capture", None)
|
||||
if isinstance(capture, PhysicalAttemptCapture) and capture.state in {"dispatched", "unresolved"}:
|
||||
raise # Outer custody owns an unknown physical outcome.
|
||||
if attempt == 2:
|
||||
log.warning("Local model request failed: %s", exc)
|
||||
|
|
|
|||
|
|
@ -59,12 +59,13 @@ def _main_transport_timeout(
|
|||
# owner deadline. Other routes use the shared dead-socket bound; local models
|
||||
# receive the same explicit bound their client already supports.
|
||||
explicit = 120 if provider_for_model(model) == "anthropic" else None
|
||||
# ``None`` is the low-level raw-deadline contract. The production round
|
||||
# dispatcher passes the finalization reserve explicitly; keeping this
|
||||
# default raw prevents admission from accepting a call and then shrinking
|
||||
# its transport to the 0.001-second floor unexpectedly.
|
||||
return transport_timeout_with_deadline(
|
||||
explicit,
|
||||
deadline_ts=deadline_ts,
|
||||
reserve_sec=(
|
||||
get_finalization_grace_sec() if reserve_sec is None else reserve_sec
|
||||
),
|
||||
explicit, deadline_ts=deadline_ts,
|
||||
reserve_sec=0.0 if reserve_sec is None else reserve_sec,
|
||||
)
|
||||
|
||||
# Retrieval transparency (v6.78.0, owner Q20/Q22): native provider web search happens
|
||||
|
|
|
|||
|
|
@ -161,3 +161,49 @@ def test_local_output_limit_error_takes_normal_retry_path_not_overflow(monkeypat
|
|||
assert excinfo.value is output_limit_exc
|
||||
assert not isinstance(excinfo.value, llm_mod.LocalContextTooLargeError)
|
||||
assert calls["n"] == 3
|
||||
|
||||
|
||||
def test_local_retry_does_not_inherit_unrelated_physical_capture(monkeypatch, tmp_path):
|
||||
"""A missing exception-owned capture must not borrow a prior operation's custody."""
|
||||
from ouroboros import llm as llm_mod
|
||||
from ouroboros import usage_accounting as ua
|
||||
|
||||
# Leave an unresolved capture in the current ContextVar, as a previous
|
||||
# provider call would. The compatibility executor below then raises an
|
||||
# ordinary local error without entering the physical-attempt seam.
|
||||
with pytest.raises(RuntimeError):
|
||||
ua.execute_physical_attempt(
|
||||
ua.AttemptRequest(
|
||||
model="seed-model", provider="local", reservation_usd=0.0,
|
||||
drive_root=tmp_path, task_id="seed-task",
|
||||
),
|
||||
lambda: (_ for _ in ()).throw(RuntimeError("seed transport failure")),
|
||||
)
|
||||
assert ua.last_physical_attempt_capture().state == "unresolved"
|
||||
|
||||
output_limit_exc = RuntimeError(
|
||||
"max_tokens 65536 exceeds maximum context length 32768"
|
||||
)
|
||||
calls = {"n": 0}
|
||||
|
||||
def _fake_execute(request, send, before):
|
||||
calls["n"] += 1
|
||||
raise output_limit_exc
|
||||
|
||||
monkeypatch.setattr(llm_mod, "_execute_candidate", _fake_execute)
|
||||
monkeypatch.setattr(llm_mod, "_attempt_request", lambda *a, **k: None)
|
||||
monkeypatch.setattr(llm_mod.time, "sleep", lambda _s: None)
|
||||
client = llm_mod.LLMClient.__new__(llm_mod.LLMClient)
|
||||
monkeypatch.setattr(client, "_get_local_client", lambda: object(), raising=False)
|
||||
monkeypatch.setattr(
|
||||
client, "_normalize_system_message_placement", lambda m: list(m), raising=False)
|
||||
monkeypatch.setattr(
|
||||
client, "_strip_openrouter_roundtrip_metadata", lambda m: list(m), raising=False)
|
||||
monkeypatch.setattr(
|
||||
client, "_copy_messages_with_cache_policy",
|
||||
lambda m, **k: [dict(x) for x in m], raising=False)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
client._chat_local([{"role": "user", "content": "hi"}], None, 512, "auto")
|
||||
assert excinfo.value is output_limit_exc
|
||||
assert calls["n"] == 3
|
||||
|
|
|
|||
|
|
@ -84,7 +84,19 @@ def test_main_llm_transport_preserves_anthropic_default_but_narrows_deadline(mon
|
|||
monkeypatch.setattr(deadlines.time, "time", lambda: 1000.0)
|
||||
monkeypatch.setattr("ouroboros.loop_llm_call.get_finalization_grace_sec", lambda: 3)
|
||||
assert _main_transport_timeout("anthropic::claude-fable-5", None) == 120
|
||||
assert _main_transport_timeout("anthropic::claude-fable-5", 1010.0) == 7.0
|
||||
assert _main_transport_timeout("anthropic::claude-fable-5", 1010.0) == 10.0
|
||||
assert _main_transport_timeout(
|
||||
"anthropic::claude-fable-5", 1010.0, reserve_sec=3,
|
||||
) == 7.0
|
||||
|
||||
|
||||
def test_low_level_main_transport_admission_and_timeout_share_raw_default(monkeypatch):
|
||||
import ouroboros.deadline_utils as deadlines
|
||||
from ouroboros.loop_llm_call import _main_transport_timeout
|
||||
|
||||
monkeypatch.setattr(deadlines.time, "time", lambda: 1000.0)
|
||||
monkeypatch.setattr("ouroboros.loop_llm_call.get_finalization_grace_sec", lambda: 120)
|
||||
assert _main_transport_timeout("openai/gpt-5.5", 1005.0) == 5.0
|
||||
|
||||
|
||||
def test_spent_main_deadline_does_not_dispatch_or_fallback(tmp_path):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue