ouroboros/tests/test_delegate_hold.py
Ouroboros e33fd69115 Make the external-executor verbs answer with a native typed result
`delegate_shared._fail` rendered `{"status":"refused", ...}` as a plain string,
and the registry's legacy text adapter classified it as OK: it only understands
a top-level `ok:false` or a first-line `⚠️ IDENTIFIER` marker. So a refused
`delegate_wait`/`delegate_cancel` — daemon unreachable, run not owned, a
containment fault, a cancel the daemon refused — was recorded as a SUCCESSFUL
tool call on the outcome axis, in the acceptance packet and in the supervising
task's own reasoning.

Owner decision Q8A: the fix is a native structured result INSIDE the family,
not a repo-wide ABI migration. `_fail` now returns a `ToolResult` whose text is
the same JSON the callers emitted, plus two ADDITIVE envelope keys — `ok:false`
and `host_code` — written beside the domain payload. The domain `reason` is
never renamed into `ToolResult.code`, and the domain extras
(`definitely_unrun`, `pending_invocation_id`, `run_id`, `reset_at`, the custody
facts) keep their places. One exact, closed table maps a reason to its class:
substrate refusals (the daemon, the engine, custody or the run said no) are
`TOOL_REPORTED_FAILURE`, recorded and never degrading; malformed or
self-contradictory calls are `TOOL_ARG_ERROR`, which degrades and feeds
reflection. Neither is a timeout or the generic tool error, and an unclassified
reason defaults to the substrate class — the safe direction.

The second literal refusal author, `supervised_wait`'s checkpoint argument
check, folds into `_fail`. `_delegate_cancel`'s own outcomes join it: `failed`
and `containment_fault_run_may_still_be_live` report a run that may still be
live and mutating, so they publish as failures, while `confirmed` and
`requested` stay successful observations of the control surface.

Publication happens once, at the four REGISTERED entries, after every
decoration and immediately before the string is returned: `subagent_runtime`
mutates the start payload after `_delegate_start`, and an earlier publish fails
the registry's equality gate silently. `exact_start` now reads the native
payload, adds the actor identity and the work-order source, and returns
`_replace_tool_result`; its `json.loads → TypeError → return result` bypass,
which dropped that decoration without saying so, is deleted.
`_mark_actor_physical_start` still reads the DOMAIN `started` /
`started_uncustodied` status, never the host class.

The consumers migrate in the same change as the type they consume: the
configured-session bootstrap, the recovery handoff, the unknown-provider hold
and the pending-wake replay all read the producer's own payload rather than a
stringified result — without this, every leaf wake would have failed its
acknowledgement and taken the no-resend terminal. The wake envelope carries
`ok`/`host_code` through both fitted-spill shapes, so a refusal too large to
inline cannot read as a successful wait. `wait_once` deliberately keeps its
`str` tick contract, and `integrate_delegated_patch` keeps its own string ABI
at the one helper the two families share.

Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
2026-09-15 18:01:50 +03:00

625 lines
30 KiB
Python

"""Incident-shaped contracts for the unknown-provider hold (nanny-leaf D1-min).
A configured-session nanny whose metered round dies ``provider_outcome_unknown``
while EXACTLY one delegated leaf is alive must hold on the LEAF (zero provider
calls) and resume with a wake-bearing NEW round — the unknown request is never
resent. Control wakes keep their no-resend terminal. When no hold applies,
ordinary managed recovery can restore the supervising model.
"""
from __future__ import annotations
import json
import queue
import time
from types import SimpleNamespace
import pytest
import ouroboros.delegate_hold as delegate_hold
import ouroboros.loop as loop_mod
import ouroboros.loop_transport as transport
from ouroboros import delegate_custody as custody
from ouroboros.delegate_shared import _fail, delegate_result
from ouroboros.delegate_supervision import read_unknown_hold, write_unknown_hold
from ouroboros.loop import run_llm_loop
from ouroboros.tools.registry import ToolRegistry
def _wake(payload):
"""The NATIVE shape the supervising wait answers with (delegate_shared owns it)."""
return delegate_result(payload)
def _read_hold_events(tmp_path):
path = tmp_path / "events.jsonl"
if not path.exists():
return []
rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
return [row for row in rows if row.get("type") == "delegate_hold"]
def _configured_registry(tmp_path, task_id="t-hold"):
registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path)
registry._ctx.task_id = task_id
registry._ctx.exact_model_route = True
registry._ctx._configured_subagent_route_kind = "agent_session"
registry._ctx.task_metadata = {"configured_subagent": {"config_fingerprint": "fp"}}
return registry
def _start_leaf(tmp_path, task_id="t-hold", run_id="run-leaf"):
custody._CUSTODY.pop(run_id, None)
row = custody.RunCustody(run_id=run_id, task_id=task_id, route_id="claude", model="m")
assert custody.record_started(tmp_path, row)
return run_id
@pytest.fixture(autouse=True)
def _quiet_probe(monkeypatch):
"""Default leaf probe: read-only poll sees a live engine state; releases are
recorded, not executed."""
import ouroboros.claudexor_daemon as daemon_mod
import ouroboros.delegate_progress as progress_mod
monkeypatch.setattr(daemon_mod, "ensure_owned_gateway",
lambda **_k: SimpleNamespace(close=lambda: None), raising=False)
monkeypatch.setattr(
progress_mod, "bounded_poll",
lambda _gw, _run, _sec, **_k: {"summary": {"state": "running"}, "lastSeq": 1},
)
released = []
monkeypatch.setattr(custody, "release_task_runs", lambda root, tid: released.append(tid))
yield released
def _loop_kwargs(tmp_path, registry, notes):
return dict(
messages=[{"role": "user", "content": "supervise"}],
tools=registry,
llm=SimpleNamespace(default_model=lambda: "test-model"),
drive_logs=tmp_path,
emit_progress=lambda text, *, incident=None: notes.append(text),
incoming_messages=queue.Queue(),
task_id=str(registry._ctx.task_id),
drive_root=tmp_path,
)
def _unknown_then_check_call(check):
calls = {"n": 0}
def fake_call(_llm, messages, _model, _tools, _effort, _max_retries, _drive_logs,
_task_id, _round_idx, _event_queue, accumulated_usage, *_a, **_k):
calls["n"] += 1
if calls["n"] == 1:
accumulated_usage["_last_llm_error_kind"] = "provider_outcome_unknown"
accumulated_usage.update(execution_status="infra_failed", reason_code="llm_api_error")
return None, 0.0
return check(messages, accumulated_usage)
return fake_call, calls
def _recover_model(monkeypatch):
monkeypatch.setattr(transport, "upstream_transport_reachable",
lambda *a, **kw: {"kind": "upstream_http", "status_code": 200})
monkeypatch.setattr(transport, "interruptible_wait_sleep", lambda *a: False)
def recovered(_messages, usage):
usage.pop("_last_llm_error_kind", None)
return {"role": "assistant", "content": "recovered"}, 0.0
return recovered
def test_unknown_with_live_leaf_holds_and_resumes_with_wake(tmp_path, monkeypatch, _quiet_probe):
monkeypatch.setattr(transport, "upstream_transport_reachable",
lambda *a, **kw: pytest.fail("live hold must precede provider recovery"))
wake_payload = {"status": "succeeded", "run_id": "run-leaf", "supervision_wake_id": "w1"}
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda _ctx, _run: _wake(wake_payload))
acks = []
monkeypatch.setattr(delegate_hold, "acknowledge_pending_wake",
lambda _ctx, delivered=None: acks.append(delivered) or True)
def check(messages, accumulated_usage):
assert "[DELEGATED LEAF WAKE / UNKNOWN-HOLD RESUME]" in messages[-1]["content"]
assert "run-leaf" in messages[-1]["content"]
accumulated_usage.pop("_last_llm_error_kind", None)
return {"role": "assistant", "content": "integrated"}, 0.0
fake_call, calls = _unknown_then_check_call(check)
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
notes = []
result, usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes))
assert result == "integrated"
assert usage.get("reason_code") != "provider_unavailable"
assert calls["n"] == 2 # the unknown request itself was never resent
phases = [(row["phase"], row.get("detail", "")) for row in _read_hold_events(tmp_path)]
assert ("entered", "") in phases
assert any(p == "resumed" for p, _d in phases)
assert acks and acks[0]["supervision_wake_id"] == "w1"
assert not read_unknown_hold(registry._ctx).get("run_id") # inactive tombstone
assert _quiet_probe == ["t-hold"] # release only at the (successful) terminal
assert any("holding on the leaf" in note for note in notes)
assert not any(json.loads(line).get("type") == "network_wait"
for line in (tmp_path / "events.jsonl").read_text().splitlines())
assert "transport_recovery" not in usage
def test_terminal_leaf_never_enters_hold(tmp_path, monkeypatch, _quiet_probe):
import ouroboros.delegate_progress as progress_mod
monkeypatch.setattr(
progress_mod, "bounded_poll",
lambda _gw, _run, _sec, **_k: {"summary": {"state": "succeeded"}},
)
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda *_a, **_k: pytest.fail("terminal leaf must not hold"))
fake_call, calls = _unknown_then_check_call(_recover_model(monkeypatch))
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
notes = []
result, usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes))
assert calls["n"] == 2 and result == "recovered"
assert usage["transport_recovery"]["old_outcome"] == "unknown"
assert _read_hold_events(tmp_path) == []
def test_control_wake_exits_through_no_call_terminal(tmp_path, monkeypatch, _quiet_probe):
monkeypatch.setattr(
delegate_hold, "supervised_wait",
lambda _ctx, _run: _wake({
"status": "progress",
"wake_events": [{"type": "cancellation_intent"}],
"supervision_wake_id": "w2",
}),
)
fake_call, calls = _unknown_then_check_call(lambda *_: pytest.fail("no dial after Stop"))
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
notes = []
_r, usage, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes))
assert calls["n"] == 1 # zero provider calls after the control wake
assert usage.get("execution_status") == "infra_failed"
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "control_wake" in details
assert _quiet_probe == ["t-hold"] # the terminal cleanup owns the leaf now
def test_finalize_now_mid_hold_takes_no_call_terminal(tmp_path, monkeypatch, _quiet_probe):
from ouroboros.owner_mailbox import KIND_FINALIZE_NOW, write_owner_message
def waiting_forever(_ctx, _run):
time.sleep(30)
pytest.fail("supervised_wait should have been pre-empted by finalize_now")
# finalize_now lands BETWEEN the failing round and the next round top, so
# the latched hold sees it in controls before any wait starts.
monkeypatch.setattr(delegate_hold, "supervised_wait", waiting_forever)
def check(_messages, _usage):
pytest.fail("no dial")
fake_call, calls = _unknown_then_check_call(check)
orig_fake = fake_call
def fake_with_mailbox(*args, **kwargs):
out = orig_fake(*args, **kwargs)
if calls["n"] == 1:
write_owner_message(tmp_path, "wrap up", "t-hold", kind=KIND_FINALIZE_NOW)
return out
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_with_mailbox)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
notes = []
_r, usage, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes))
assert calls["n"] == 1
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "finalize_now" in details
def test_generic_task_and_multi_run_never_hold(tmp_path, monkeypatch, _quiet_probe):
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda *_a, **_k: pytest.fail("ineligible shapes must not hold"))
fake_call, calls = _unknown_then_check_call(_recover_model(monkeypatch))
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
# Generic tasks use the network owner, never the single-leaf nanny hold.
registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path)
registry._ctx.task_id = "t-generic"
_start_leaf(tmp_path, task_id="t-generic", run_id="run-g")
result, usage, _t = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert calls["n"] == 2 and result == "recovered"
assert usage["transport_recovery"]["old_outcome"] == "unknown"
# Configured but TWO live leaves.
calls["n"] = 0
registry2 = _configured_registry(tmp_path, task_id="t-multi")
_start_leaf(tmp_path, task_id="t-multi", run_id="run-m1")
_start_leaf(tmp_path, task_id="t-multi", run_id="run-m2")
result, usage2, _t = run_llm_loop(**_loop_kwargs(tmp_path, registry2, []))
assert calls["n"] == 2 and result == "recovered"
assert usage2["transport_recovery"]["old_outcome"] == "unknown"
assert _read_hold_events(tmp_path) == []
def test_recovered_latch_reenters_hold_before_any_dispatch(tmp_path, monkeypatch, _quiet_probe):
"""The durable latch (worker-crash adoption) parks the successor's FIRST
round in the hold before any LLM call — sol #5 contract."""
wake_payload = {"status": "attention", "run_id": "run-leaf", "supervision_wake_id": "w3"}
order = []
monkeypatch.setattr(
delegate_hold, "supervised_wait",
lambda _ctx, _run: order.append("wait") or _wake(wake_payload),
)
monkeypatch.setattr(delegate_hold, "acknowledge_pending_wake", lambda *_a, **_k: True)
def fake_call(_llm, messages, *_a, **_k):
order.append("dispatch")
assert "[DELEGATED LEAF WAKE / UNKNOWN-HOLD RESUME]" in messages[-1]["content"]
return {"role": "assistant", "content": "resumed"}, 0.0
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
write_unknown_hold(registry._ctx, "run-leaf", {
"run_id": "run-leaf", "entered_at": "2026-08-30T00:00:00Z", "hold_cycles": 1,
})
result, _u, _t = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert result == "resumed"
assert order == ["wait", "dispatch"]
def test_repeated_unknown_reholds_with_backoff_floor(tmp_path, monkeypatch, _quiet_probe):
wake_payload = {"status": "progress_report", "run_id": "run-leaf", "supervision_wake_id": "w4"}
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda _ctx, _run: _wake(wake_payload))
monkeypatch.setattr(delegate_hold, "acknowledge_pending_wake", lambda *_a, **_k: True)
sleeps = []
monkeypatch.setattr(delegate_hold.time, "sleep", lambda sec: sleeps.append(sec))
calls = {"n": 0}
def fake_call(_llm, _messages, _model, _tools, _effort, _max_retries, _drive_logs,
_task_id, _round_idx, _event_queue, accumulated_usage, *_a, **_k):
calls["n"] += 1
if calls["n"] <= 2:
accumulated_usage["_last_llm_error_kind"] = "provider_outcome_unknown"
accumulated_usage.update(execution_status="infra_failed", reason_code="llm_api_error")
return None, 0.0
accumulated_usage.pop("_last_llm_error_kind", None)
return {"role": "assistant", "content": "done"}, 0.0
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
result, _u, _t = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert result == "done"
assert calls["n"] == 3
cycle_counts = [row["hold_cycles"] for row in _read_hold_events(tmp_path)
if row["phase"] == "entered"]
assert cycle_counts == [1, 2]
# The patch is on the GLOBAL time.sleep, so a daemon thread leaked by an earlier test on the
# same xdist worker (a 0.5 s poll loop; windows-latest, rc.13 dispatch) lands in ``sleeps``
# too: pin the backoff floor of the second cycle by presence, not by position.
assert any(4.0 <= sec <= 15.0 for sec in sleeps), sleeps # backoff floor on the second cycle
def test_refused_probe_and_state_less_payload_never_hold(tmp_path, monkeypatch, _quiet_probe):
"""A daemon refusal or a state-less payload is not evidence of a live leaf
and does not prohibit recovery of the supervising model."""
import ouroboros.delegate_progress as progress_mod
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda *_a, **_k: pytest.fail("refused probe must not hold"))
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
def raising_poll(_gw, _run, _sec, **_k):
raise RuntimeError("daemon unreachable")
monkeypatch.setattr(progress_mod, "bounded_poll", raising_poll)
fake_call, calls = _unknown_then_check_call(_recover_model(monkeypatch))
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
registry = _configured_registry(tmp_path, task_id="t-refused")
_start_leaf(tmp_path, task_id="t-refused", run_id="run-r1")
result, usage, _t = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert calls["n"] == 2 and result == "recovered"
assert usage["transport_recovery"]["old_outcome"] == "unknown"
monkeypatch.setattr(progress_mod, "bounded_poll", lambda _gw, _run, _sec, **_k: {})
calls["n"] = 0
registry2 = _configured_registry(tmp_path, task_id="t-stateless")
_start_leaf(tmp_path, task_id="t-stateless", run_id="run-r2")
result, usage2, _t = run_llm_loop(**_loop_kwargs(tmp_path, registry2, []))
assert calls["n"] == 2 and result == "recovered"
assert usage2["transport_recovery"]["old_outcome"] == "unknown"
assert _read_hold_events(tmp_path) == []
def test_refused_wait_takes_terminal_not_paid_resume(tmp_path, monkeypatch, _quiet_probe):
"""A refused/fault wait status is a daemon statement, not a leaf wake
(fable F3): no paid resume round is bought on it."""
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda _ctx, _run: _fail("delegate_wait", "daemon_unreachable",
"the daemon answered nothing"))
fake_call, calls = _unknown_then_check_call(lambda *_: pytest.fail("no dial on refusal"))
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
_r, usage, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert calls["n"] == 1
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "wait_refused" in details
def test_ack_failure_fails_closed_without_dispatch(tmp_path, monkeypatch, _quiet_probe):
"""One wake = one dispatch (sol CRITICAL #1): a wake that cannot be durably
acknowledged is never dispatched — the appended receipt is removed and the
task takes the honest no-resend terminal."""
wake_payload = {"status": "succeeded", "run_id": "run-leaf", "supervision_wake_id": "w5"}
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda _ctx, _run: _wake(wake_payload))
monkeypatch.setattr(delegate_hold, "acknowledge_pending_wake", lambda *_a, **_k: False)
monkeypatch.setattr(delegate_hold.time, "sleep", lambda _s: None)
seen_messages = []
def fake_call(_llm, messages, _model, _tools, _effort, _max_retries, _drive_logs,
_task_id, _round_idx, _event_queue, accumulated_usage, *_a, **_k):
seen_messages.append(list(messages))
accumulated_usage["_last_llm_error_kind"] = "provider_outcome_unknown"
accumulated_usage.update(execution_status="infra_failed", reason_code="llm_api_error")
return None, 0.0
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
_r, _u, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert len(seen_messages) == 1 # only the original unknown round dialed
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "ack_failed" in details
def test_owner_input_resumes_without_wait(tmp_path, monkeypatch, _quiet_probe):
"""An owner message drained at the round top IS material new input (sol
HIGH #4): the hold resumes on it without entering supervised_wait."""
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda *_a, **_k: pytest.fail("owner input must resume without waiting"))
def fake_call(_llm, messages, *_a, **_k):
assert any("please integrate" in str(m.get("content")) for m in messages)
return {"role": "assistant", "content": "resumed-on-owner-input"}, 0.0
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
write_unknown_hold(registry._ctx, "run-leaf", {
"run_id": "run-leaf", "entered_at": "2026-08-30T00:00:00Z", "hold_cycles": 1,
})
kwargs = _loop_kwargs(tmp_path, registry, [])
# The transcript tail is LONGER than the owner message: an appended short
# message must still read as new input (final-pair fable F1 — a naive
# length-sum signature missed exactly this common shape).
kwargs["messages"] = [
{"role": "user", "content": "supervise"},
{"role": "assistant", "content": "long tool result " * 50},
]
kwargs["incoming_messages"].put("please integrate")
result, _u, _t = run_llm_loop(**kwargs)
assert result == "resumed-on-owner-input"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "resumed"]
assert "owner_input" in details
def test_recovered_latch_control_wake_stays_no_call(tmp_path, monkeypatch, _quiet_probe):
"""Fable F1: after crash recovery the usage record is fresh — a control
wake must still exit through the no-call unknown terminal, never a paid
[PROVIDER_UNAVAILABLE] forced final."""
monkeypatch.setattr(
delegate_hold, "supervised_wait",
lambda _ctx, _run: _wake({
"status": "progress", "wake_events": [{"type": "cancellation_intent"}],
}),
)
monkeypatch.setattr(loop_mod, "call_llm_with_retry",
lambda *_a, **_k: pytest.fail("Stop after recovery must not dial"))
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
write_unknown_hold(registry._ctx, "run-leaf", {
"run_id": "run-leaf", "entered_at": "2026-08-30T00:00:00Z", "hold_cycles": 1,
})
_r, _u, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
def test_round_limit_with_live_hold_takes_no_call_terminal(tmp_path, monkeypatch, _quiet_probe):
"""Sol CRITICAL #2 / fable F2: an unknown on the last legal round must not
buy a paid [ROUND_LIMIT] dial — the hold closes into the no-call unknown
terminal and the latch does not dangle."""
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda *_a, **_k: pytest.fail("round-limit hold must not wait"))
calls = {"n": 0}
def fake_call(_llm, _messages, _model, _tools, _effort, _max_retries, _drive_logs,
_task_id, _round_idx, _event_queue, accumulated_usage, *_a, **_k):
calls["n"] += 1
accumulated_usage["_last_llm_error_kind"] = "provider_outcome_unknown"
accumulated_usage.update(execution_status="infra_failed", reason_code="llm_api_error")
return None, 0.0
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.setenv("OUROBOROS_MAX_ROUNDS", "1")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path)
_start_leaf(tmp_path)
_r, _u, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert calls["n"] == 1 # the [ROUND_LIMIT] wrap-up never dialed
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "round_limit" in details
assert not read_unknown_hold(registry._ctx) # no stale latch left behind
def test_latch_survives_real_supervised_wait_state_reset(tmp_path, monkeypatch, _quiet_probe):
"""Final-pair CRITICAL (sol #1 / fable F2): the REAL supervised_wait's
_load_state rebuild for a new run id must carry the durable latch — a
worker crash mid-wait must find it, or the successor resends."""
import ouroboros.delegate_supervision as sup
registry = _configured_registry(tmp_path, task_id="t-reset")
_start_leaf(tmp_path, task_id="t-reset", run_id="run-reset")
write_unknown_hold(registry._ctx, "run-reset", {
"run_id": "run-reset", "entered_at": "2026-08-30T00:00:00Z", "hold_cycles": 1,
})
def wait_once(_ctx, _run, _sec, _seq):
# Mid-wait crash shape: the durable file must STILL carry the latch
# after supervised_wait's entry persisted its (rebuilt) state.
data = json.loads(sup._state_path(registry._ctx).read_text())
assert data.get("unknown_provider_hold", {}).get("run_id") == "run-reset"
return json.dumps({"status": "succeeded", "run_id": "run-reset", "last_seq": 2})
raw = sup.supervised_wait(registry._ctx, "run-reset", wait_once=wait_once)
assert json.loads(raw.text).get("status") == "succeeded"
assert (raw.status, raw.code) == ("ok", "OK") # a leaf wake is a successful observation
assert read_unknown_hold(registry._ctx).get("run_id") == "run-reset"
def test_unreadable_latch_fails_closed_to_terminal(tmp_path, monkeypatch, _quiet_probe):
"""Final-pair sol #2: an existing-but-corrupt latch file must not read as
'no hold' — dispatching there could resend the unknown request."""
import ouroboros.delegate_supervision as sup
registry = _configured_registry(tmp_path, task_id="t-corrupt")
_start_leaf(tmp_path, task_id="t-corrupt", run_id="run-c")
path = sup._state_path(registry._ctx)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("{corrupt json", encoding="utf-8")
monkeypatch.setattr(delegate_hold, "supervised_wait",
lambda *_a, **_k: pytest.fail("corrupt latch must not wait"))
monkeypatch.setattr(loop_mod, "call_llm_with_retry",
lambda *_a, **_k: pytest.fail("corrupt latch must not dispatch"))
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
_r, _u, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, []))
assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend"
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "latch_unreadable" in details
def test_eligibility_probe_closes_its_gateway(tmp_path, monkeypatch, _quiet_probe):
"""Final-pair sol #5 / fable F3: the probe owns close() on the gateway."""
import ouroboros.claudexor_daemon as daemon_mod
closed = []
class _Gw:
def close(self):
closed.append(True)
monkeypatch.setattr(daemon_mod, "ensure_owned_gateway",
lambda **_k: _Gw(), raising=False)
registry = _configured_registry(tmp_path, task_id="t-gw")
_start_leaf(tmp_path, task_id="t-gw", run_id="run-gw")
assert delegate_hold._single_live_run(registry._ctx) == "run-gw"
assert closed == [True]
def test_transport_dead_observation_keeps_the_hold_instead_of_a_refused_exit(
tmp_path, monkeypatch, _quiet_probe,
):
"""A dead socket during the hold is a quiet renewal with the typed reason
``daemon_unreachable`` (never a ``refused`` wait), so ``_NON_WAKE_STATUSES``
must not take the no-resend terminal: the hold rides out the outage and resumes
on the leaf's real wake. Drives the REAL supervising wait over a scripted daemon."""
import ouroboros.delegate_progress as progress_mod
import ouroboros.delegate_supervision as supervision_mod
from ouroboros.gateways import claudexor as gateway_module
monkeypatch.setattr(supervision_mod.time, "sleep", lambda _sec: None)
polls = []
def scripted_poll(_gw, _run, _sec, **_k):
polls.append(1)
if len(polls) == 1: # the hold's own liveness probe: the leaf is alive
return {"summary": {"state": "running", "effectiveAccess": "readonly"}, "lastSeq": 1}
if len(polls) == 2: # first supervised tick: the daemon socket is dead
raise gateway_module.ClaudexorUnavailable(
"daemon_unreachable", "ConnectError: [Errno 61]", observation_timeout=True)
return {"lastSeq": 2, "summary": {
"state": "succeeded", "effectiveAccess": "readonly", "runDir": str(tmp_path / "run"),
}, "primaryOutput": {"kind": "answer", "text": "leaf result", "truncated": False}}
class _Gateway:
engine_version = ""
def handshake(self, **_kw):
self.engine_version = "3.10.2"
return {}
def close(self):
pass
monkeypatch.setattr(progress_mod, "bounded_poll", scripted_poll)
monkeypatch.setattr(gateway_module, "ClaudexorGateway", lambda: _Gateway())
def check(messages, accumulated_usage):
assert "[DELEGATED LEAF WAKE / UNKNOWN-HOLD RESUME]" in messages[-1]["content"]
accumulated_usage.pop("_last_llm_error_kind", None)
return {"role": "assistant", "content": "integrated"}, 0.0
fake_call, calls = _unknown_then_check_call(check)
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call)
monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off")
monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False)
registry = _configured_registry(tmp_path, task_id="t-dead-socket")
_start_leaf(tmp_path, task_id="t-dead-socket", run_id="run-dead-socket")
notes = []
result, _usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes))
assert result == "integrated" and calls["n"] == 2
assert len(polls) == 3
details = [row.get("detail") for row in _read_hold_events(tmp_path) if row["phase"] == "ended"]
assert "wait_refused" not in details and "wait_observation_pending" not in details
assert [row["phase"] for row in _read_hold_events(tmp_path)] == ["entered", "resumed"]