ouroboros/tests/test_retry_project_binding.py
Ouroboros 913034c60e Bind a timeout retry to its predecessor's Project at admission
A Project-bound root that timed out was retried under a new physical id
which copied the owner-message origin ref but carried NO durable binding,
so the retried work painted a Main card offering "Turn into project" — a
second convertible unit for one piece of work — until some later implicit
act adopted it. docs/DEVELOPMENT.md described that as deliberate; it was
the #900 implementer's size-cap deferral, and the owner accepted that a
retry inherits the old id's binding on 2026-09-14.

The reaper now binds the successor inside its own retry admission
transaction (task_reaper._run_retry_admission_transaction ->
worker_promotion.bind_retry_to_origin_project), so admission and bind are
one step:

- the claim lock moves OUTSIDE the queue lock, the order an implicit claim
  already uses (project_id_for_origin's live-task tie-break takes the
  queue lock under it), so nothing inverts and a concurrent conversion of
  the same owner message cannot interleave;
- the bind lands only after cancellation has lost the boundary. A binding
  is immutable, so a bound-but-never-admitted retry id would answer
  project_id_for_task forever; a retry suppressed by a cancelled or
  already-terminal root is therefore never bound;
- the predecessor's own binding answers first (the SSOT for that task's
  project), then the origin-keyed lookup for a root that was never bound
  itself while another task id of the same message was; the new row reuses
  the predecessor's stored origin BY VALUE, so the retry joins that
  message's one convertible unit instead of starting a second;
- a refused bind or an unreadable store leaves the retry unbound, admits
  it anyway, and discloses project_binding_failed on the reaper's own
  drive (P1) instead of holding up the retry.

supervisor/task_reaper.py sat exactly on the 1600-line hard cap, so the
call site was paid for in place: "the retry id never ran, drop its copied
inputs" existed three times in _enqueue_retry and is now one
_discard_retry_inputs owner (1600 -> 1598 lines).

Docs flip with the behaviour: docs/DEVELOPMENT.md (identity-of-the-work
section) and docs/ARCHITECTURE.md (the origin membership list and the
implicit-claim list now name the reaper's retry admission).

Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
2026-09-15 17:01:48 +03:00

246 lines
10 KiB
Python

"""A timeout retry keeps the Project its predecessor's work already has.
The incident: a Project-bound root that timed out was retried under a new physical
id which copied the owner-message origin ref but carried no durable binding, so the
retried work painted a Main card offering "Turn into project" — a second convertible
unit for one piece of work — until some later implicit act adopted it.
The reaper now binds the successor inside its own admission transaction
(``task_reaper._run_retry_admission_transaction`` ->
``worker_promotion.bind_retry_to_origin_project``), under the claim lock every
implicit claim holds, and only once cancellation has lost that boundary: a binding
is immutable, so a bound-but-never-admitted retry id would answer
``project_id_for_task`` forever.
"""
from __future__ import annotations
import json
import types
from ouroboros import cancel_intents as ci
from ouroboros.contracts.chat_id_policy import project_chat_id
from ouroboros.project_dialogue import build_owner_message_ref
from ouroboros.projects_registry import (
all_task_project_bindings,
begin_project_deletion,
bind_task_to_project,
create_project,
project_binding_for_task,
project_id_for_task,
)
from ouroboros.task_results import STATUS_CANCELLED, STATUS_RUNNING, write_task_result
from tests._cancel_intents_shared import qenv as _qenv
qenv = _qenv
OWNER_TEXT = "the importer keeps timing out, please fix it"
def _owner_ref(client_message_id: str = "msg-retry-1", chat_id: int = 1) -> dict:
return build_owner_message_ref(
chat_id=chat_id,
client_message_id=client_message_id,
ts="2026-09-15T09:30:00+00:00",
text=OWNER_TEXT,
)
def _patch_retry_input_handoff(monkeypatch):
"""Attachment/mailbox carry-over is exercised by the reaper's own suites; these
tests are about what the admission transaction binds."""
monkeypatch.setattr(
"ouroboros.artifacts.handoff_task_attachments_for_retry",
lambda *_args, **_kwargs: ({}, ""),
)
monkeypatch.setattr(
"ouroboros.owner_mailbox.copy_owner_mailbox_for_retry",
lambda *_args, **_kwargs: True,
)
def _root_task(task_id: str, *, ref: dict | None = None) -> dict:
"""A pooled root exactly as the queue carries it. A root converted post-hoc
("Turn into project") carries NO ``project_id`` on its row — the durable
binding is the only truth about its Project, and it is the shape the retry
used to lose.
"""
task = {
"id": task_id,
"type": "task",
"chat_id": 1,
"depth": 0,
"root_task_id": task_id,
"parent_task_id": "",
"delegation_role": "root",
}
if ref is not None:
task["origin_message_ref"] = dict(ref)
task["origin_message_text"] = OWNER_TEXT
return task
def _bind_root(drive, task_id: str, pid: str, *, ref: dict | None) -> dict:
create_project(drive, pid, name="Importer room", origin="owner_ui")
origin = (
{"ref": dict(ref), "text": OWNER_TEXT} if ref is not None
else {"absent": "mid_task_no_origin"}
)
return bind_task_to_project(drive, task_id, pid, origin=origin)
def _retry(qenv, task: dict, old_id: str, new_id: str):
from supervisor import task_reaper as tr
return tr._enqueue_retry(
qenv.q,
task,
task_id=old_id,
retry_task_id=new_id,
attempt=1,
terminal_reason="idle_timeout",
recon_fields={},
)
def _events(drive) -> list[dict]:
path = drive / "logs" / "events.jsonl"
if not path.exists():
return []
return [
json.loads(line)
for line in path.read_text(encoding="utf-8").splitlines() if line.strip()
]
def test_retry_of_a_bound_root_is_in_the_project_before_its_first_round(qenv, monkeypatch):
"""(a) The successor is bound while it is still PENDING — no worker has picked
it up, so the owner never sees the retried work as an unclaimed Main card."""
_patch_retry_input_handoff(monkeypatch)
old_id, new_id, pid = "bound-old", "bound-new", "importer-room"
ref = _owner_ref()
root = _root_task(old_id, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_RUNNING, result="working")
predecessor = _bind_root(qenv.drive, old_id, pid, ref=ref)
requeued, new_attempt, _reason, suppression = _retry(qenv, root, old_id, new_id)
assert (requeued, new_attempt, suppression) == (True, 2, {})
assert [row["id"] for row in qenv.q.PENDING] == [new_id]
assert qenv.q.RUNNING == {}
assert project_id_for_task(qenv.drive, new_id) == pid
# The UI census (/api/state) is what paints the card's room.
assert all_task_project_bindings(qenv.drive)[new_id] == {
"project_id": pid, "chat_id": project_chat_id(pid),
}
# The retry joins the SAME owner-message unit: its origin is the predecessor's
# stored one, carried by value rather than re-derived.
binding = project_binding_for_task(qenv.drive, new_id)
assert binding["source_ref"] == predecessor["source_ref"]
assert binding["source_text"] == predecessor["source_text"]
def test_retry_adopts_the_project_a_sibling_of_the_same_owner_message_holds(qenv, monkeypatch):
"""The origin-keyed half: the timed-out root was never bound itself, but the
direct turn that received the same owner message was. One message, one room."""
_patch_retry_input_handoff(monkeypatch)
old_id, new_id, pid = "adopt-old", "adopt-new", "sibling-room"
ref = _owner_ref("msg-retry-sibling")
root = _root_task(old_id, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_RUNNING, result="working")
_bind_root(qenv.drive, "turn-that-received-it", pid, ref=ref)
assert project_id_for_task(qenv.drive, old_id) == ""
requeued, _attempt, _reason, suppression = _retry(qenv, root, old_id, new_id)
assert (requeued, suppression) == (True, {})
assert project_id_for_task(qenv.drive, new_id) == pid
def test_retry_of_a_cancelled_root_is_never_bound(qenv, monkeypatch):
"""(b) A root that already settled as cancelled has no successor to place: the
admission transaction suppresses the retry, so nothing durable claims its id."""
_patch_retry_input_handoff(monkeypatch)
old_id, new_id, pid = "cancelled-old", "cancelled-new", "cancelled-room"
ref = _owner_ref("msg-retry-cancelled")
root = _root_task(old_id, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_RUNNING, result="working")
_bind_root(qenv.drive, old_id, pid, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_CANCELLED, result="owner stopped it")
requeued, _attempt, reason, suppression = _retry(qenv, root, old_id, new_id)
assert requeued is False
assert reason == "terminal_result_retry_suppressed"
assert suppression["kind"] == "terminal_result"
assert qenv.q.PENDING == []
assert project_binding_for_task(qenv.drive, new_id) is None
assert new_id not in all_task_project_bindings(qenv.drive)
# The predecessor keeps its own room; only the successor is absent.
assert project_id_for_task(qenv.drive, old_id) == pid
def test_cancellation_winning_the_admission_leaves_no_retry_binding(qenv, monkeypatch):
"""(c) An intent recorded before the admission boundary suppresses the successor
inside the same locked transition, so the immutable bind never lands."""
_patch_retry_input_handoff(monkeypatch)
old_id, new_id, pid = "race-old", "race-new", "race-room"
ref = _owner_ref("msg-retry-race")
root = _root_task(old_id, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_RUNNING, result="working")
_bind_root(qenv.drive, old_id, pid, ref=ref)
ci.request_cancel(qenv.drive, old_id, reason="stop before retry")
requeued, _attempt, reason, suppression = _retry(qenv, root, old_id, new_id)
assert requeued is False
assert reason == "cancel_pending_retry_suppressed"
assert suppression == {"kind": "cancel_intent", "target": old_id}
assert qenv.q.PENDING == []
assert project_binding_for_task(qenv.drive, new_id) is None
def test_first_implicit_promote_from_a_retry_lands_in_the_origins_project(qenv, monkeypatch):
"""(d) What the owner actually sees next: work the retry promotes with no
explicit target goes to the room, not to Main."""
from ouroboros.tools.control_routing import _inherited_project_scope
_patch_retry_input_handoff(monkeypatch)
old_id, new_id, pid = "promote-old", "promote-new", "promote-room"
ref = _owner_ref("msg-retry-promote")
root = _root_task(old_id, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_RUNNING, result="working")
_bind_root(qenv.drive, old_id, pid, ref=ref)
monkeypatch.setattr("ouroboros.config.DATA_DIR", qenv.drive)
requeued, _attempt, _reason, _suppression = _retry(qenv, root, old_id, new_id)
assert requeued is True
# The retry worker's own scope copy is empty (a post-hoc conversion never
# reached the dead attempt), so only the durable binding can answer.
ctx = types.SimpleNamespace(task_id=new_id, task_metadata={}, project_id="")
assert _inherited_project_scope(ctx) == pid
def test_a_room_that_stopped_accepting_bindings_discloses_and_admits_the_retry(qenv, monkeypatch):
"""A refused bind is LOUD but never blocks the retry: the successor is still
admitted, and the failure is a typed row on the reaper's OWN drive."""
_patch_retry_input_handoff(monkeypatch)
old_id, new_id, pid = "fenced-old", "fenced-new", "fenced-room"
ref = _owner_ref("msg-retry-fenced")
root = _root_task(old_id, ref=ref)
write_task_result(qenv.drive, old_id, STATUS_RUNNING, result="working")
_bind_root(qenv.drive, old_id, pid, ref=ref)
begin_project_deletion(qenv.drive, pid)
requeued, _attempt, _reason, suppression = _retry(qenv, root, old_id, new_id)
assert (requeued, suppression) == (True, {})
assert [row["id"] for row in qenv.q.PENDING] == [new_id]
assert project_binding_for_task(qenv.drive, new_id) is None
failures = [
row for row in _events(qenv.drive)
if row.get("type") == "project_binding_failed" and row.get("task_id") == new_id
]
assert [row["bind_path"] for row in failures] == ["timeout_retry_admission"]
assert failures[0]["project_id"] == pid