ouroboros/tests/test_loop_compaction.py
Ouroboros d5b4676b6b feat(loop): per-class transient LLM retry, encrypted-reasoning strip-retry, robust context compaction (v6.28.0)
Motivated by benchmark forensics (terminal-bench/SWE-bench/GAIA): most
trial deaths were harness infrastructure, not agent inability. All three
sub-blocks are general-purpose robustness for normal users.

1a. Per-class same-model transient retry (loop_llm_call.py):
- Transient classes (finish_reason=null / empty-response shapes,
  provider_transient 429/5xx/overloaded) retry the SAME model with a
  larger budget: transient_retry_max() (OUROBOROS_TRANSIENT_RETRY_MAX,
  SSOT default 6 in SETTINGS_DEFAULTS + apply_settings_to_env, floored
  at the caller budget), exponential backoff capped 60s.
- Backoff sleeps are deadline-bounded (task_metadata.deadline_at threaded
  as deadline_ts through the main loop, budget-limit and round-limit
  wrap-up calls); stopping emits a durable llm_retry_deadline_exhausted
  event from BOTH transient paths.
- Permanent classes (auth/quota/bad_request/request_too_large) fail fast
  unchanged. NO cross-model fallback is introduced: single-model setups
  (all slots one model, empty fallback) die only after the real budget,
  and the failure text reports actual attempts used.

1b. Encrypted-reasoning strip-retry (llm.py):
- _is_openrouter_signature_error also matches "encrypted reasoning",
  "encrypted content for item" (observed gpt-5 shape "...for item
  rs_..."), "reasoning item", "reasoning_details" - reusing the existing
  one-shot roundtrip-metadata strip-and-retry on the same model.
  The allow_fallbacks pin is untouched.

1c. Compaction robustness (context_compaction.py + loop.py):
- Per-batch isolation: a failed batch leaves only its own rounds raw;
  the old whole-pass try/except discarded every successful summary.
- Per-round degradation: a missing summary leaves that round raw instead
  of the all-or-nothing completeness ValueError.
- Structured emit_round_summaries tool protocol (tool_choice=required,
  reliable round_id keying) with text-protocol fallback for local light
  models or prose answers; spend from failed batches is accounted
  (_BatchSummaryError carries usage, including across fallback failures).
- Warning protection scans the first two non-empty lines (autocorrect
  notes can prefix the marker); SHELL_EXIT_ERROR rounds are deliberately
  compactable - trial-and-error history must compact, with the first
  error line preserved by summarizer instruction.
- Emergency compaction adapts keep_recent to
  min(50, max(6, spans//2), max(1, spans-1)) so oversized transcripts
  with few huge rounds actually compact instead of no-opping.

Review of record: triad+scope rounds 1-4 via run_external_review.py;
round 4 blocked=False with zero criticals (scope fable-5 responded,
851,542 real tokens). Remaining advisory (param count on two
pre-existing over-limit signatures) is documented pre-existing debt;
context-object consolidation is out of block scope.

Carriers: VERSION, pyproject.toml, web/package.json, api_types.js
GATEWAY_CONTRACT_VERSION, README badge+history (oldest minor row
trimmed per P9 cap), ARCHITECTURE.md header + retry/compaction docs.
2026-06-12 17:45:28 +03:00

142 lines
5.3 KiB
Python

from types import SimpleNamespace
def _messages(count=41):
return [{"role": "assistant", "content": f"msg-{idx}"} for idx in range(count)]
def test_routine_compaction_runs_for_low_remote_but_not_max_remote(monkeypatch, tmp_path):
from ouroboros import loop
calls = []
def fake_checkpoint(messages, **kwargs):
calls.append(("checkpoint", kwargs["reason"], kwargs["keep_recent"]))
return True
def fake_compact(messages, keep_recent, **kwargs):
calls.append(("compact", keep_recent, kwargs.get("drive_root"), kwargs.get("task_id")))
return [{"role": "system", "content": "compacted"}], {"prompt_tokens": 1}
monkeypatch.setattr(loop, "_persist_compaction_checkpoint", fake_checkpoint)
monkeypatch.setattr(loop, "compact_tool_history_llm", fake_compact)
base = dict(
tools=SimpleNamespace(_ctx=SimpleNamespace(_pending_compaction=None)),
drive_root=tmp_path,
drive_logs=tmp_path / "logs",
task_id="task-1",
round_idx=7,
event_queue=None,
checkpoint_injected=False,
emit_progress=lambda _msg: None,
)
low_messages, low_usage = loop._run_round_compaction(
_messages(),
loop._CompactionRoundContext(active_use_local=False, active_context_mode="low", **base),
)
assert low_messages == [{"role": "system", "content": "compacted"}]
assert low_usage == {"prompt_tokens": 1}
assert calls == [("checkpoint", "routine", 20), ("compact", 20, tmp_path, "task-1")]
calls.clear()
max_messages, max_usage = loop._run_round_compaction(
_messages(),
loop._CompactionRoundContext(active_use_local=False, active_context_mode="max", **base),
)
assert len(max_messages) == 41
assert max_usage is None
assert calls == []
local_messages, local_usage = loop._run_round_compaction(
_messages(),
loop._CompactionRoundContext(active_use_local=True, active_context_mode="max", **base),
)
assert local_messages == [{"role": "system", "content": "compacted"}]
assert local_usage == {"prompt_tokens": 1}
def test_emergency_compaction_shrinks_keep_recent_to_span_count(monkeypatch, tmp_path):
"""Emergency compaction must pass keep_recent BELOW the span count or the
compactor no-ops exactly when the transcript is too big (<=50 huge rounds
over the byte threshold never compacted at all)."""
from ouroboros import loop
calls = []
def fake_checkpoint(messages, **kwargs):
calls.append(("checkpoint", kwargs["reason"], kwargs["keep_recent"]))
return True
def fake_compact(messages, keep_recent, **kwargs):
calls.append(("compact", keep_recent))
return [{"role": "system", "content": "compacted"}], None
monkeypatch.setattr(loop, "_persist_compaction_checkpoint", fake_checkpoint)
monkeypatch.setattr(loop, "compact_tool_history_llm", fake_compact)
monkeypatch.setattr(loop, "_estimate_messages_chars", lambda _m: 10**9)
# 30 tool rounds -> emergency keep_recent must be 15 (30 // 2), not 50.
messages = []
for i in range(30):
messages.append({
"role": "assistant", "content": f"r{i}",
"tool_calls": [{"id": f"c{i}", "function": {"name": "x", "arguments": "{}"}}],
})
messages.append({"role": "tool", "tool_call_id": f"c{i}", "content": "ok"})
ctx = loop._CompactionRoundContext(
tools=SimpleNamespace(_ctx=SimpleNamespace(_pending_compaction=None)),
drive_root=tmp_path,
drive_logs=tmp_path / "logs",
task_id="task-em",
round_idx=3,
event_queue=None,
active_use_local=False,
active_context_mode="max",
checkpoint_injected=False,
emit_progress=lambda _msg: None,
)
compacted, _usage = loop._run_round_compaction(messages, ctx)
assert compacted == [{"role": "system", "content": "compacted"}]
assert calls == [("checkpoint", "emergency_context_size", 15), ("compact", 15)]
# Few huge rounds (<= 6 spans): keep_recent clamps BELOW the span count so
# the compactor's len(spans) <= keep_recent gate cannot no-op forever.
calls.clear()
small = []
for i in range(4):
small.append({
"role": "assistant", "content": f"r{i}",
"tool_calls": [{"id": f"s{i}", "function": {"name": "x", "arguments": "{}"}}],
})
small.append({"role": "tool", "tool_call_id": f"s{i}", "content": "huge"})
loop._run_round_compaction(small, ctx)
assert calls == [("checkpoint", "emergency_context_size", 3), ("compact", 3)]
def test_context_compaction_observability_uses_current_task_drive(monkeypatch, tmp_path):
from ouroboros import context_compaction
from ouroboros import llm_observability
seen = {}
def fake_chat_observed(_client, **kwargs):
seen.update(kwargs)
return {"content": "[round:1]\nsummary"}, {"prompt_tokens": 1}
monkeypatch.setattr(llm_observability, "chat_observed", fake_chat_observed)
monkeypatch.setattr(context_compaction, "LLMClient", lambda: object(), raising=False)
summary, usage = context_compaction._summarize_round_batch(
[(1, "TOOL_CALL x: {}")],
drive_root=tmp_path,
task_id="task-42",
)
assert summary == {1: "summary"}
assert usage == {"prompt_tokens": 1}
assert seen["drive_root"] == tmp_path
assert seen["task_id"] == "task-42"