mirror of
https://github.com/razzant/ouroboros.git
synced 2026-08-05 00:29:46 +00:00
feat(narration): narrate empty tool-round bubbles with provider-returned reasoning
A pure tool-call round produced no visible text, so its progress bubble was blank even when the provider returned readable reasoning. Add LLMClient.extract_display_reasoning — a provider-agnostic, SHAPE-based reader that surfaces readable reasoning the model already returned (flat `reasoning`, `reasoning_details` of readable types, Anthropic `thinking`, Gemini `part.thought`) and SKIPS opaque/encrypted payloads (reasoning.encrypted, redacted_thinking, signature/data-only) that carry no display text and must round-trip byte-for-byte. The run_llm_loop display seam falls back to it (visible content preferred), gated by OUROBOROS_REASONING_SUMMARY=auto|off. DISPLAY-ONLY and round-trip-safe: the reader never mutates the message, the result stays in a local variable (the appended assistant_msg is the raw msg, unchanged), and it is never sent to a provider — so the reasoning round-trip guards (sanitize_reasoning_on_model_switch, _strip_openrouter_roundtrip_metadata, the allow_fallbacks pin) are untouched. Verified against live gpt-5.5, which returns a readable reasoning.summary the extractor surfaces while skipping the 1380-byte encrypted block. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
b662500030
commit
48cbf4afd0
6 changed files with 225 additions and 4 deletions
|
|
@ -1489,6 +1489,7 @@ Runtime floors:
|
|||
| OUROBOROS_EFFORT_DEEP_SELF_REVIEW | high | Reasoning effort for deep self-review |
|
||||
| OUROBOROS_EFFORT_CONSCIOUSNESS | high | Reasoning effort for background consciousness |
|
||||
| OUROBOROS_RETURN_REASONING | true | OpenRouter reasoning continuity switch. Unset means return reasoning payloads by default; false-like values or an explicit empty string opt out. Direct/local routes strip OpenRouter-only reasoning fields on copied payloads. |
|
||||
| OUROBOROS_REASONING_SUMMARY | auto | Narration display switch. `auto` (default) narrates an otherwise-empty tool-round bubble with readable reasoning the provider already returned (`LLMClient.extract_display_reasoning`, shape-based: flat `reasoning` / `reasoning_details` of readable types / Anthropic `thinking` / Gemini `part.thought`; opaque/encrypted skipped). `off` disables the fallback. DISPLAY-ONLY — never added to the transcript or sent back to a provider, so it cannot affect round-trip. Verified against live gpt-5.5, which returns a readable `reasoning.summary` alongside the encrypted block. |
|
||||
| OUROBOROS_SOFT_TIMEOUT_SEC | 600 | Soft timeout warning (10 min) |
|
||||
| OUROBOROS_HARD_TIMEOUT_SEC | 1800 | (v6.38.0) NO LONGER a kill axis. The flat blanket wall-clock kill was replaced by the activity model below; this value now only feeds the soft-warning/status display. Task termination is governed by the idle/ceiling/deadline axes. |
|
||||
| OUROBOROS_TASK_IDLE_TIMEOUT_SEC | 900 | (v6.38.0) Activity-based idle window: a task is stopped only after it has made NO real progress (`llm_usage`/progress events — NOT the unconditional 30s liveness heartbeat) AND has no progressing/queued subtree for this long. Effective value is floored to the per-call timeout ceiling (`max(idle, per_call_ceiling+120)`) so a single legitimate long tool/LLM call is never idle-killed mid-work. |
|
||||
|
|
|
|||
|
|
@ -203,6 +203,7 @@ SETTINGS_DEFAULTS = {
|
|||
"OUROBOROS_EFFORT_DEEP_SELF_REVIEW": "high",
|
||||
"OUROBOROS_EFFORT_CONSCIOUSNESS": "high",
|
||||
"OUROBOROS_RETURN_REASONING": True,
|
||||
"OUROBOROS_REASONING_SUMMARY": "auto",
|
||||
"GITHUB_TOKEN": "",
|
||||
"GITHUB_REPO": "",
|
||||
# Local model (llama-cpp-python server)
|
||||
|
|
@ -1188,6 +1189,7 @@ def apply_settings_to_env(settings: dict) -> None:
|
|||
"OUROBOROS_EFFORT_DEEP_SELF_REVIEW",
|
||||
"OUROBOROS_EFFORT_CONSCIOUSNESS",
|
||||
"OUROBOROS_RETURN_REASONING",
|
||||
"OUROBOROS_REASONING_SUMMARY",
|
||||
"LOCAL_MODEL_SOURCE", "LOCAL_MODEL_FILENAME",
|
||||
"LOCAL_MODEL_PORT", "LOCAL_MODEL_N_GPU_LAYERS", "LOCAL_MODEL_CONTEXT_LENGTH",
|
||||
"LOCAL_MODEL_CHAT_FORMAT",
|
||||
|
|
|
|||
|
|
@ -2320,6 +2320,64 @@ class LLMClient:
|
|||
|
||||
return msg, usage
|
||||
|
||||
@staticmethod
|
||||
def extract_display_reasoning(msg: Dict[str, Any]) -> str:
|
||||
"""Provider-agnostic, SHAPE-based reader for human-readable reasoning to NARRATE in an
|
||||
otherwise-empty tool-round bubble. Reads only the readable forms a provider may already
|
||||
leave on the normalized message — flat ``reasoning`` (OpenRouter / some OpenAI-compatible),
|
||||
structured ``reasoning_details`` of readable types, or ``content`` thinking/thought blocks
|
||||
(Anthropic ``thinking`` / Gemini ``part.thought``) — and SKIPS opaque/encrypted payloads
|
||||
(``reasoning.encrypted``, ``redacted_thinking``, signature/data-only blocks), which carry no
|
||||
display text and must round-trip byte-for-byte. DISPLAY-ONLY: the caller keeps the result in
|
||||
a local variable and never appends it to the transcript nor sends it to a provider — the raw
|
||||
fields it reads are already on the message and handled by the outbound scrubbers."""
|
||||
if not isinstance(msg, dict):
|
||||
return ""
|
||||
parts: List[str] = []
|
||||
|
||||
flat = msg.get("reasoning")
|
||||
if isinstance(flat, str) and flat.strip():
|
||||
parts.append(flat.strip())
|
||||
|
||||
details = msg.get("reasoning_details")
|
||||
if isinstance(details, list):
|
||||
for d in details:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
if str(d.get("type") or "") in ("reasoning.text", "reasoning.summary"):
|
||||
txt = d.get("text") or d.get("summary")
|
||||
if isinstance(txt, str) and txt.strip():
|
||||
parts.append(txt.strip())
|
||||
# reasoning.encrypted / signature / data-only payloads are opaque -> skipped.
|
||||
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
btype = str(block.get("type") or "")
|
||||
if btype == "thinking":
|
||||
txt = block.get("thinking")
|
||||
elif btype == "reasoning":
|
||||
txt = block.get("text") or block.get("reasoning")
|
||||
elif block.get("thought") is True: # Gemini part.thought == true
|
||||
txt = block.get("text")
|
||||
else:
|
||||
continue # text / tool_use / redacted_thinking / encrypted -> not display text
|
||||
if isinstance(txt, str) and txt.strip():
|
||||
parts.append(txt.strip())
|
||||
|
||||
# De-dup across the whole set (order-preserving): a provider often carries the SAME
|
||||
# readable rollup in both flat ``reasoning`` and a ``reasoning.summary`` detail (verified
|
||||
# against live gpt-5.5), so a consecutive-only check would still double it.
|
||||
deduped: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for p in parts:
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
deduped.append(p)
|
||||
return "\n".join(deduped).strip()
|
||||
|
||||
def _create_chat_completion_with_retries(
|
||||
self,
|
||||
create_fn: Any,
|
||||
|
|
|
|||
|
|
@ -1514,6 +1514,30 @@ def _apply_overrides_and_regate_mode(ctx, active_model, active_use_local, active
|
|||
return active_model, active_use_local, active_effort, active_context_mode
|
||||
|
||||
|
||||
def _visible_round_text(content: Any) -> str:
|
||||
"""The round's visible assistant text as a plain string. A provider may return ``content`` as
|
||||
a string OR a list of typed blocks; collect the ``text`` of every block EXCEPT reasoning ones
|
||||
(Anthropic ``thinking``/``redacted_thinking``, Gemini ``part.thought``) — the exact complement
|
||||
of extract_display_reasoning. A regular Gemini part carries ``text`` with NO ``type``, so keying
|
||||
on the ABSENCE of a reasoning marker (not on ``type == 'text'``) avoids dropping real answer
|
||||
text; a non-empty block list never stringifies to a raw Python repr, and a thinking-only list
|
||||
correctly reads as 'no visible text' (letting narration fall back to readable reasoning)."""
|
||||
if isinstance(content, str):
|
||||
return content.strip()
|
||||
if isinstance(content, list):
|
||||
out: List[str] = []
|
||||
for b in content:
|
||||
if not isinstance(b, dict):
|
||||
continue
|
||||
if str(b.get("type") or "") in ("thinking", "reasoning", "redacted_thinking") or b.get("thought") is True:
|
||||
continue # reasoning/thinking blocks are display reasoning, not visible answer text
|
||||
txt = b.get("text")
|
||||
if isinstance(txt, str):
|
||||
out.append(txt)
|
||||
return "".join(out).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def run_llm_loop(
|
||||
messages: List[Dict[str, Any]],
|
||||
tools: ToolRegistry,
|
||||
|
|
@ -1748,10 +1772,20 @@ def run_llm_loop(
|
|||
assistant_msg.setdefault("role", "assistant")
|
||||
messages.append(assistant_msg)
|
||||
|
||||
progress_text = str(content or "").strip()
|
||||
if progress_text:
|
||||
emit_progress(progress_text.strip())
|
||||
llm_trace["reasoning_notes"].append(progress_text.strip())
|
||||
visible_text = _visible_round_text(content)
|
||||
if visible_text:
|
||||
emit_progress(visible_text)
|
||||
llm_trace["reasoning_notes"].append(visible_text)
|
||||
elif str(os.environ.get("OUROBOROS_REASONING_SUMMARY", "auto")).strip().lower() != "off":
|
||||
# Narration: a pure tool-call round had no visible text — surface readable reasoning
|
||||
# the provider already returned (shape-based, opaque skipped) so the bubble is not
|
||||
# blank. DISPLAY-ONLY: emitted to the UI bubble but NOT recorded in reasoning_notes
|
||||
# (which feeds build_trace_summary / task summaries) and never appended to the
|
||||
# transcript (assistant_msg above is the raw msg) — so it cannot leak out of the
|
||||
# display path into the durable trace or back to a provider.
|
||||
display_reasoning = LLMClient.extract_display_reasoning(msg)
|
||||
if display_reasoning:
|
||||
emit_progress(display_reasoning)
|
||||
|
||||
handle_tool_calls(
|
||||
tool_calls, tools, drive_logs, task_id, stateful_executor,
|
||||
|
|
|
|||
|
|
@ -472,6 +472,59 @@ def test_run_llm_loop_preserves_assistant_tool_call_metadata(tmp_path, monkeypat
|
|||
assert assistant_msg["response_id"] == "gen-123"
|
||||
|
||||
|
||||
def test_run_llm_loop_narrates_reasoning_to_bubble_not_trace(tmp_path, monkeypatch):
|
||||
"""Display-only contract: a pure tool-call round with no visible content narrates the
|
||||
provider's readable reasoning to the progress BUBBLE, but never records it in the durable
|
||||
trace (``reasoning_notes`` feeds build_trace_summary / task summaries) — so display-only
|
||||
reasoning cannot leak out of the display path."""
|
||||
from ouroboros.tools.registry import ToolRegistry
|
||||
|
||||
messages = [{"role": "user", "content": "go"}]
|
||||
tool_round = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "c1", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}],
|
||||
"reasoning": "Let me read the file before answering.",
|
||||
}
|
||||
calls = {"count": 0}
|
||||
emitted: list = []
|
||||
|
||||
class FakeLLM:
|
||||
def default_model(self):
|
||||
return "test-model"
|
||||
|
||||
def fake_call_llm_with_retry(_llm, request_messages, *_a, **_k):
|
||||
calls["count"] += 1
|
||||
if calls["count"] == 1:
|
||||
return dict(tool_round), 0.0
|
||||
return {"role": "assistant", "content": "final answer"}, 0.0
|
||||
|
||||
def fake_handle_tool_calls(tool_calls, _tools, _dl, _tid, _ex, request_messages, _tr, _pg):
|
||||
request_messages.append({"role": "tool", "tool_call_id": tool_calls[0]["id"], "content": "file body"})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call_llm_with_retry)
|
||||
monkeypatch.setattr(loop_mod, "handle_tool_calls", fake_handle_tool_calls)
|
||||
monkeypatch.setenv("OUROBOROS_REASONING_SUMMARY", "auto")
|
||||
|
||||
result, _usage, trace = run_llm_loop(
|
||||
messages=messages,
|
||||
tools=ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path),
|
||||
llm=FakeLLM(),
|
||||
drive_logs=tmp_path,
|
||||
emit_progress=lambda text: emitted.append(text),
|
||||
incoming_messages=queue.Queue(),
|
||||
task_id="narrate",
|
||||
drive_root=tmp_path,
|
||||
)
|
||||
|
||||
assert result == "final answer"
|
||||
# the readable reasoning reached the display bubble...
|
||||
assert any("read the file before answering" in str(e) for e in emitted)
|
||||
# ...but did NOT leak into the durable trace (display-only).
|
||||
assert all("read the file before answering" not in str(n) for n in trace["reasoning_notes"])
|
||||
|
||||
|
||||
def test_run_llm_loop_finalize_now_control_forces_best_effort_answer(tmp_path, monkeypatch):
|
||||
"""A supervisor finalize_now control makes the loop extract one tool-less
|
||||
final answer and stamp the finalization_grace reason (typed best_effort
|
||||
|
|
|
|||
73
tests/test_narration_display.py
Normal file
73
tests/test_narration_display.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""Provider-agnostic narration: LLMClient.extract_display_reasoning reads readable reasoning by
|
||||
SHAPE and skips opaque/encrypted payloads, so empty tool-round bubbles get narrated without ever
|
||||
touching the transcript or the round-trip-sensitive metadata."""
|
||||
|
||||
from ouroboros.llm import LLMClient
|
||||
|
||||
|
||||
def test_flat_reasoning_string():
|
||||
assert LLMClient.extract_display_reasoning({"reasoning": " thinking about X "}) == "thinking about X"
|
||||
|
||||
|
||||
def test_reasoning_details_readable_types():
|
||||
msg = {"reasoning_details": [
|
||||
{"type": "reasoning.text", "text": "step one"},
|
||||
{"type": "reasoning.summary", "summary": "summary two"},
|
||||
]}
|
||||
assert LLMClient.extract_display_reasoning(msg) == "step one\nsummary two"
|
||||
|
||||
|
||||
def test_reasoning_details_encrypted_is_skipped():
|
||||
msg = {"reasoning_details": [
|
||||
{"type": "reasoning.encrypted", "data": "BASE64OPAQUE=="},
|
||||
{"type": "reasoning.text", "text": "visible"},
|
||||
]}
|
||||
# opaque encrypted contributes nothing; only the readable text shows.
|
||||
assert LLMClient.extract_display_reasoning(msg) == "visible"
|
||||
|
||||
|
||||
def test_anthropic_thinking_block_read_redacted_skipped():
|
||||
msg = {"content": [
|
||||
{"type": "thinking", "thinking": "let me reason", "signature": "sig"},
|
||||
{"type": "redacted_thinking", "data": "OPAQUE"},
|
||||
{"type": "text", "text": "the answer"},
|
||||
]}
|
||||
# the readable thinking is surfaced; redacted (opaque) and the plain answer text are not reasoning.
|
||||
assert LLMClient.extract_display_reasoning(msg) == "let me reason"
|
||||
|
||||
|
||||
def test_gemini_thought_part():
|
||||
msg = {"content": [
|
||||
{"thought": True, "text": "gemini thought"},
|
||||
{"text": "regular part"},
|
||||
]}
|
||||
assert LLMClient.extract_display_reasoning(msg) == "gemini thought"
|
||||
|
||||
|
||||
def test_no_reasoning_returns_empty_and_string_content_is_safe():
|
||||
assert LLMClient.extract_display_reasoning({"content": "plain string answer"}) == ""
|
||||
assert LLMClient.extract_display_reasoning({}) == ""
|
||||
assert LLMClient.extract_display_reasoning(None) == ""
|
||||
|
||||
|
||||
def test_does_not_mutate_message():
|
||||
msg = {"reasoning": "x", "content": [{"type": "thinking", "thinking": "y"}]}
|
||||
before = dict(msg)
|
||||
LLMClient.extract_display_reasoning(msg)
|
||||
# display-only: the reader never adds/removes fields (transcript boundary stays clean).
|
||||
assert msg == before
|
||||
|
||||
|
||||
def test_visible_round_text_string_and_list_never_reprs():
|
||||
from ouroboros.loop import _visible_round_text
|
||||
|
||||
assert _visible_round_text(" hi ") == "hi"
|
||||
# a list of provider blocks joins ONLY text blocks — never a raw Python list repr.
|
||||
assert _visible_round_text([{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]) == "ab"
|
||||
# a thinking/thought-only list has NO visible text → reads empty so narration can fall back.
|
||||
assert _visible_round_text([{"type": "thinking", "thinking": "x"}, {"thought": True, "text": "y"}]) == ""
|
||||
# a regular Gemini part carries `text` with NO `type` — it is still visible answer text, and a
|
||||
# sibling thought block is excluded (visible text is the complement of display reasoning).
|
||||
assert _visible_round_text([{"thought": True, "text": "pondering"}, {"text": "the answer"}]) == "the answer"
|
||||
assert _visible_round_text(None) == ""
|
||||
assert _visible_round_text({"type": "text"}) == ""
|
||||
Loading…
Add table
Add a link
Reference in a new issue