mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-03 04:07:04 +00:00
fix: expose typed Claudexor provider failures
Show preserved provider code and parameter details after classification without changing retry, compaction, or custody semantics. Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
This commit is contained in:
parent
031183e723
commit
46b07fe976
6 changed files with 181 additions and 6 deletions
|
|
@ -1321,6 +1321,11 @@ text, including when the prior provider supplied no ordinary content. The exact
|
|||
`model_request_invalid` create refusal proves validation failed before command
|
||||
admission; a generic HTTP error or failed status read cannot prove non-dispatch.
|
||||
|
||||
`ClaudexorModelError.display_message` adds sanitized typed `vendorCode` and
|
||||
`parameter` details to the existing error event and terminal preview only after
|
||||
classification. Exception text and machine fields retain their retry, compaction
|
||||
and custody semantics; unknown outcomes do not expose underlying provider details.
|
||||
|
||||
`ClaudexorGateway` uses purpose-bound byte uploads, one idempotent operation ID,
|
||||
status/result reads, cancellation and explicit result acknowledgement. A lost
|
||||
local HTTP reply rejoins the same operation; an unknown provider outcome cannot
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ from ouroboros.usage_accounting import (
|
|||
execute_physical_attempt, execute_physical_attempt_async,
|
||||
last_physical_attempt_capture,
|
||||
)
|
||||
from ouroboros.utils import append_jsonl, utc_now_iso
|
||||
from ouroboros.utils import append_jsonl, sanitize_tool_result_for_log, utc_now_iso
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -77,6 +77,18 @@ class ClaudexorModelError(RuntimeError):
|
|||
self.operation_id = operation_id
|
||||
self.route = copy.deepcopy(route or {})
|
||||
|
||||
@property
|
||||
def display_message(self) -> str:
|
||||
"""Show typed provider details without changing exception classification text."""
|
||||
context = self.problem.get("context") or {}
|
||||
details = [] if self.code == "model_outcome_unknown" else [
|
||||
f"{label}={value.strip()}"
|
||||
for key, label in (("vendorCode", "provider_code"), ("parameter", "parameter"))
|
||||
if isinstance(value := context.get(key), str) and value.strip()
|
||||
]
|
||||
# Details lead so the existing terminal preview can name the refusal.
|
||||
return sanitize_tool_result_for_log("; ".join([", ".join(details), str(self)]) if details else str(self))
|
||||
|
||||
|
||||
class ClaudexorModelNotDispatched(ClaudexorModelError, ProviderNotDispatched):
|
||||
"""Only a terminal engine receipt proving dispatch.state=not_started mints this."""
|
||||
|
|
|
|||
|
|
@ -925,6 +925,9 @@ def _record_llm_call_error(
|
|||
safe_error = sanitize_tool_result_for_log(repr(error))
|
||||
classification = classify_llm_exception(error, safe_error)
|
||||
provider_message = _exception_provider_message(error, safe_error)
|
||||
# Display metadata must not enter the classifier's text heuristics.
|
||||
display_message = getattr(error, "display_message", None)
|
||||
display_error = sanitize_tool_result_for_log(display_message) if isinstance(display_message, str) and display_message else safe_error
|
||||
custody_fields = attempt_custody_event_fields(error)
|
||||
will_retry = classification.retry_same_request
|
||||
repeats = _transport_death_repeats(ctx.accumulated_usage, ctx.round_id)
|
||||
|
|
@ -968,7 +971,7 @@ def _record_llm_call_error(
|
|||
# preserving its identity for live/backfill dedupe. No llm_round_error
|
||||
# sibling here; Background Consciousness keeps its own separate producer.
|
||||
error_event = {
|
||||
"ts": utc_now_iso(), "type": "llm_api_error", **identity, "error": safe_error,
|
||||
"ts": utc_now_iso(), "type": "llm_api_error", **identity, "error": display_error,
|
||||
"error_kind": classification.kind, "retry_same_request": will_retry,
|
||||
"status_code": classification.status_code, "provider_code": classification.provider_code,
|
||||
"provider_message": provider_message,
|
||||
|
|
@ -978,7 +981,7 @@ def _record_llm_call_error(
|
|||
}
|
||||
if not append_jsonl(ctx.drive_logs / "events.jsonl", error_event) or not has_log_sink():
|
||||
emit_log_event(ctx.event_queue, error_event, log_label="LLM call error")
|
||||
ctx.accumulated_usage.update(_last_llm_error=_short_error_text(safe_error),
|
||||
ctx.accumulated_usage.update(_last_llm_error=_short_error_text(display_error),
|
||||
_last_llm_error_kind=classification.kind, _last_llm_retry_same_request=will_retry)
|
||||
if classification.retry_after_sec is not None:
|
||||
ctx.accumulated_usage["_last_llm_retry_after_sec"] = classification.retry_after_sec
|
||||
|
|
|
|||
|
|
@ -26,6 +26,70 @@ ROUTE = {"source": "codex", "credentialProfileId": "account-a", "accountFingerpr
|
|||
REF = {"resourceId": "res-one", "sha256": "sha256:" + "a" * 64, "sizeBytes": 99}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("unknown", [False, True])
|
||||
def test_display_diagnostics_preserve_exception_and_private_problem(unknown):
|
||||
problem = {"code": "invalid_request", "message": "Codex model request was refused (HTTP 400).",
|
||||
"retryable": True, "context": {"httpStatus": 400, "resetsAt": "2099-01-01T00:00:00Z",
|
||||
"vendorCode": "string_above_max_length", "parameter": "instructions",
|
||||
"providerMessage": "private provider body", "requestId": "private-request"}}
|
||||
original = deepcopy(problem)
|
||||
error = transport.ClaudexorModelError(problem, model_role="main", operation_id="op-1", route=ROUTE, unknown=unknown)
|
||||
code = "model_outcome_unknown" if unknown else "invalid_request"
|
||||
generic = f"{code}: {problem['message']}"
|
||||
assert error.args == (generic,) and str(error) == generic
|
||||
assert repr(error) == f"ClaudexorModelError({generic!r})"
|
||||
assert error.code == code and error.body == ({"code": code} if unknown else original)
|
||||
assert error.status_code == (0 if unknown else 400) and error.retryable is (not unknown)
|
||||
assert error.reset_at == original["context"]["resetsAt"]
|
||||
assert error.model_role == "main" and error.operation_id == "op-1" and error.route == ROUTE
|
||||
display = error.display_message
|
||||
assert ("provider_code=string_above_max_length, parameter=instructions" in display[:220]) is (not unknown)
|
||||
assert "private provider body" not in display and "private-request" not in display
|
||||
if unknown:
|
||||
assert display == generic
|
||||
problem["context"]["parameter"] = "changed after construction"
|
||||
assert error.problem == original and error.display_message == display
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context", [{}, {"vendorCode": None, "parameter": 42},
|
||||
{"vendorCode": " ", "parameter": ["private provider data"]}])
|
||||
def test_display_ignores_absent_or_nontext_details(context):
|
||||
error = transport.ClaudexorModelError({"code": "invalid_request", "message": "Controlled refusal", "context": context})
|
||||
assert error.display_message == str(error)
|
||||
|
||||
|
||||
def test_display_redacts_typed_details_without_mutating_custody():
|
||||
token = "sk-" + "secretfixture" * 4
|
||||
context = {"vendorCode": token, "parameter": "https://user:private-password@example.test/input"}
|
||||
error = transport.ClaudexorModelError({"code": "invalid_request", "message": "Controlled refusal", "context": context})
|
||||
assert token not in error.display_message and "private-password" not in error.display_message
|
||||
assert "REDACTED" in error.display_message and error.problem["context"] == context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code,status,unknown,kind,retry,wait", [
|
||||
("invalid_request", 400, False, "bad_request", False, ""),
|
||||
("auth_required", 401, False, "auth_error", False, "auth"),
|
||||
("subscription_window_exhausted", 429, False, "subscription_window_exhausted", True, "quota"),
|
||||
("invalid_request", 400, True, "provider_outcome_unknown", False, ""),
|
||||
])
|
||||
@pytest.mark.parametrize("vendor,parameter", [("string_above_max_length", "instructions"),
|
||||
("string_above_max_length", "max_tokens"), ("rate_limit_exceeded", "input"), ("context_length_exceeded", "input")])
|
||||
def test_display_never_changes_classification_wait_or_compaction(code, status, unknown, kind, retry, wait, vendor, parameter):
|
||||
from ouroboros.context_compaction import _typed_context_overflow
|
||||
from ouroboros.loop_llm_call import classify_llm_exception
|
||||
from ouroboros.model_wait import model_wait_reason
|
||||
|
||||
error = transport.ClaudexorModelError({"code": code, "message": "Controlled model refusal", "retryable": True,
|
||||
"context": {"httpStatus": status, "vendorCode": vendor, "parameter": parameter}}, unknown=unknown)
|
||||
assert error.display_message # Computing a human view must not mutate behavioral readers.
|
||||
classified = classify_llm_exception(error)
|
||||
assert classified.kind == kind and classified.retry_same_request is retry
|
||||
assert model_wait_reason(error) == wait
|
||||
assert not _typed_context_overflow(error)
|
||||
typed = transport.ClaudexorModelError({"code": "context_length_exceeded", "message": "Controlled refusal"})
|
||||
assert _typed_context_overflow(typed) and classify_llm_exception(typed).kind == "context_overflow"
|
||||
|
||||
|
||||
def result(*, outcome="completed", cash=None, knowledge="unknown", route=None, problem=None):
|
||||
route = dict(ROUTE if route is None else route)
|
||||
return {"outcome": outcome, "message": {"role": "assistant", "content": "Ответ 🐍", "tool_calls": [
|
||||
|
|
@ -355,6 +419,35 @@ def test_confirmed_provider_failure_settles_real_usage_before_raising(setup):
|
|||
assert retained(root) == gateway.results[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_field_refusal_retains_then_acknowledges_once_with_display(setup, asynchronous):
|
||||
root, gateway, client = setup
|
||||
problem = {"code": "invalid_request", "message": "Codex model request was refused (HTTP 400).", "retryable": False,
|
||||
"context": {"httpStatus": 400, "vendorCode": "string_above_max_length", "parameter": "instructions"}}
|
||||
gateway.results = [result(outcome="failed", problem=problem)]
|
||||
acknowledge = gateway.acknowledge_model_result
|
||||
|
||||
def after_retention(*args):
|
||||
assert retained(root) == gateway.results[0]
|
||||
return acknowledge(*args)
|
||||
|
||||
gateway.acknowledge_model_result = after_retention
|
||||
with pytest.raises(transport.ClaudexorModelError) as caught:
|
||||
if asynchronous:
|
||||
asyncio.run(client.chat_async([], MODEL, model_role="main"))
|
||||
else:
|
||||
client.chat([], MODEL, model_role="main")
|
||||
error = caught.value
|
||||
assert error.problem == problem and error.body == problem and error.code == "invalid_request"
|
||||
assert error.status_code == 400 and error.retryable is False
|
||||
assert error.operation_id == "op-0" and error.model_role == "main" and error.route == ROUTE
|
||||
assert "provider_code=string_above_max_length, parameter=instructions" in error.display_message[:220]
|
||||
assert error.physical_attempt_capture.state == "settled"
|
||||
assert error.usage["claudexor"]["result_custody"]["state"] == "acknowledged"
|
||||
assert len(gateway.operations) == len(gateway.creates) == len(gateway.acks) == 1
|
||||
assert [row["state"] for row in ledger(root)] == ["reserved", "dispatched", "settled"]
|
||||
|
||||
|
||||
def test_proven_not_started_releases_and_never_fabricates_provider_usage(setup):
|
||||
root, gateway, client = setup
|
||||
gateway.results = [result(outcome="failed", problem={"code": "unsupported_parameter", "message": "temperature unsupported"})]
|
||||
|
|
|
|||
|
|
@ -811,6 +811,43 @@ def test_provider_failure_hint_empty_without_error():
|
|||
assert _provider_failure_hint({}) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("vendor,parameter", [("string_above_max_length", "instructions"),
|
||||
("string_above_max_length", "max_tokens"), ("context_length_exceeded", "input"), ("rate_limit_exceeded", "input")])
|
||||
def test_claudexor_display_reaches_error_and_terminal_without_retry(tmp_path, monkeypatch, vendor, parameter):
|
||||
import json
|
||||
from ouroboros.llm_claudexor import ClaudexorModelError
|
||||
from ouroboros.loop_transport import provider_recovery_hint
|
||||
|
||||
error = ClaudexorModelError({"code": "invalid_request", "message": "Codex model request was refused (HTTP 400).",
|
||||
"context": {"httpStatus": 400, "vendorCode": vendor, "parameter": parameter, "providerMessage": "private-body"}})
|
||||
baseline = classify_llm_exception(error)
|
||||
events, usage = _RecordingEvents(), {"_context_fit_mode": "max"}
|
||||
|
||||
class Refused:
|
||||
calls = 0
|
||||
|
||||
def chat(self, **kwargs):
|
||||
self.calls += 1
|
||||
raise error
|
||||
|
||||
llm = Refused()
|
||||
monkeypatch.setattr("ouroboros.loop_llm_call.time.sleep", lambda _s: pytest.fail("Field refusal cannot retry"))
|
||||
msg, _cost = call_llm_with_retry(llm, [{"role": "user", "content": "hi"}], "claudexor::fixture=exact-model",
|
||||
None, "high", 3, tmp_path, "task-refusal", 1, events, usage, "task", False)
|
||||
assert msg is None and llm.calls == 1
|
||||
assert usage["_context_fit_mode"] == "max" and usage["_last_llm_error_kind"] == baseline.kind == "bad_request"
|
||||
assert usage["_last_llm_retry_same_request"] is baseline.retry_same_request is False
|
||||
rows = [json.loads(line) for line in (tmp_path / "events.jsonl").read_text().splitlines()]
|
||||
event, = [row for row in rows if row["type"] == "llm_api_error"]
|
||||
assert [row["data"] for row in events.events if row.get("data", {}).get("type") == "llm_api_error"] == [event]
|
||||
assert event["error"] == error.display_message and event["status_code"] == baseline.status_code
|
||||
assert event["provider_code"] == baseline.provider_code and event["provider_message"] == error.problem["message"]
|
||||
assert "private-body" not in event["error"] and "remote_context_overflow" not in [row["type"] for row in rows]
|
||||
hint = _provider_failure_hint(usage)
|
||||
assert f"provider_code={vendor}" in hint and f"parameter={parameter}" in hint
|
||||
assert provider_recovery_hint(usage) == provider_recovery_hint({**usage, "_last_llm_error": repr(error)})
|
||||
|
||||
|
||||
def test_call_llm_with_retry_accumulates_live_catalog_estimated_cost(tmp_path):
|
||||
import queue
|
||||
|
||||
|
|
|
|||
|
|
@ -50,12 +50,16 @@ def _install_child_fixture(mode, events_path):
|
|||
|
||||
def detail(self):
|
||||
pending = mode in {"pending", "cancel"} or mode == "slow" and self.polls < 8
|
||||
refused = mode in {"quota", "auth", "mixed"}
|
||||
not_started = mode in {"quota", "auth", "mixed"}
|
||||
refused = not_started or mode == "field"
|
||||
problem = {"code": {"quota": "subscription_window_exhausted", "auth": "auth_required",
|
||||
"mixed": "credential_pool_exhausted"}.get(mode, "fixture_error"),
|
||||
"mixed": "credential_pool_exhausted", "field": "invalid_request"}.get(mode, "fixture_error"),
|
||||
"message": "controlled refusal", "context": {"resetsAt": "2099-01-01T00:00:00Z"}}
|
||||
if mode == "field":
|
||||
problem["context"].update(httpStatus=400, vendorCode="string_above_max_length",
|
||||
parameter="instructions", providerMessage="private provider body")
|
||||
return {"id": self.operation, "state": "running" if pending else "failed" if refused else "succeeded",
|
||||
"dispatch": {"state": "started" if pending else "not_started" if refused else "response_received",
|
||||
"dispatch": {"state": "started" if pending else "not_started" if not_started else "response_received",
|
||||
"route": ROUTE},
|
||||
"response": {"state": "absent"} if pending else {"state": "ready", "ref": REF},
|
||||
"problem": problem if refused else None}
|
||||
|
|
@ -203,6 +207,27 @@ def test_real_child_preserves_typed_refusal_receipt(child_fixture, mode, code):
|
|||
assert error.ledger_attempt_ids == [error.physical_attempt_capture.attempt_id]
|
||||
|
||||
|
||||
def test_real_child_reconstructs_provider_field_display_after_settlement(child_fixture):
|
||||
state, events, root = child_fixture
|
||||
state["mode"] = "field"
|
||||
with pytest.raises(ClaudexorModelError) as caught:
|
||||
_call()
|
||||
error = caught.value
|
||||
assert type(error) is ClaudexorModelError
|
||||
assert error.code == "invalid_request" and error.status_code == 400 and error.retryable is False
|
||||
assert error.operation_id == "operation-exact" and error.route == ROUTE and error.model_role == "vision"
|
||||
assert error.body == error.problem and error.problem["context"]["providerMessage"] == "private provider body"
|
||||
assert "provider_code=string_above_max_length, parameter=instructions" in error.display_message[:220]
|
||||
assert "private provider body" not in error.display_message
|
||||
assert str(error) == "invalid_request: controlled refusal"
|
||||
assert error.physical_attempt_capture.state == "settled"
|
||||
assert error.ledger_attempt_ids == [error.physical_attempt_capture.attempt_id]
|
||||
rows = _events(events)
|
||||
assert sum(row["kind"] == "generation" for row in rows) == sum(row["kind"] == "ack" for row in rows) == 1
|
||||
attempts = [json.loads(line) for line in (root / ua.LEDGER_REL).read_text().splitlines()]
|
||||
assert [row["state"] for row in attempts] == ["reserved", "dispatched", "settled"]
|
||||
|
||||
|
||||
def test_parent_cancel_reaches_same_live_operation(child_fixture):
|
||||
state, events, _ = child_fixture
|
||||
state["mode"] = "cancel"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue