mirror of
https://github.com/Skyvern-AI/skyvern.git
synced 2026-07-09 16:09:13 +00:00
SKY-11865: mint-time satisfiability + typed fail-safe degrade on the classifier-fallback completion producer (#7135)
Co-authored-by: Aaron Perez <aperez0295@gmail.com>
This commit is contained in:
Andrew Neilson
GitHub
parent
d71ea50d9a
commit
fdfaf3bbf7
7 changed files with 687 additions and 10 deletions
|
|
@ -13,12 +13,16 @@ import yaml
|
|||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
|
||||
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
|
||||
from skyvern.forge.sdk.copilot.completion_verification import (
|
||||
CompletionVerificationResult,
|
||||
CriterionVerdict,
|
||||
only_degraded_blocking,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
|
||||
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
|
||||
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
|
||||
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
|
||||
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
|
||||
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, RunOutcomeReasonCode
|
||||
|
||||
LOG = structlog.get_logger()
|
||||
|
||||
|
|
@ -51,6 +55,7 @@ BuildTestOutcomeReasonCode = Literal[
|
|||
"output_policy_reject",
|
||||
"scout_act_observe_hollow_after_interaction",
|
||||
"required_input_unbound",
|
||||
"fallback_floor_turn_unsatisfiable",
|
||||
]
|
||||
|
||||
_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
|
||||
|
|
@ -175,6 +180,19 @@ BindingFrontierFacet = Literal[
|
|||
"selector_frontier",
|
||||
]
|
||||
_UNCROSSABLE_DIAGNOSTIC_REASONS = frozenset({"empty_page", "challenge_gated", "capture_degraded"})
|
||||
_AMBIGUOUS_NON_DEMONSTRATION_RUN_REASON_CODES: frozenset[RunOutcomeReasonCode] = frozenset(
|
||||
{"outcome_not_demonstrated", "no_meaningful_output"}
|
||||
)
|
||||
|
||||
|
||||
def _recorded_outcome_degrade_eligible(
|
||||
recorded_run_outcome: RecordedRunOutcome,
|
||||
failed_block: Mapping[str, object] | None,
|
||||
) -> bool:
|
||||
if failed_block is not None:
|
||||
return False
|
||||
reason_code = recorded_run_outcome.reason_code
|
||||
return reason_code is None or reason_code in _AMBIGUOUS_NON_DEMONSTRATION_RUN_REASON_CODES
|
||||
|
||||
|
||||
class RecordedOutcomeBindingConstraint(BaseModel):
|
||||
|
|
@ -956,6 +974,22 @@ def recorded_outcome_from_run_blocks_result(
|
|||
observed_evidence_summary=recorded_run_outcome.display_reason or "",
|
||||
key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},
|
||||
)
|
||||
if (
|
||||
completion_verification is not None
|
||||
and only_degraded_blocking(completion_verification)
|
||||
and _recorded_outcome_degrade_eligible(recorded_run_outcome, failed_block)
|
||||
):
|
||||
return RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="not_authoritative",
|
||||
reason_code="fallback_floor_turn_unsatisfiable",
|
||||
workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
|
||||
block_labels=block_labels,
|
||||
authored_structure_signature=authored_structure_signature,
|
||||
observed_evidence_summary=recorded_run_outcome.display_reason or "",
|
||||
key_provenance={"structural_failure_identity": "turn-unsatisfiable fallback floor, no reachable route"},
|
||||
)
|
||||
return RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
|
|
@ -1008,12 +1042,22 @@ def recorded_outcome_from_run_blocks_result(
|
|||
verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"
|
||||
if not structural_identity and not page_refs and not output_refs:
|
||||
verdict = "not_authoritative"
|
||||
reason_code = "runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run"
|
||||
has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)
|
||||
if (
|
||||
verdict == "repairable_failure"
|
||||
and not has_runtime_failure_evidence
|
||||
and completion_verification is not None
|
||||
and only_degraded_blocking(completion_verification)
|
||||
):
|
||||
verdict = "not_authoritative"
|
||||
reason_code = "fallback_floor_turn_unsatisfiable"
|
||||
return RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",
|
||||
verdict=verdict,
|
||||
reason_code="runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run",
|
||||
reason_code=reason_code,
|
||||
workflow_run_id=workflow_run_id or None,
|
||||
block_labels=block_labels,
|
||||
structural_failure_identity=structural_identity,
|
||||
|
|
|
|||
|
|
@ -145,6 +145,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
|
|||
}
|
||||
if criterion.requested_output_corroborator:
|
||||
item["requested_output_corroborator"] = True
|
||||
if criterion.mint_degrade is not None:
|
||||
item["mint_degrade"] = criterion.mint_degrade
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
|
@ -217,6 +219,9 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
|
|||
classification_output_key=classification_output_key,
|
||||
expected_classification=expected_classification,
|
||||
requested_output_corroborator=bool(item.get("requested_output_corroborator")),
|
||||
mint_degrade="turn_unsatisfiable_fallback"
|
||||
if item.get("mint_degrade") == "turn_unsatisfiable_fallback"
|
||||
else None,
|
||||
)
|
||||
)
|
||||
return tuple(criteria)
|
||||
|
|
@ -225,7 +230,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
|
|||
def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
|
||||
contingent_key = criterion.contingent_on or ""
|
||||
contingent_path_key = criterion.contingent_antecedent_output_path or ""
|
||||
deliverable_kind_key = criterion.deliverable_kind or ""
|
||||
deliverable_kind_key = f"{criterion.deliverable_kind or ''}\x1fmint_degrade:{criterion.mint_degrade or ''}"
|
||||
expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)
|
||||
expected_output_shape_key = criterion.expected_output_shape or ""
|
||||
requested_output_evidence_source_key = criterion.requested_output_evidence_source
|
||||
|
|
@ -379,6 +384,7 @@ def apply_requested_output_producer_floor(
|
|||
if criterion.expected_output_value is None
|
||||
and criterion.expected_output_shape is None
|
||||
and criterion.deliverable_kind is None
|
||||
and criterion.mint_degrade is None
|
||||
}
|
||||
if not presence_only_ids:
|
||||
return criteria, ()
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
|
|||
CompletionCriterion,
|
||||
TerminalActionFamily,
|
||||
is_fallback_floor_base_criterion,
|
||||
is_turn_unsatisfiable_fallback_degraded,
|
||||
redact_raw_secrets_for_prompt,
|
||||
)
|
||||
from skyvern.utils.strings import escape_code_fences
|
||||
|
|
@ -111,6 +112,7 @@ class CompletionVerificationResult:
|
|||
contingent_on_by_criterion_id: dict[str, str] = field(default_factory=dict)
|
||||
contingent_antecedent_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)
|
||||
structural_unfired_criterion_ids: list[str] = field(default_factory=list)
|
||||
degraded_criterion_ids: list[str] = field(default_factory=list)
|
||||
|
||||
def is_fully_satisfied(self) -> bool:
|
||||
if self.status != "evaluated" or not self.criterion_ids:
|
||||
|
|
@ -123,6 +125,7 @@ class CompletionVerificationResult:
|
|||
_is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)
|
||||
for verdict in verdict_by_id.values()
|
||||
)
|
||||
degraded_id_set = set(self.degraded_criterion_ids)
|
||||
satisfied_run_plane_count = 0
|
||||
for criterion_id in self.criterion_ids:
|
||||
verdict = verdict_by_id.get(criterion_id)
|
||||
|
|
@ -133,6 +136,10 @@ class CompletionVerificationResult:
|
|||
if not verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX):
|
||||
satisfied_run_plane_count += 1
|
||||
continue
|
||||
# A turn-unsatisfiable fallback criterion has no reachable satisfaction route,
|
||||
# so an unsatisfied one can never be credited via corroboration or value-blind excusal.
|
||||
if criterion_id in degraded_id_set:
|
||||
return False
|
||||
# A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,
|
||||
# so it must not veto a run whose observable outcome evidence is fully confirmed.
|
||||
if verdict is not None and _is_definition_plane_abstention(verdict):
|
||||
|
|
@ -495,6 +502,8 @@ def structural_unfired_contingent_criterion_ids(
|
|||
) -> list[str]:
|
||||
unfired_ids: list[str] = []
|
||||
for criterion in criteria:
|
||||
if is_turn_unsatisfiable_fallback_degraded(criterion):
|
||||
continue
|
||||
path = criterion.contingent_antecedent_output_path
|
||||
if not path:
|
||||
continue
|
||||
|
|
@ -1266,7 +1275,10 @@ def grade_fallback_floor_reached_end_state_criteria(
|
|||
eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]
|
||||
if not eligible_criteria:
|
||||
return []
|
||||
evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)
|
||||
floor_degraded = any(is_turn_unsatisfiable_fallback_degraded(criterion) for criterion in eligible_criteria)
|
||||
evidence_ref = None
|
||||
if not floor_degraded:
|
||||
evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)
|
||||
if evidence_ref is None:
|
||||
evidence_ref = _fallback_floor_carrier_evidence_ref(snapshot, carrier_verdicts)
|
||||
if evidence_ref is None:
|
||||
|
|
@ -2153,6 +2165,7 @@ def combine_verification_results(
|
|||
contingent_on_by_id = dict(contingent_on_by_criterion_id or {})
|
||||
contingent_path_by_id = dict(contingent_antecedent_output_path_by_criterion_id or {})
|
||||
structural_unfired_ids = list(structural_unfired_criterion_ids)
|
||||
degraded_ids: list[str] = []
|
||||
if run_result is not None:
|
||||
contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))
|
||||
contingent_on_by_id.update(run_result.contingent_on_by_criterion_id)
|
||||
|
|
@ -2160,6 +2173,7 @@ def combine_verification_results(
|
|||
structural_unfired_ids = list(
|
||||
dict.fromkeys([*structural_unfired_ids, *run_result.structural_unfired_criterion_ids])
|
||||
)
|
||||
degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))
|
||||
if run_result is not None and run_result.status != "evaluated":
|
||||
return CompletionVerificationResult(
|
||||
status=run_result.status,
|
||||
|
|
@ -2169,6 +2183,7 @@ def combine_verification_results(
|
|||
contingent_on_by_criterion_id=contingent_on_by_id,
|
||||
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
|
||||
structural_unfired_criterion_ids=structural_unfired_ids,
|
||||
degraded_criterion_ids=degraded_ids,
|
||||
)
|
||||
if run_result is not None:
|
||||
run_result = replace(
|
||||
|
|
@ -2197,9 +2212,47 @@ def combine_verification_results(
|
|||
contingent_on_by_criterion_id=contingent_on_by_id,
|
||||
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
|
||||
structural_unfired_criterion_ids=structural_unfired_ids,
|
||||
degraded_criterion_ids=degraded_ids,
|
||||
)
|
||||
|
||||
|
||||
def only_degraded_blocking(result: CompletionVerificationResult) -> bool:
|
||||
"""True only when every criterion that still blocks full satisfaction is a
|
||||
turn-unsatisfiable fallback criterion. A genuine unsatisfied/unknown non-degraded
|
||||
criterion, an empty blocking set, or no degraded ids all return False, so this
|
||||
never masks a real runtime failure or a mixed degraded+legitimate result."""
|
||||
if result.status != "evaluated":
|
||||
return False
|
||||
degraded = set(result.degraded_criterion_ids)
|
||||
if not degraded:
|
||||
return False
|
||||
verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}
|
||||
blocking: set[str] = set()
|
||||
for criterion_id in result.criterion_ids:
|
||||
verdict = verdict_by_id.get(criterion_id)
|
||||
if verdict is not None and verdict.satisfied:
|
||||
continue
|
||||
if verdict is not None and _is_definition_plane_abstention(verdict):
|
||||
continue
|
||||
if verdict is not None and result.is_structural_contingent_abstention(verdict):
|
||||
continue
|
||||
blocking.add(criterion_id)
|
||||
if not blocking:
|
||||
return False
|
||||
return blocking <= degraded
|
||||
|
||||
|
||||
def carry_degraded_criterion_ids(
|
||||
result: CompletionVerificationResult,
|
||||
criteria: Iterable[CompletionCriterion],
|
||||
) -> CompletionVerificationResult:
|
||||
degraded = [criterion.id for criterion in criteria if is_turn_unsatisfiable_fallback_degraded(criterion)]
|
||||
if not degraded and not result.degraded_criterion_ids:
|
||||
return result
|
||||
merged = list(dict.fromkeys([*result.degraded_criterion_ids, *degraded]))
|
||||
return replace(result, degraded_criterion_ids=merged)
|
||||
|
||||
|
||||
async def evaluate_completion_criteria(
|
||||
criteria: list[CompletionCriterion],
|
||||
snapshot: RunEvidenceSnapshot,
|
||||
|
|
|
|||
|
|
@ -267,6 +267,7 @@ class CompletionCriterion:
|
|||
classification_output_key: str | None = None
|
||||
expected_classification: ClassificationTarget | None = None
|
||||
requested_output_corroborator: bool = False
|
||||
mint_degrade: Literal["turn_unsatisfiable_fallback"] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -349,6 +350,8 @@ class RequestPolicy:
|
|||
data[f"{prefix}_output_path"] = criterion.output_path
|
||||
data[f"{prefix}_grounding_mode"] = _criterion_grounding_mode(criterion)
|
||||
data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not None
|
||||
if criterion.mint_degrade is not None:
|
||||
data[f"{prefix}_mint_degrade"] = criterion.mint_degrade
|
||||
data[f"{prefix}_evidence_source"] = criterion.requested_output_evidence_source
|
||||
if criterion.expected_output_shape:
|
||||
data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape
|
||||
|
|
@ -1361,7 +1364,7 @@ def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicy
|
|||
|
||||
|
||||
def _apply_requested_output_completion_criteria(
|
||||
policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None
|
||||
policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None, *, extract_literals: bool = False
|
||||
) -> None:
|
||||
schema_aliases = schema_output_path_aliases_from_criteria(policy.completion_criteria)
|
||||
config_aliases = _normalize_requested_output_aliases(aliases)
|
||||
|
|
@ -1391,6 +1394,9 @@ def _apply_requested_output_completion_criteria(
|
|||
return
|
||||
|
||||
value_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)
|
||||
if extract_literals:
|
||||
for output_path, literal in _fallback_literal_expected_values(user_message, requested_specs).items():
|
||||
value_by_output_path.setdefault(output_path, literal)
|
||||
shape_by_output_path = _requested_output_shapes_from_criteria(policy.completion_criteria, requested_specs)
|
||||
source_by_output_path = _requested_output_evidence_sources_from_criteria(
|
||||
policy.completion_criteria, requested_specs
|
||||
|
|
@ -1555,6 +1561,88 @@ def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:
|
|||
return criterion.id == _FALLBACK_FLOOR_BASE_ID
|
||||
|
||||
|
||||
def is_turn_unsatisfiable_fallback_degraded(criterion: CompletionCriterion) -> bool:
|
||||
return criterion.mint_degrade == "turn_unsatisfiable_fallback"
|
||||
|
||||
|
||||
_FALLBACK_LITERAL_MIN_CHARS = 4
|
||||
_FALLBACK_LITERAL_BINDER_RE = r"(?:equal to|equals|should be|must be|is expected to be|expected to be|will be|is|:|=)"
|
||||
_FALLBACK_LITERAL_MAX_SCAN_CHARS = 4000
|
||||
|
||||
|
||||
def _fallback_literal_field_surface_forms(field_name: str, field_label: str) -> list[str]:
|
||||
forms: set[str] = set()
|
||||
for source in (field_name, field_label):
|
||||
collapsed = " ".join(source.split()).strip()
|
||||
if collapsed:
|
||||
forms.add(collapsed)
|
||||
forms.add(collapsed.replace(" ", "_"))
|
||||
return [form for form in sorted(forms, key=len, reverse=True) if form]
|
||||
|
||||
|
||||
def _fallback_literal_excluded_forms(field_name: str, field_label: str, output_path: str) -> set[str]:
|
||||
forms = {
|
||||
" ".join(_word_tokens(field_name)),
|
||||
" ".join(_word_tokens(field_label)),
|
||||
" ".join(_word_tokens(output_path)),
|
||||
}
|
||||
return {form for form in forms if form}
|
||||
|
||||
|
||||
def _fallback_literal_candidates_for_field(user_message: str, field_name: str, field_label: str) -> list[str]:
|
||||
bounded_message = user_message[:_FALLBACK_LITERAL_MAX_SCAN_CHARS]
|
||||
candidates: list[str] = []
|
||||
for surface in _fallback_literal_field_surface_forms(field_name, field_label):
|
||||
pattern = re.compile(
|
||||
r"\b"
|
||||
+ re.escape(surface)
|
||||
+ r"\b\s{0,8}+(?:"
|
||||
+ _FALLBACK_LITERAL_BINDER_RE
|
||||
+ r"\s{0,8}+)?(?P<quote>['\"`])(?P<quoted>[^'\"`]{1,80}+)(?P=quote)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
for match in pattern.finditer(bounded_message):
|
||||
quoted = match.group("quoted")
|
||||
if quoted is not None:
|
||||
candidates.append(quoted)
|
||||
return candidates
|
||||
|
||||
|
||||
def _fallback_literal_expected_values(
|
||||
user_message: str, requested_specs: list[tuple[str, str, str]]
|
||||
) -> dict[str, ExpectedOutputValue]:
|
||||
if not user_message.strip():
|
||||
return {}
|
||||
values: dict[str, ExpectedOutputValue] = {}
|
||||
for field_name, output_path, field_label in requested_specs:
|
||||
excluded = _fallback_literal_excluded_forms(field_name, field_label, output_path)
|
||||
accepted: list[str] = []
|
||||
for raw in _fallback_literal_candidates_for_field(user_message, field_name, field_label):
|
||||
coerced = _coerce_expected_output_value(raw)
|
||||
if not isinstance(coerced, str) or len(coerced) < _FALLBACK_LITERAL_MIN_CHARS:
|
||||
continue
|
||||
if " ".join(_word_tokens(coerced)) in excluded:
|
||||
continue
|
||||
accepted.append(coerced)
|
||||
unique = list(dict.fromkeys(accepted))
|
||||
if len(unique) == 1:
|
||||
values[output_path] = unique[0]
|
||||
return values
|
||||
|
||||
|
||||
def _mark_turn_unsatisfiable_fallback_criteria(policy: RequestPolicy) -> None:
|
||||
marked: list[CompletionCriterion] = []
|
||||
for criterion in policy.completion_criteria:
|
||||
value_less = criterion.expected_output_value is None and criterion.expected_output_shape is None
|
||||
is_floor_base = is_fallback_floor_base_criterion(criterion)
|
||||
is_requested_output = criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)
|
||||
if value_less and (is_floor_base or is_requested_output):
|
||||
marked.append(replace(criterion, mint_degrade="turn_unsatisfiable_fallback"))
|
||||
else:
|
||||
marked.append(criterion)
|
||||
policy.completion_criteria = marked
|
||||
|
||||
|
||||
def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:
|
||||
floor = [
|
||||
CompletionCriterion(
|
||||
|
|
@ -1617,8 +1705,11 @@ def _classifier_fallback_policy(
|
|||
classifier_retry_count=retry_count,
|
||||
completion_contract_status="present" if fallback_criteria else "unknown",
|
||||
)
|
||||
_apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
|
||||
_apply_requested_output_completion_criteria(
|
||||
policy, user_message, requested_output_path_aliases, extract_literals=True
|
||||
)
|
||||
_apply_classifier_typed_requested_output_corroborators(policy)
|
||||
_mark_turn_unsatisfiable_fallback_criteria(policy)
|
||||
if policy.graded_completion_criteria():
|
||||
policy.completion_contract_status = "present"
|
||||
return policy
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
|
|||
RunEvidenceSnapshot,
|
||||
_contingent_metadata_for_criteria,
|
||||
_is_structural_requested_output_abstention,
|
||||
carry_degraded_criterion_ids,
|
||||
combine_verification_results,
|
||||
evaluate_completion_criteria,
|
||||
grade_definition_criteria,
|
||||
|
|
@ -32,6 +33,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
|
|||
grade_validation_classification_criteria,
|
||||
is_fallback_floor_base_criterion,
|
||||
is_registered_download_completion_criterion,
|
||||
only_degraded_blocking,
|
||||
registered_download_completion_criterion,
|
||||
structural_unfired_contingent_criterion_ids,
|
||||
summarize_unsatisfied_outcomes,
|
||||
|
|
@ -399,6 +401,7 @@ async def _maybe_run_completion_verification_from_page_observation(
|
|||
)
|
||||
if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:
|
||||
verification = _no_gradeable_run_plane_result(criterion_ids)
|
||||
verification = carry_degraded_criterion_ids(verification, criteria)
|
||||
copilot_ctx.completion_verification_result = verification
|
||||
record_completion_verification(copilot_ctx, verification)
|
||||
_record_adjudication_on_turn_state(copilot_ctx, verification)
|
||||
|
|
@ -552,6 +555,7 @@ async def _maybe_run_completion_verification_from_page_observation(
|
|||
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
|
||||
)
|
||||
|
||||
verification = carry_degraded_criterion_ids(verification, criteria)
|
||||
if (
|
||||
isinstance(existing, CompletionVerificationResult)
|
||||
and not verification.is_fully_satisfied()
|
||||
|
|
@ -1156,6 +1160,13 @@ async def _maybe_run_completion_verification(
|
|||
return None
|
||||
criteria = _completion_verification_criteria(copilot_ctx)
|
||||
criteria = _reconcile_download_completion_criterion(copilot_ctx, result, criteria)
|
||||
verification = await _completion_verification_from_run_result(copilot_ctx, result, handler_start, criteria)
|
||||
return carry_degraded_criterion_ids(verification, criteria) if verification is not None else None
|
||||
|
||||
|
||||
async def _completion_verification_from_run_result(
|
||||
copilot_ctx: Any, result: dict[str, Any], handler_start: float, criteria: list[CompletionCriterion]
|
||||
) -> CompletionVerificationResult | None:
|
||||
run_criteria, definition_criteria = _split_criteria_by_plane(criteria)
|
||||
criterion_ids = [criterion.id for criterion in criteria]
|
||||
contingent_ids, contingent_on_by_id, contingent_path_by_id = _contingent_metadata_for_criteria(criteria)
|
||||
|
|
@ -1469,6 +1480,8 @@ def _outcome_failure_warrants_repair(
|
|||
"""
|
||||
if completion_verification is None:
|
||||
return False
|
||||
if only_degraded_blocking(completion_verification):
|
||||
return False
|
||||
if any(verdict.reason_code == "evidence_contradicts" for verdict in completion_verification.verdicts):
|
||||
return True
|
||||
# Repair needs at least one affirmatively unsatisfied criterion; unknown alone
|
||||
|
|
|
|||
|
|
@ -2345,9 +2345,8 @@ def _terminal_challenge_completion_verification(
|
|||
criterion_ids = list(completion_verification.criterion_ids)
|
||||
if not criterion_ids:
|
||||
return completion_verification
|
||||
return CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=criterion_ids,
|
||||
return replace(
|
||||
completion_verification,
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id=criterion_id,
|
||||
|
|
|
|||
471
tests/unit/test_copilot_sky11865_mint_satisfiability.py
Normal file
471
tests/unit/test_copilot_sky11865_mint_satisfiability.py
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import replace
|
||||
|
||||
from skyvern.forge.sdk.copilot.build_test_outcome import recorded_outcome_from_run_blocks_result
|
||||
from skyvern.forge.sdk.copilot.completion_criteria_store import (
|
||||
_criterion_reconcile_key,
|
||||
apply_requested_output_producer_floor,
|
||||
criteria_from_json,
|
||||
criteria_to_json,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.completion_verification import (
|
||||
CompletionVerificationResult,
|
||||
CriterionVerdict,
|
||||
RunEvidenceSnapshot,
|
||||
carry_degraded_criterion_ids,
|
||||
combine_verification_results,
|
||||
grade_fallback_floor_reached_end_state_criteria,
|
||||
only_degraded_blocking,
|
||||
structural_unfired_contingent_criterion_ids,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.request_policy import (
|
||||
CompletionCriterion,
|
||||
_classifier_fallback_policy,
|
||||
_fallback_literal_candidates_for_field,
|
||||
build_classifier_fallback_floor,
|
||||
is_turn_unsatisfiable_fallback_degraded,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
|
||||
from skyvern.forge.sdk.copilot.tools.run_execution import _terminal_challenge_completion_verification
|
||||
|
||||
_DEGRADE = "turn_unsatisfiable_fallback"
|
||||
|
||||
|
||||
def _fallback_policy(user_message: str):
|
||||
return _classifier_fallback_policy(
|
||||
[], raw_secret_present=False, failure_kind="provider_error", user_message=user_message
|
||||
)
|
||||
|
||||
|
||||
def _requested_criterion(policy, suffix: str) -> CompletionCriterion | None:
|
||||
for criterion in policy.completion_criteria:
|
||||
if criterion.id.endswith(suffix):
|
||||
return criterion
|
||||
return None
|
||||
|
||||
|
||||
def _validation_review_payload() -> dict[str, object]:
|
||||
return {
|
||||
"all_checks_passed": True,
|
||||
"validation_only": True,
|
||||
"review_page_visible": True,
|
||||
"submit_or_finalize_clicked": False,
|
||||
"submitted_request": False,
|
||||
"confirmation_page_visible": False,
|
||||
"review_values": {
|
||||
"visible_reference": "Reference AB123456",
|
||||
"visible_start_date": "2026-06-22",
|
||||
"visible_account_holder": "EXAMPLE LABS INC",
|
||||
},
|
||||
"evidence_text": (
|
||||
"Visible Review page showed Reference AB123456, start date 2026-06-22, "
|
||||
"and account holder EXAMPLE LABS INC. No final control was clicked."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_quoted_literal_mints_typed_value_and_leaves_floor_degraded() -> None:
|
||||
policy = _fallback_policy('Go to the portal and return the order number "ABC12345" and the status')
|
||||
|
||||
minted = _requested_criterion(policy, "output_order_number")
|
||||
assert minted is not None
|
||||
assert minted.expected_output_value == "ABC12345"
|
||||
assert minted.mint_degrade is None
|
||||
|
||||
floor = _requested_criterion(policy, "__copilot_fallback_floor__run")
|
||||
assert floor is not None
|
||||
assert floor.mint_degrade == _DEGRADE
|
||||
|
||||
|
||||
def test_unquoted_value_noun_degrades_instead_of_minting() -> None:
|
||||
policy = _fallback_policy("read the record; the status should be active for each location")
|
||||
|
||||
minted = _requested_criterion(policy, "output_status")
|
||||
assert minted is not None
|
||||
assert minted.expected_output_value is None
|
||||
assert minted.mint_degrade == _DEGRADE
|
||||
|
||||
|
||||
def test_extractor_mints_quoted_literal_after_binder() -> None:
|
||||
assert set(_fallback_literal_candidates_for_field('the status should be "active"', "status", "Status")) == {
|
||||
"active"
|
||||
}
|
||||
|
||||
|
||||
def test_extractor_rejects_unquoted_participle_predicate() -> None:
|
||||
assert _fallback_literal_candidates_for_field("the status should be updated", "status", "Status") == []
|
||||
|
||||
|
||||
def test_extractor_rejects_unquoted_verb_predicate() -> None:
|
||||
assert _fallback_literal_candidates_for_field("the status should be filled", "status", "Status") == []
|
||||
|
||||
|
||||
def test_extractor_rejects_unquoted_code_after_binder() -> None:
|
||||
assert _fallback_literal_candidates_for_field("order_id must be ABC12345", "order_id", "Order Id") == []
|
||||
|
||||
|
||||
def test_quoted_field_name_is_not_minted_as_value() -> None:
|
||||
policy = _fallback_policy('return the "status" field and the name for the record')
|
||||
|
||||
minted = _requested_criterion(policy, "output_status")
|
||||
assert minted is None or minted.expected_output_value is None
|
||||
|
||||
|
||||
def test_multiple_literal_candidates_mint_nothing() -> None:
|
||||
policy = _fallback_policy("the status should be active but the status should be pending")
|
||||
|
||||
minted = _requested_criterion(policy, "output_status")
|
||||
assert minted is None or minted.expected_output_value is None
|
||||
|
||||
|
||||
def test_short_literal_degrades_instead_of_minting() -> None:
|
||||
policy = _fallback_policy('return the status "ok" for the record')
|
||||
|
||||
minted = _requested_criterion(policy, "output_status")
|
||||
assert minted is None or minted.expected_output_value is None
|
||||
|
||||
|
||||
def test_value_less_fallback_marks_floor_and_requested_outputs_degraded() -> None:
|
||||
policy = _fallback_policy("return the status for the record")
|
||||
|
||||
degraded = [c for c in policy.completion_criteria if is_turn_unsatisfiable_fallback_degraded(c)]
|
||||
assert any(c.id == "__copilot_fallback_floor__run" for c in degraded)
|
||||
assert any(c.id.endswith("output_status") for c in degraded)
|
||||
|
||||
|
||||
def _degraded_floor() -> list[CompletionCriterion]:
|
||||
return [replace(criterion, mint_degrade=_DEGRADE) for criterion in build_classifier_fallback_floor([])]
|
||||
|
||||
|
||||
def _carrier_verdict() -> CriterionVerdict:
|
||||
return CriterionVerdict(
|
||||
criterion_id="__copilot_requested_output__output_reference",
|
||||
state="satisfied",
|
||||
reason_code="evidence_confirms",
|
||||
evidence_ref="block_outputs:lookup",
|
||||
evidence_source="independent_page_evidence",
|
||||
)
|
||||
|
||||
|
||||
def test_degraded_floor_still_satisfies_via_carrier_arm() -> None:
|
||||
snapshot = RunEvidenceSnapshot(block_outputs={})
|
||||
verdicts = grade_fallback_floor_reached_end_state_criteria(
|
||||
_degraded_floor(), snapshot, carrier_verdicts=(_carrier_verdict(),)
|
||||
)
|
||||
assert [verdict.state for verdict in verdicts] == ["satisfied"]
|
||||
|
||||
|
||||
def test_degraded_floor_blocks_reached_end_state_arm() -> None:
|
||||
snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})
|
||||
assert grade_fallback_floor_reached_end_state_criteria(_degraded_floor(), snapshot) == []
|
||||
|
||||
|
||||
def test_non_degraded_floor_accepts_reached_end_state_arm() -> None:
|
||||
snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})
|
||||
verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)
|
||||
assert [verdict.state for verdict in verdicts] == ["satisfied"]
|
||||
|
||||
|
||||
def test_structural_unfired_skips_degraded_criterion() -> None:
|
||||
criterion = CompletionCriterion(
|
||||
id="c_degraded",
|
||||
outcome="captured blocker",
|
||||
level="run",
|
||||
contingent_antecedent_output_path="output.blocker",
|
||||
mint_degrade=_DEGRADE,
|
||||
)
|
||||
snapshot = RunEvidenceSnapshot(block_outputs={"blocker_output": None})
|
||||
assert structural_unfired_contingent_criterion_ids([criterion], snapshot) == []
|
||||
|
||||
|
||||
def _degraded_abstention_result() -> CompletionVerificationResult:
|
||||
return CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["floor", "req"],
|
||||
verdicts=[
|
||||
CriterionVerdict(criterion_id="floor", state="satisfied", reason_code="evidence_confirms"),
|
||||
CriterionVerdict(
|
||||
criterion_id="req",
|
||||
state="unsatisfied",
|
||||
reason_code="structurally_abstained",
|
||||
evidence_ref="block_outputs:lookup",
|
||||
output_path="output.status",
|
||||
),
|
||||
],
|
||||
degraded_criterion_ids=["req"],
|
||||
)
|
||||
|
||||
|
||||
def test_degraded_abstention_is_not_credited_even_with_corroboration() -> None:
|
||||
assert _degraded_abstention_result().is_fully_satisfied() is False
|
||||
|
||||
|
||||
def test_non_degraded_abstention_credit_path_is_not_broken() -> None:
|
||||
result = replace(_degraded_abstention_result(), degraded_criterion_ids=[])
|
||||
# A satisfied observed-end-state corroborator credits the structural abstention when NOT degraded.
|
||||
result = replace(
|
||||
result,
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="floor",
|
||||
state="satisfied",
|
||||
reason_code="evidence_confirms",
|
||||
evidence_ref="observed_end_state_url",
|
||||
),
|
||||
result.verdicts[1],
|
||||
],
|
||||
)
|
||||
assert result.is_fully_satisfied() is True
|
||||
|
||||
|
||||
def test_sky11842_zero_evidence_non_fallback_still_fails_closed() -> None:
|
||||
result = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["c1"],
|
||||
verdicts=[CriterionVerdict(criterion_id="c1", state="unsatisfied", reason_code="no_evidence")],
|
||||
)
|
||||
assert result.is_fully_satisfied() is False
|
||||
assert only_degraded_blocking(result) is False
|
||||
|
||||
|
||||
def test_degraded_criterion_ids_survive_combine_and_carry() -> None:
|
||||
run_result = carry_degraded_criterion_ids(
|
||||
CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["req"],
|
||||
verdicts=[CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence")],
|
||||
),
|
||||
[CompletionCriterion(id="req", outcome="x", level="run", mint_degrade=_DEGRADE)],
|
||||
)
|
||||
assert run_result is not None
|
||||
combined = combine_verification_results(["req"], run_result, [])
|
||||
carried = carry_degraded_criterion_ids(
|
||||
combined, [CompletionCriterion(id="req", outcome="x", level="run", mint_degrade=_DEGRADE)]
|
||||
)
|
||||
assert carried is not None
|
||||
assert "req" in carried.degraded_criterion_ids
|
||||
|
||||
|
||||
def test_combine_alone_propagates_degraded_criterion_ids() -> None:
|
||||
run_result = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["req"],
|
||||
verdicts=[CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence")],
|
||||
degraded_criterion_ids=["req"],
|
||||
)
|
||||
combined = combine_verification_results(["req"], run_result, [])
|
||||
assert combined.degraded_criterion_ids == ["req"]
|
||||
assert only_degraded_blocking(combined) is True
|
||||
|
||||
|
||||
def test_fallback_literal_candidates_bounded_runtime_on_adversarial_message() -> None:
|
||||
adversarial = "status" + (" " * 20 + "=") * 4000
|
||||
start = time.perf_counter()
|
||||
_fallback_literal_candidates_for_field(adversarial, "status", "Status")
|
||||
assert time.perf_counter() - start < 2.0
|
||||
|
||||
|
||||
def test_degraded_criterion_ids_survive_terminal_challenge() -> None:
|
||||
criteria = [
|
||||
CompletionCriterion(id="__copilot_fallback_floor__run", outcome="x", level="run", mint_degrade=_DEGRADE)
|
||||
]
|
||||
result = carry_degraded_criterion_ids(
|
||||
CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["__copilot_fallback_floor__run"],
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_fallback_floor__run", state="unsatisfied", reason_code="no_evidence"
|
||||
)
|
||||
],
|
||||
),
|
||||
criteria,
|
||||
)
|
||||
assert result is not None
|
||||
assert result.degraded_criterion_ids == ["__copilot_fallback_floor__run"]
|
||||
|
||||
challenged = _terminal_challenge_completion_verification(result, "run did not reach a verifiable terminal state")
|
||||
assert challenged is not None
|
||||
assert challenged.degraded_criterion_ids == ["__copilot_fallback_floor__run"]
|
||||
assert only_degraded_blocking(challenged) is True
|
||||
|
||||
|
||||
def test_only_degraded_blocking_true_for_degraded_only() -> None:
|
||||
assert only_degraded_blocking(_degraded_abstention_result()) is True
|
||||
|
||||
|
||||
def test_only_degraded_blocking_false_on_empty_blocking_set() -> None:
|
||||
result = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["req"],
|
||||
verdicts=[CriterionVerdict(criterion_id="req", state="satisfied", reason_code="evidence_confirms")],
|
||||
degraded_criterion_ids=["req"],
|
||||
)
|
||||
assert only_degraded_blocking(result) is False
|
||||
|
||||
|
||||
def test_only_degraded_blocking_false_when_mixed_with_legitimate_unsatisfied() -> None:
|
||||
result = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["req", "legit"],
|
||||
verdicts=[
|
||||
CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence"),
|
||||
CriterionVerdict(criterion_id="legit", state="unsatisfied", reason_code="evidence_contradicts"),
|
||||
],
|
||||
degraded_criterion_ids=["req"],
|
||||
)
|
||||
assert only_degraded_blocking(result) is False
|
||||
|
||||
|
||||
def test_mint_degrade_survives_json_round_trip() -> None:
|
||||
criteria = (CompletionCriterion(id="req", outcome="return the status", level="run", mint_degrade=_DEGRADE),)
|
||||
restored = criteria_from_json(criteria_to_json(criteria))
|
||||
assert restored[0].mint_degrade == _DEGRADE
|
||||
|
||||
|
||||
def test_mint_degrade_is_identity_bearing_in_reconcile_key() -> None:
|
||||
base = CompletionCriterion(id="req", outcome="return the status", level="run", output_path="output.status")
|
||||
degraded = replace(base, mint_degrade=_DEGRADE)
|
||||
assert _criterion_reconcile_key(base) != _criterion_reconcile_key(degraded)
|
||||
|
||||
|
||||
def test_producer_floor_excludes_degraded_from_rekey() -> None:
|
||||
degraded = CompletionCriterion(
|
||||
id="__copilot_requested_output__output_status",
|
||||
outcome="return the status",
|
||||
level="run",
|
||||
output_path="output.status",
|
||||
mint_degrade=_DEGRADE,
|
||||
)
|
||||
floored, rekeyed = apply_requested_output_producer_floor([degraded])
|
||||
assert rekeyed == ()
|
||||
assert floored[0].output_path == "output.status"
|
||||
|
||||
|
||||
def _degraded_verification() -> CompletionVerificationResult:
|
||||
return CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["__copilot_fallback_floor__run"],
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_fallback_floor__run", state="unsatisfied", reason_code="no_evidence"
|
||||
)
|
||||
],
|
||||
degraded_criterion_ids=["__copilot_fallback_floor__run"],
|
||||
)
|
||||
|
||||
|
||||
def test_recorded_branch_degraded_only_is_not_repairable() -> None:
|
||||
result = {
|
||||
"data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),
|
||||
completion_verification=_degraded_verification(),
|
||||
)
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "not_authoritative"
|
||||
assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
|
||||
|
||||
|
||||
def test_raw_branch_degraded_only_is_not_repairable() -> None:
|
||||
result = {
|
||||
"ok": False,
|
||||
"data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]},
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(result, completion_verification=_degraded_verification())
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "not_authoritative"
|
||||
assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
|
||||
|
||||
|
||||
def test_raw_branch_runtime_failure_stays_repairable() -> None:
|
||||
result = {
|
||||
"ok": False,
|
||||
"data": {
|
||||
"workflow_run_id": "wr_x",
|
||||
"failure_type": "TERMINATED",
|
||||
"blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],
|
||||
},
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(result, completion_verification=_degraded_verification())
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "repairable_failure"
|
||||
|
||||
|
||||
def test_recorded_branch_blocker_reported_stays_repairable() -> None:
|
||||
result = {
|
||||
"data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="blocker_reported"),
|
||||
completion_verification=_degraded_verification(),
|
||||
)
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "repairable_failure"
|
||||
assert outcome.reason_code == "blocker_reported"
|
||||
|
||||
|
||||
def test_recorded_branch_no_reason_code_with_failed_block_stays_repairable() -> None:
|
||||
result = {
|
||||
"data": {
|
||||
"workflow_run_id": "wr_x",
|
||||
"blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],
|
||||
}
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated"),
|
||||
completion_verification=_degraded_verification(),
|
||||
)
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "repairable_failure"
|
||||
|
||||
|
||||
def test_recorded_branch_no_reason_code_without_failed_block_degrades() -> None:
|
||||
result = {
|
||||
"data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated"),
|
||||
completion_verification=_degraded_verification(),
|
||||
)
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "not_authoritative"
|
||||
assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
|
||||
|
||||
|
||||
def test_recorded_branch_no_meaningful_output_degrades() -> None:
|
||||
result = {
|
||||
"data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="no_meaningful_output"),
|
||||
completion_verification=_degraded_verification(),
|
||||
)
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "not_authoritative"
|
||||
assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
|
||||
|
||||
|
||||
def test_recorded_branch_ambiguous_code_with_failed_block_stays_repairable() -> None:
|
||||
result = {
|
||||
"data": {
|
||||
"workflow_run_id": "wr_x",
|
||||
"blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],
|
||||
}
|
||||
}
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),
|
||||
completion_verification=_degraded_verification(),
|
||||
)
|
||||
assert outcome is not None
|
||||
assert outcome.verdict == "repairable_failure"
|
||||
assert outcome.reason_code != "fallback_floor_turn_unsatisfiable"
|
||||
Loading…
Add table
Add a link
Reference in a new issue