mirror of
https://github.com/Skyvern-AI/skyvern.git
synced 2026-07-09 16:09:13 +00:00
fix(SKY-11958): fail closed on ambiguous output-contract owner resolution (#7223)
This commit is contained in:
Andrew Neilson
GitHub
parent
8b78d8e8f1
commit
bb7fe7158f
4 changed files with 746 additions and 8 deletions
|
|
@ -1561,7 +1561,7 @@ def _runtime_output_repair_facts(
|
|||
return []
|
||||
facts: list[dict[str, object]] = []
|
||||
for verdict in completion_verification.verdicts:
|
||||
if verdict.satisfied or not verdict.output_path:
|
||||
if not verdict.output_path:
|
||||
continue
|
||||
output_path = _bounded_ref(verdict.output_path)
|
||||
if not _output_path_has_child(output_path):
|
||||
|
|
@ -1575,7 +1575,13 @@ def _runtime_output_repair_facts(
|
|||
workflow_run_id,
|
||||
output_path,
|
||||
)
|
||||
value_status = _runtime_output_value_status(values, verdict)
|
||||
owner_labels = _runtime_output_owner_labels(blocks, block_labels, verdict)
|
||||
if verdict.satisfied:
|
||||
if not owner_labels:
|
||||
continue
|
||||
value_status = "satisfied"
|
||||
else:
|
||||
value_status = _runtime_output_value_status(values, verdict)
|
||||
fact: dict[str, object] = {
|
||||
"workflow_run_id": _bounded_ref(workflow_run_id),
|
||||
"output_path": output_path,
|
||||
|
|
@ -1584,8 +1590,10 @@ def _runtime_output_repair_facts(
|
|||
"reason_code": _bounded_ref(verdict.reason_code),
|
||||
"value_status": value_status,
|
||||
}
|
||||
if len(block_labels) == 1:
|
||||
fact["block_label"] = block_labels[0]
|
||||
if verdict.satisfied or len(owner_labels) > 1:
|
||||
fact["owner_labels"] = owner_labels
|
||||
if len(owner_labels) == 1:
|
||||
fact["block_label"] = owner_labels[0]
|
||||
if verdict.grounding_mode:
|
||||
fact["grounding_mode"] = verdict.grounding_mode
|
||||
if verdict.expected_output_shape:
|
||||
|
|
@ -1646,6 +1654,7 @@ def _runtime_output_values_for_path(
|
|||
values: list[object] = []
|
||||
evidence_refs: list[str] = []
|
||||
block_labels: list[str] = []
|
||||
current_labels = {label for block in blocks for label in [_bounded_ref(block.get("label"))] if label}
|
||||
for item in registered_output_parameter_payloads:
|
||||
item_run_id = _safe_str(item.get("workflow_run_id"))
|
||||
if item_run_id != workflow_run_id:
|
||||
|
|
@ -1654,7 +1663,7 @@ def _runtime_output_values_for_path(
|
|||
if not present:
|
||||
continue
|
||||
values.append(value)
|
||||
label = _bounded_ref(item.get("block_label"))
|
||||
label = _registered_output_owner_label(item, current_labels)
|
||||
key = _bounded_ref(item.get("output_parameter_key"))
|
||||
if label:
|
||||
block_labels.append(label)
|
||||
|
|
@ -1675,6 +1684,34 @@ def _runtime_output_values_for_path(
|
|||
return values, list(dict.fromkeys(evidence_refs)), sorted(dict.fromkeys(block_labels))
|
||||
|
||||
|
||||
def _registered_output_owner_label(item: Mapping[str, object], current_labels: set[str]) -> str:
|
||||
label = _bounded_ref(item.get("block_label"))
|
||||
if label in current_labels:
|
||||
return label
|
||||
return ""
|
||||
|
||||
|
||||
def _runtime_output_owner_labels(
|
||||
blocks: Sequence[Mapping[str, object]],
|
||||
block_labels: Sequence[str],
|
||||
verdict: CriterionVerdict,
|
||||
) -> list[str]:
|
||||
if not verdict.satisfied and verdict.requested_output_evidence_source == "independent_run_evidence":
|
||||
return []
|
||||
current_labels = {label for block in blocks for label in [_bounded_ref(block.get("label"))] if label}
|
||||
labels = {label for label in block_labels if label in current_labels}
|
||||
evidence_label = _block_output_evidence_ref_label(verdict.evidence_ref)
|
||||
if evidence_label in current_labels:
|
||||
labels.add(evidence_label)
|
||||
return sorted(labels)
|
||||
|
||||
|
||||
def _block_output_evidence_ref_label(evidence_ref: str | None) -> str:
|
||||
if not evidence_ref or not evidence_ref.startswith("block_outputs:"):
|
||||
return ""
|
||||
return _bounded_ref(evidence_ref.removeprefix("block_outputs:").split(".", 1)[0])
|
||||
|
||||
|
||||
def _registered_output_value_for_path(item: Mapping[str, object], output_path: str) -> tuple[object | None, bool]:
|
||||
value = item.get("value")
|
||||
key = _safe_str(item.get("output_parameter_key"))
|
||||
|
|
@ -1683,6 +1720,10 @@ def _registered_output_value_for_path(item: Mapping[str, object], output_path: s
|
|||
if output_path.startswith("output.") and key == output_path.split(".", 1)[1]:
|
||||
return value, True
|
||||
if isinstance(value, Mapping):
|
||||
if output_path.startswith("output."):
|
||||
unwrapped_value, unwrapped_present = _value_at_output_path(value, output_path.split(".", 1)[1])
|
||||
if unwrapped_present:
|
||||
return unwrapped_value, True
|
||||
return _value_at_output_path(value, output_path)
|
||||
return None, False
|
||||
|
||||
|
|
|
|||
|
|
@ -1848,6 +1848,8 @@ class _RuntimeOutputRepairContract:
|
|||
required_paths: set[str]
|
||||
facts: list[dict[str, Any]]
|
||||
workflow_run_id: str
|
||||
owner_labels: list[str]
|
||||
owner_labels_by_path: dict[str, list[str]]
|
||||
source: str = "runtime_output_repair"
|
||||
reason_code: str = "runtime_output_repair_required"
|
||||
|
||||
|
|
@ -2145,6 +2147,8 @@ def _runtime_output_contract_signature(runtime_contract: _RuntimeOutputRepairCon
|
|||
payload = {
|
||||
"workflow_run_id": runtime_contract.workflow_run_id,
|
||||
"required_paths": sorted(runtime_contract.required_paths),
|
||||
"owner_labels": runtime_contract.owner_labels,
|
||||
"owner_labels_by_path": runtime_contract.owner_labels_by_path,
|
||||
"facts": runtime_contract.facts,
|
||||
}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
|
||||
|
|
@ -2199,6 +2203,35 @@ def _target_output_contract_block_label(
|
|||
raw_code_artifact_metadata: object,
|
||||
required_paths: set[str],
|
||||
) -> tuple[str, list[str]]:
|
||||
runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
|
||||
if runtime_contract is not None:
|
||||
code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
|
||||
runtime_owner_labels: set[str] = set()
|
||||
missing_owner = False
|
||||
ambiguous_owner = False
|
||||
for path in sorted(runtime_contract.required_paths):
|
||||
raw_path_owners = sorted(runtime_contract.owner_labels_by_path.get(path, []))
|
||||
path_owner_labels = sorted(label for label in raw_path_owners if label in code_blocks)
|
||||
# Ambiguity is judged before filtering: dropping stale labels must not resolve a
|
||||
# contested path down to a lone survivor.
|
||||
if len(raw_path_owners) > 1:
|
||||
ambiguous_owner = True
|
||||
runtime_owner_labels.update(path_owner_labels)
|
||||
continue
|
||||
if len(path_owner_labels) != 1:
|
||||
missing_owner = missing_owner or not path_owner_labels
|
||||
runtime_owner_labels.update(path_owner_labels)
|
||||
continue
|
||||
runtime_owner_labels.add(path_owner_labels[0])
|
||||
if missing_owner:
|
||||
return "", []
|
||||
current_owner_labels = sorted(runtime_owner_labels)
|
||||
if ambiguous_owner:
|
||||
return "", current_owner_labels
|
||||
if len(current_owner_labels) == 1:
|
||||
_pin_output_contract_block_label(ctx, workflow_yaml, required_paths, current_owner_labels[0])
|
||||
return current_owner_labels[0], current_owner_labels
|
||||
return "", current_owner_labels
|
||||
pinned_label = _pinned_output_contract_block_label(ctx, workflow_yaml, required_paths)
|
||||
if pinned_label:
|
||||
return pinned_label, [pinned_label]
|
||||
|
|
@ -2226,7 +2259,8 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) ->
|
|||
return None
|
||||
facts: list[dict[str, Any]] = []
|
||||
required_paths: set[str] = set()
|
||||
block_labels: set[str] = set()
|
||||
owner_labels: set[str] = set()
|
||||
owner_labels_by_path: dict[str, set[str]] = {}
|
||||
for raw_fact in outcome.runtime_output_repair_facts:
|
||||
if not isinstance(raw_fact, Mapping):
|
||||
return None
|
||||
|
|
@ -2238,17 +2272,24 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) ->
|
|||
fact = dict(raw_fact)
|
||||
fact["output_path"] = path
|
||||
fact["output_root"] = _output_path_root(path)
|
||||
path_owner_labels = owner_labels_by_path.setdefault(path, set())
|
||||
raw_owner_labels = fact.get("owner_labels")
|
||||
if isinstance(raw_owner_labels, list):
|
||||
path_owner_labels.update(str(label).strip() for label in raw_owner_labels if str(label).strip())
|
||||
label = str(fact.get("block_label") or "").strip()
|
||||
if label:
|
||||
block_labels.add(label)
|
||||
path_owner_labels.add(label)
|
||||
owner_labels.update(path_owner_labels)
|
||||
required_paths.add(path)
|
||||
facts.append(fact)
|
||||
if not facts or not required_paths or len(block_labels) > 1:
|
||||
if not facts or not required_paths:
|
||||
return None
|
||||
return _RuntimeOutputRepairContract(
|
||||
required_paths=required_paths,
|
||||
facts=sorted(facts, key=lambda item: str(item.get("output_path") or "")),
|
||||
workflow_run_id=outcome.workflow_run_id,
|
||||
owner_labels=sorted(owner_labels),
|
||||
owner_labels_by_path={path: sorted(labels) for path, labels in sorted(owner_labels_by_path.items())},
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4261,6 +4261,354 @@ class TestCodeRepairProgressClassification:
|
|||
assert source == "runtime_output_repair"
|
||||
assert reason_code == "runtime_output_repair_required"
|
||||
|
||||
def test_runtime_output_facts_preserve_satisfied_output_owner_label(self) -> None:
|
||||
result = {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"workflow_run_id": "wr_current",
|
||||
"blocks": [
|
||||
{
|
||||
"label": "download_statement",
|
||||
"status": "completed",
|
||||
"extracted_data": {"output": {"statement_pdf": "statement.pdf"}},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
verification = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["__copilot_fallback_floor__run", "requested_statement_pdf"],
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_fallback_floor__run",
|
||||
state="unsatisfied",
|
||||
reason_code="no_evidence",
|
||||
),
|
||||
CriterionVerdict(
|
||||
criterion_id="requested_statement_pdf",
|
||||
state="satisfied",
|
||||
reason_code="evidence_confirms",
|
||||
output_path="output.statement_pdf",
|
||||
grounding_mode="exact_value",
|
||||
expected_output_shape="string",
|
||||
has_exact_value=True,
|
||||
evidence_ref="block_outputs:download_statement.output.statement_pdf",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(
|
||||
verdict="not_demonstrated",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
),
|
||||
completion_verification=verification,
|
||||
)
|
||||
|
||||
assert outcome is not None
|
||||
assert outcome.reason_code == "outcome_not_demonstrated"
|
||||
assert outcome.runtime_output_repair_facts == [
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"block_label": "download_statement",
|
||||
"owner_labels": ["download_statement"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"grounding_mode": "exact_value",
|
||||
"expected_output_shape": "string",
|
||||
"value_status": "satisfied",
|
||||
"evidence_refs": ["output:download_statement"],
|
||||
}
|
||||
]
|
||||
|
||||
def test_runtime_output_facts_preserve_flat_registered_output_owner_label(self) -> None:
|
||||
result = {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"workflow_run_id": "wr_current",
|
||||
"blocks": [
|
||||
{"label": "apex_portal_login", "status": "completed", "extracted_data": {}},
|
||||
{"label": "apex_open_monthly_statement", "status": "completed", "extracted_data": {}},
|
||||
{"label": "apex_download_invoice_pdf", "status": "completed", "extracted_data": {}},
|
||||
],
|
||||
},
|
||||
}
|
||||
verification = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["__copilot_authored_output__output_file_name"],
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_file_name",
|
||||
state="unsatisfied",
|
||||
reason_code="no_evidence",
|
||||
output_path="output.file_name",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="runtime_output",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(
|
||||
verdict="not_demonstrated",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
),
|
||||
completion_verification=verification,
|
||||
registered_output_parameter_payloads=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"block_label": "apex_download_invoice_pdf",
|
||||
"output_parameter_key": "apex_download_invoice_pdf_output",
|
||||
"value": {
|
||||
"file_name": "statement.pdf",
|
||||
"downloaded_files": [{"filename": "statement.pdf"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert outcome is not None
|
||||
assert outcome.runtime_output_repair_facts == [
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"block_label": "apex_download_invoice_pdf",
|
||||
"output_path": "output.file_name",
|
||||
"output_root": "output",
|
||||
"criterion_id": "__copilot_authored_output__output_file_name",
|
||||
"reason_code": "no_evidence",
|
||||
"grounding_mode": "missing",
|
||||
"value_status": "no_typed_value",
|
||||
"evidence_refs": ["registered_output:apex_download_invoice_pdf:apex_download_invoice_pdf_output"],
|
||||
}
|
||||
]
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = outcome
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: apex_portal_login
|
||||
code: |
|
||||
return {"logged_in": True}
|
||||
- block_type: code
|
||||
label: apex_open_monthly_statement
|
||||
code: |
|
||||
return {"matched": True}
|
||||
- block_type: code
|
||||
label: apex_download_invoice_pdf
|
||||
code: |
|
||||
return {"file_name": "statement.pdf"}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == "apex_download_invoice_pdf"
|
||||
assert evaluation.payload["output_owner_labels"] == ["apex_download_invoice_pdf"]
|
||||
assert "missing_output_owner" not in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_facts_do_not_infer_registered_owner_from_parameter_key(self) -> None:
|
||||
result = {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"workflow_run_id": "wr_current",
|
||||
"blocks": [
|
||||
{"label": "download_invoice", "status": "completed", "extracted_data": {}},
|
||||
],
|
||||
},
|
||||
}
|
||||
verification = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=["__copilot_authored_output__output_file_name"],
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_file_name",
|
||||
state="unsatisfied",
|
||||
reason_code="no_evidence",
|
||||
output_path="output.file_name",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="runtime_output",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(
|
||||
verdict="not_demonstrated",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
),
|
||||
completion_verification=verification,
|
||||
registered_output_parameter_payloads=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"output_parameter_key": "download_invoice_output",
|
||||
"value": {"file_name": "statement.pdf"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert outcome is not None
|
||||
assert outcome.runtime_output_repair_facts == [
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"output_path": "output.file_name",
|
||||
"output_root": "output",
|
||||
"criterion_id": "__copilot_authored_output__output_file_name",
|
||||
"reason_code": "no_evidence",
|
||||
"grounding_mode": "missing",
|
||||
"value_status": "no_typed_value",
|
||||
"evidence_refs": ["registered_output:unknown:download_invoice_output"],
|
||||
}
|
||||
]
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = outcome
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: download_invoice
|
||||
code: |
|
||||
return {"file_name": "statement.pdf"}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == []
|
||||
assert "missing_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_owner_ignores_unsatisfied_independent_self_emitted_fields(self) -> None:
|
||||
result = {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"workflow_run_id": "wr_current",
|
||||
"blocks": [
|
||||
{
|
||||
"label": "login_to_service",
|
||||
"status": "completed",
|
||||
"extracted_data": {"output": {"logged_in": True}},
|
||||
},
|
||||
{
|
||||
"label": "open_statement",
|
||||
"status": "completed",
|
||||
"extracted_data": {
|
||||
"output": {
|
||||
"matched": True,
|
||||
"statement_date": "2026-05",
|
||||
"visible_page_label": "Statement details",
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
verification = CompletionVerificationResult(
|
||||
status="evaluated",
|
||||
criterion_ids=[
|
||||
"__copilot_authored_output__output_logged_in",
|
||||
"__copilot_authored_output__output_matched",
|
||||
"__copilot_authored_output__output_statement_date",
|
||||
"__copilot_authored_output__output_visible_page_label",
|
||||
"__copilot_authored_output__output_downloaded",
|
||||
],
|
||||
verdicts=[
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_logged_in",
|
||||
state="unsatisfied",
|
||||
reason_code="structurally_abstained",
|
||||
output_path="output.logged_in",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="independent_run_evidence",
|
||||
evidence_ref="block_outputs:login_to_service.output.logged_in",
|
||||
),
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_matched",
|
||||
state="unsatisfied",
|
||||
reason_code="structurally_abstained",
|
||||
output_path="output.matched",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="independent_run_evidence",
|
||||
evidence_ref="block_outputs:open_statement.output.matched",
|
||||
),
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_statement_date",
|
||||
state="unsatisfied",
|
||||
reason_code="structurally_abstained",
|
||||
output_path="output.statement_date",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="runtime_output",
|
||||
evidence_ref="block_outputs:open_statement.output.statement_date",
|
||||
),
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_visible_page_label",
|
||||
state="unsatisfied",
|
||||
reason_code="structurally_abstained",
|
||||
output_path="output.visible_page_label",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="runtime_output",
|
||||
evidence_ref="block_outputs:open_statement.output.visible_page_label",
|
||||
),
|
||||
CriterionVerdict(
|
||||
criterion_id="__copilot_authored_output__output_downloaded",
|
||||
state="unsatisfied",
|
||||
reason_code="no_evidence",
|
||||
output_path="output.downloaded",
|
||||
grounding_mode="missing",
|
||||
requested_output_evidence_source="independent_run_evidence",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
outcome = recorded_outcome_from_run_blocks_result(
|
||||
result,
|
||||
recorded_run_outcome=RecordedRunOutcome(
|
||||
verdict="not_demonstrated",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
),
|
||||
completion_verification=verification,
|
||||
)
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = outcome
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement workflow
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: login_to_service
|
||||
code: |
|
||||
return {"output": {"logged_in": True}}
|
||||
- block_type: code
|
||||
label: open_statement
|
||||
code: |
|
||||
return {"output": {"statement_date": "2026-05"}}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert outcome is not None
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == []
|
||||
assert "missing_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_facts_ignore_other_run_registered_values(self) -> None:
|
||||
result = {
|
||||
"ok": True,
|
||||
|
|
@ -4467,6 +4815,313 @@ class TestCodeRepairProgressClassification:
|
|||
key=lambda item: str(item.get("output_path") or ""),
|
||||
)
|
||||
|
||||
def test_runtime_output_owner_selects_current_block_without_metadata(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["download_statement"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
}
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: login
|
||||
code: |
|
||||
return {"logged_in": True}
|
||||
- block_type: code
|
||||
label: download_statement
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "statement.pdf"}}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == "download_statement"
|
||||
assert evaluation.payload["output_owner_labels"] == ["download_statement"]
|
||||
assert "missing_output_owner" not in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_zero_owner_does_not_fall_through_to_single_block_default(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
}
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: only_block
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "statement.pdf"}}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == []
|
||||
assert "missing_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_multi_owner_rejects_without_picking_metadata_owner(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["download_a", "download_b"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
}
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: download_a
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "a.pdf"}}
|
||||
- block_type: code
|
||||
label: download_b
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "b.pdf"}}
|
||||
"""
|
||||
)
|
||||
metadata = [{"block_label": "download_a", "claimed_outcomes": [{"goal_value_paths": ["output.statement_pdf"]}]}]
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, metadata)
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == ["download_a", "download_b"]
|
||||
assert "ambiguous_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_stale_multi_owner_rejects_when_one_owner_current(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["current_download", "stale_download"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
}
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: current_download
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "a.pdf"}}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == ["current_download"]
|
||||
assert "ambiguous_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_multi_owner_all_stale_rejects_as_missing_owner(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["stale_one", "stale_two"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
}
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: current_download
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "a.pdf"}}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == []
|
||||
assert "missing_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_paths_with_disagreeing_single_owners_reject(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["download_pdf"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
},
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["extract_total"],
|
||||
"output_path": "output.statement_total",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_total",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
},
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: download_pdf
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "a.pdf"}}
|
||||
- block_type: code
|
||||
label: extract_total
|
||||
code: |
|
||||
return {"output": {"statement_total": "12.00"}}
|
||||
"""
|
||||
)
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == ""
|
||||
assert evaluation.payload["output_owner_labels"] == ["download_pdf", "extract_total"]
|
||||
assert "ambiguous_output_owner" in evaluation.shape_violations
|
||||
|
||||
def test_runtime_output_owner_overrides_stale_pin(self) -> None:
|
||||
ctx = _code_only_ctx()
|
||||
ctx.turn_id = "runtime-owner-pin"
|
||||
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
|
||||
phase="persisted_block_run",
|
||||
attempted_tool="update_and_run_blocks",
|
||||
verdict="repairable_failure",
|
||||
reason_code="outcome_not_demonstrated",
|
||||
workflow_run_id="wr_current",
|
||||
structural_failure_identity="completion:runtime-output",
|
||||
runtime_output_repair_facts=[
|
||||
{
|
||||
"workflow_run_id": "wr_current",
|
||||
"owner_labels": ["new_owner"],
|
||||
"output_path": "output.statement_pdf",
|
||||
"output_root": "output",
|
||||
"criterion_id": "requested_statement_pdf",
|
||||
"reason_code": "evidence_confirms",
|
||||
"value_status": "satisfied",
|
||||
}
|
||||
],
|
||||
)
|
||||
workflow_yaml = _yaml(
|
||||
"""
|
||||
title: Statement download
|
||||
workflow_definition:
|
||||
blocks:
|
||||
- block_type: code
|
||||
label: stale_owner
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "old.pdf"}}
|
||||
- block_type: code
|
||||
label: new_owner
|
||||
code: |
|
||||
return {"output": {"statement_pdf": "new.pdf"}}
|
||||
"""
|
||||
)
|
||||
pin_key = workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, {"output.statement_pdf"})
|
||||
ctx.output_contract_pinned_block_label_by_signature = {pin_key: "stale_owner"}
|
||||
|
||||
evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
|
||||
|
||||
assert evaluation is not None
|
||||
assert evaluation.block_label == "new_owner"
|
||||
assert evaluation.payload["output_owner_labels"] == ["new_owner"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_output_repair_facts_trigger_one_envelope_attempt_before_budget(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
|
|||
|
|
@ -525,6 +525,7 @@ def _advisory_ctx() -> SimpleNamespace:
|
|||
recorded_build_test_outcome_history=[],
|
||||
scouted_output_covered_paths=set(),
|
||||
composition_page_evidence=None,
|
||||
recorded_outcome_binding_constraint=None,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue