diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py index c7b53c72d..b0e17d970 100644 --- a/skyvern/forge/sdk/copilot/build_test_outcome.py +++ b/skyvern/forge/sdk/copilot/build_test_outcome.py @@ -1561,7 +1561,7 @@ def _runtime_output_repair_facts( return [] facts: list[dict[str, object]] = [] for verdict in completion_verification.verdicts: - if verdict.satisfied or not verdict.output_path: + if not verdict.output_path: continue output_path = _bounded_ref(verdict.output_path) if not _output_path_has_child(output_path): @@ -1575,7 +1575,13 @@ def _runtime_output_repair_facts( workflow_run_id, output_path, ) - value_status = _runtime_output_value_status(values, verdict) + owner_labels = _runtime_output_owner_labels(blocks, block_labels, verdict) + if verdict.satisfied: + if not owner_labels: + continue + value_status = "satisfied" + else: + value_status = _runtime_output_value_status(values, verdict) fact: dict[str, object] = { "workflow_run_id": _bounded_ref(workflow_run_id), "output_path": output_path, @@ -1584,8 +1590,10 @@ def _runtime_output_repair_facts( "reason_code": _bounded_ref(verdict.reason_code), "value_status": value_status, } - if len(block_labels) == 1: - fact["block_label"] = block_labels[0] + if verdict.satisfied or len(owner_labels) > 1: + fact["owner_labels"] = owner_labels + if len(owner_labels) == 1: + fact["block_label"] = owner_labels[0] if verdict.grounding_mode: fact["grounding_mode"] = verdict.grounding_mode if verdict.expected_output_shape: @@ -1646,6 +1654,7 @@ def _runtime_output_values_for_path( values: list[object] = [] evidence_refs: list[str] = [] block_labels: list[str] = [] + current_labels = {label for block in blocks for label in [_bounded_ref(block.get("label"))] if label} for item in registered_output_parameter_payloads: item_run_id = _safe_str(item.get("workflow_run_id")) if item_run_id != workflow_run_id: @@ -1654,7 +1663,7 @@ def _runtime_output_values_for_path( if not present: continue values.append(value) - label = _bounded_ref(item.get("block_label")) + label = _registered_output_owner_label(item, current_labels) key = _bounded_ref(item.get("output_parameter_key")) if label: block_labels.append(label) @@ -1675,6 +1684,34 @@ def _runtime_output_values_for_path( return values, list(dict.fromkeys(evidence_refs)), sorted(dict.fromkeys(block_labels)) +def _registered_output_owner_label(item: Mapping[str, object], current_labels: set[str]) -> str: + label = _bounded_ref(item.get("block_label")) + if label in current_labels: + return label + return "" + + +def _runtime_output_owner_labels( + blocks: Sequence[Mapping[str, object]], + block_labels: Sequence[str], + verdict: CriterionVerdict, +) -> list[str]: + if not verdict.satisfied and verdict.requested_output_evidence_source == "independent_run_evidence": + return [] + current_labels = {label for block in blocks for label in [_bounded_ref(block.get("label"))] if label} + labels = {label for label in block_labels if label in current_labels} + evidence_label = _block_output_evidence_ref_label(verdict.evidence_ref) + if evidence_label in current_labels: + labels.add(evidence_label) + return sorted(labels) + + +def _block_output_evidence_ref_label(evidence_ref: str | None) -> str: + if not evidence_ref or not evidence_ref.startswith("block_outputs:"): + return "" + return _bounded_ref(evidence_ref.removeprefix("block_outputs:").split(".", 1)[0]) + + def _registered_output_value_for_path(item: Mapping[str, object], output_path: str) -> tuple[object | None, bool]: value = item.get("value") key = _safe_str(item.get("output_parameter_key")) @@ -1683,6 +1720,10 @@ def _registered_output_value_for_path(item: Mapping[str, object], output_path: s if output_path.startswith("output.") and key == output_path.split(".", 1)[1]: return value, True if isinstance(value, Mapping): + if output_path.startswith("output."): + unwrapped_value, unwrapped_present = _value_at_output_path(value, output_path.split(".", 1)[1]) + if unwrapped_present: + return unwrapped_value, True return _value_at_output_path(value, output_path) return None, False diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py index 25bdc6ff6..f3cd77416 100644 --- a/skyvern/forge/sdk/copilot/tools/workflow_update.py +++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py @@ -1848,6 +1848,8 @@ class _RuntimeOutputRepairContract: required_paths: set[str] facts: list[dict[str, Any]] workflow_run_id: str + owner_labels: list[str] + owner_labels_by_path: dict[str, list[str]] source: str = "runtime_output_repair" reason_code: str = "runtime_output_repair_required" @@ -2145,6 +2147,8 @@ def _runtime_output_contract_signature(runtime_contract: _RuntimeOutputRepairCon payload = { "workflow_run_id": runtime_contract.workflow_run_id, "required_paths": sorted(runtime_contract.required_paths), + "owner_labels": runtime_contract.owner_labels, + "owner_labels_by_path": runtime_contract.owner_labels_by_path, "facts": runtime_contract.facts, } return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest() @@ -2199,6 +2203,35 @@ def _target_output_contract_block_label( raw_code_artifact_metadata: object, required_paths: set[str], ) -> tuple[str, list[str]]: + runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx) + if runtime_contract is not None: + code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml) + runtime_owner_labels: set[str] = set() + missing_owner = False + ambiguous_owner = False + for path in sorted(runtime_contract.required_paths): + raw_path_owners = sorted(runtime_contract.owner_labels_by_path.get(path, [])) + path_owner_labels = sorted(label for label in raw_path_owners if label in code_blocks) + # Ambiguity is judged before filtering: dropping stale labels must not resolve a + # contested path down to a lone survivor. + if len(raw_path_owners) > 1: + ambiguous_owner = True + runtime_owner_labels.update(path_owner_labels) + continue + if len(path_owner_labels) != 1: + missing_owner = missing_owner or not path_owner_labels + runtime_owner_labels.update(path_owner_labels) + continue + runtime_owner_labels.add(path_owner_labels[0]) + if missing_owner: + return "", [] + current_owner_labels = sorted(runtime_owner_labels) + if ambiguous_owner: + return "", current_owner_labels + if len(current_owner_labels) == 1: + _pin_output_contract_block_label(ctx, workflow_yaml, required_paths, current_owner_labels[0]) + return current_owner_labels[0], current_owner_labels + return "", current_owner_labels pinned_label = _pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) if pinned_label: return pinned_label, [pinned_label] @@ -2226,7 +2259,8 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) -> return None facts: list[dict[str, Any]] = [] required_paths: set[str] = set() - block_labels: set[str] = set() + owner_labels: set[str] = set() + owner_labels_by_path: dict[str, set[str]] = {} for raw_fact in outcome.runtime_output_repair_facts: if not isinstance(raw_fact, Mapping): return None @@ -2238,17 +2272,24 @@ def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) -> fact = dict(raw_fact) fact["output_path"] = path fact["output_root"] = _output_path_root(path) + path_owner_labels = owner_labels_by_path.setdefault(path, set()) + raw_owner_labels = fact.get("owner_labels") + if isinstance(raw_owner_labels, list): + path_owner_labels.update(str(label).strip() for label in raw_owner_labels if str(label).strip()) label = str(fact.get("block_label") or "").strip() if label: - block_labels.add(label) + path_owner_labels.add(label) + owner_labels.update(path_owner_labels) required_paths.add(path) facts.append(fact) - if not facts or not required_paths or len(block_labels) > 1: + if not facts or not required_paths: return None return _RuntimeOutputRepairContract( required_paths=required_paths, facts=sorted(facts, key=lambda item: str(item.get("output_path") or "")), workflow_run_id=outcome.workflow_run_id, + owner_labels=sorted(owner_labels), + owner_labels_by_path={path: sorted(labels) for path, labels in sorted(owner_labels_by_path.items())}, ) diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py index e6dbea27f..6643c3afd 100644 --- a/tests/unit/test_copilot_code_block_persist_seam.py +++ b/tests/unit/test_copilot_code_block_persist_seam.py @@ -4261,6 +4261,354 @@ class TestCodeRepairProgressClassification: assert source == "runtime_output_repair" assert reason_code == "runtime_output_repair_required" + def test_runtime_output_facts_preserve_satisfied_output_owner_label(self) -> None: + result = { + "ok": True, + "data": { + "workflow_run_id": "wr_current", + "blocks": [ + { + "label": "download_statement", + "status": "completed", + "extracted_data": {"output": {"statement_pdf": "statement.pdf"}}, + } + ], + }, + } + verification = CompletionVerificationResult( + status="evaluated", + criterion_ids=["__copilot_fallback_floor__run", "requested_statement_pdf"], + verdicts=[ + CriterionVerdict( + criterion_id="__copilot_fallback_floor__run", + state="unsatisfied", + reason_code="no_evidence", + ), + CriterionVerdict( + criterion_id="requested_statement_pdf", + state="satisfied", + reason_code="evidence_confirms", + output_path="output.statement_pdf", + grounding_mode="exact_value", + expected_output_shape="string", + has_exact_value=True, + evidence_ref="block_outputs:download_statement.output.statement_pdf", + ), + ], + ) + + outcome = recorded_outcome_from_run_blocks_result( + result, + recorded_run_outcome=RecordedRunOutcome( + verdict="not_demonstrated", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + ), + completion_verification=verification, + ) + + assert outcome is not None + assert outcome.reason_code == "outcome_not_demonstrated" + assert outcome.runtime_output_repair_facts == [ + { + "workflow_run_id": "wr_current", + "block_label": "download_statement", + "owner_labels": ["download_statement"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "grounding_mode": "exact_value", + "expected_output_shape": "string", + "value_status": "satisfied", + "evidence_refs": ["output:download_statement"], + } + ] + + def test_runtime_output_facts_preserve_flat_registered_output_owner_label(self) -> None: + result = { + "ok": True, + "data": { + "workflow_run_id": "wr_current", + "blocks": [ + {"label": "apex_portal_login", "status": "completed", "extracted_data": {}}, + {"label": "apex_open_monthly_statement", "status": "completed", "extracted_data": {}}, + {"label": "apex_download_invoice_pdf", "status": "completed", "extracted_data": {}}, + ], + }, + } + verification = CompletionVerificationResult( + status="evaluated", + criterion_ids=["__copilot_authored_output__output_file_name"], + verdicts=[ + CriterionVerdict( + criterion_id="__copilot_authored_output__output_file_name", + state="unsatisfied", + reason_code="no_evidence", + output_path="output.file_name", + grounding_mode="missing", + requested_output_evidence_source="runtime_output", + ) + ], + ) + + outcome = recorded_outcome_from_run_blocks_result( + result, + recorded_run_outcome=RecordedRunOutcome( + verdict="not_demonstrated", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + ), + completion_verification=verification, + registered_output_parameter_payloads=[ + { + "workflow_run_id": "wr_current", + "block_label": "apex_download_invoice_pdf", + "output_parameter_key": "apex_download_invoice_pdf_output", + "value": { + "file_name": "statement.pdf", + "downloaded_files": [{"filename": "statement.pdf"}], + }, + } + ], + ) + + assert outcome is not None + assert outcome.runtime_output_repair_facts == [ + { + "workflow_run_id": "wr_current", + "block_label": "apex_download_invoice_pdf", + "output_path": "output.file_name", + "output_root": "output", + "criterion_id": "__copilot_authored_output__output_file_name", + "reason_code": "no_evidence", + "grounding_mode": "missing", + "value_status": "no_typed_value", + "evidence_refs": ["registered_output:apex_download_invoice_pdf:apex_download_invoice_pdf_output"], + } + ] + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = outcome + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: apex_portal_login + code: | + return {"logged_in": True} + - block_type: code + label: apex_open_monthly_statement + code: | + return {"matched": True} + - block_type: code + label: apex_download_invoice_pdf + code: | + return {"file_name": "statement.pdf"} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "apex_download_invoice_pdf" + assert evaluation.payload["output_owner_labels"] == ["apex_download_invoice_pdf"] + assert "missing_output_owner" not in evaluation.shape_violations + + def test_runtime_output_facts_do_not_infer_registered_owner_from_parameter_key(self) -> None: + result = { + "ok": True, + "data": { + "workflow_run_id": "wr_current", + "blocks": [ + {"label": "download_invoice", "status": "completed", "extracted_data": {}}, + ], + }, + } + verification = CompletionVerificationResult( + status="evaluated", + criterion_ids=["__copilot_authored_output__output_file_name"], + verdicts=[ + CriterionVerdict( + criterion_id="__copilot_authored_output__output_file_name", + state="unsatisfied", + reason_code="no_evidence", + output_path="output.file_name", + grounding_mode="missing", + requested_output_evidence_source="runtime_output", + ) + ], + ) + + outcome = recorded_outcome_from_run_blocks_result( + result, + recorded_run_outcome=RecordedRunOutcome( + verdict="not_demonstrated", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + ), + completion_verification=verification, + registered_output_parameter_payloads=[ + { + "workflow_run_id": "wr_current", + "output_parameter_key": "download_invoice_output", + "value": {"file_name": "statement.pdf"}, + } + ], + ) + + assert outcome is not None + assert outcome.runtime_output_repair_facts == [ + { + "workflow_run_id": "wr_current", + "output_path": "output.file_name", + "output_root": "output", + "criterion_id": "__copilot_authored_output__output_file_name", + "reason_code": "no_evidence", + "grounding_mode": "missing", + "value_status": "no_typed_value", + "evidence_refs": ["registered_output:unknown:download_invoice_output"], + } + ] + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = outcome + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: download_invoice + code: | + return {"file_name": "statement.pdf"} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == [] + assert "missing_output_owner" in evaluation.shape_violations + + def test_runtime_output_owner_ignores_unsatisfied_independent_self_emitted_fields(self) -> None: + result = { + "ok": True, + "data": { + "workflow_run_id": "wr_current", + "blocks": [ + { + "label": "login_to_service", + "status": "completed", + "extracted_data": {"output": {"logged_in": True}}, + }, + { + "label": "open_statement", + "status": "completed", + "extracted_data": { + "output": { + "matched": True, + "statement_date": "2026-05", + "visible_page_label": "Statement details", + } + }, + }, + ], + }, + } + verification = CompletionVerificationResult( + status="evaluated", + criterion_ids=[ + "__copilot_authored_output__output_logged_in", + "__copilot_authored_output__output_matched", + "__copilot_authored_output__output_statement_date", + "__copilot_authored_output__output_visible_page_label", + "__copilot_authored_output__output_downloaded", + ], + verdicts=[ + CriterionVerdict( + criterion_id="__copilot_authored_output__output_logged_in", + state="unsatisfied", + reason_code="structurally_abstained", + output_path="output.logged_in", + grounding_mode="missing", + requested_output_evidence_source="independent_run_evidence", + evidence_ref="block_outputs:login_to_service.output.logged_in", + ), + CriterionVerdict( + criterion_id="__copilot_authored_output__output_matched", + state="unsatisfied", + reason_code="structurally_abstained", + output_path="output.matched", + grounding_mode="missing", + requested_output_evidence_source="independent_run_evidence", + evidence_ref="block_outputs:open_statement.output.matched", + ), + CriterionVerdict( + criterion_id="__copilot_authored_output__output_statement_date", + state="unsatisfied", + reason_code="structurally_abstained", + output_path="output.statement_date", + grounding_mode="missing", + requested_output_evidence_source="runtime_output", + evidence_ref="block_outputs:open_statement.output.statement_date", + ), + CriterionVerdict( + criterion_id="__copilot_authored_output__output_visible_page_label", + state="unsatisfied", + reason_code="structurally_abstained", + output_path="output.visible_page_label", + grounding_mode="missing", + requested_output_evidence_source="runtime_output", + evidence_ref="block_outputs:open_statement.output.visible_page_label", + ), + CriterionVerdict( + criterion_id="__copilot_authored_output__output_downloaded", + state="unsatisfied", + reason_code="no_evidence", + output_path="output.downloaded", + grounding_mode="missing", + requested_output_evidence_source="independent_run_evidence", + ), + ], + ) + + outcome = recorded_outcome_from_run_blocks_result( + result, + recorded_run_outcome=RecordedRunOutcome( + verdict="not_demonstrated", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + ), + completion_verification=verification, + ) + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = outcome + workflow_yaml = _yaml( + """ + title: Statement workflow + workflow_definition: + blocks: + - block_type: code + label: login_to_service + code: | + return {"output": {"logged_in": True}} + - block_type: code + label: open_statement + code: | + return {"output": {"statement_date": "2026-05"}} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert outcome is not None + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == [] + assert "missing_output_owner" in evaluation.shape_violations + def test_runtime_output_facts_ignore_other_run_registered_values(self) -> None: result = { "ok": True, @@ -4467,6 +4815,313 @@ class TestCodeRepairProgressClassification: key=lambda item: str(item.get("output_path") or ""), ) + def test_runtime_output_owner_selects_current_block_without_metadata(self) -> None: + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "owner_labels": ["download_statement"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + } + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: login + code: | + return {"logged_in": True} + - block_type: code + label: download_statement + code: | + return {"output": {"statement_pdf": "statement.pdf"}} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "download_statement" + assert evaluation.payload["output_owner_labels"] == ["download_statement"] + assert "missing_output_owner" not in evaluation.shape_violations + + def test_runtime_output_zero_owner_does_not_fall_through_to_single_block_default(self) -> None: + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + } + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: only_block + code: | + return {"output": {"statement_pdf": "statement.pdf"}} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == [] + assert "missing_output_owner" in evaluation.shape_violations + + def test_runtime_output_multi_owner_rejects_without_picking_metadata_owner(self) -> None: + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "owner_labels": ["download_a", "download_b"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + } + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: download_a + code: | + return {"output": {"statement_pdf": "a.pdf"}} + - block_type: code + label: download_b + code: | + return {"output": {"statement_pdf": "b.pdf"}} + """ + ) + metadata = [{"block_label": "download_a", "claimed_outcomes": [{"goal_value_paths": ["output.statement_pdf"]}]}] + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, metadata) + + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == ["download_a", "download_b"] + assert "ambiguous_output_owner" in evaluation.shape_violations + + def test_runtime_output_stale_multi_owner_rejects_when_one_owner_current(self) -> None: + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "owner_labels": ["current_download", "stale_download"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + } + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: current_download + code: | + return {"output": {"statement_pdf": "a.pdf"}} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == ["current_download"] + assert "ambiguous_output_owner" in evaluation.shape_violations + + def test_runtime_output_multi_owner_all_stale_rejects_as_missing_owner(self) -> None: + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "owner_labels": ["stale_one", "stale_two"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + } + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: current_download + code: | + return {"output": {"statement_pdf": "a.pdf"}} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == [] + assert "missing_output_owner" in evaluation.shape_violations + + def test_runtime_output_paths_with_disagreeing_single_owners_reject(self) -> None: + ctx = _code_only_ctx() + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "owner_labels": ["download_pdf"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + }, + { + "workflow_run_id": "wr_current", + "owner_labels": ["extract_total"], + "output_path": "output.statement_total", + "output_root": "output", + "criterion_id": "requested_statement_total", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + }, + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: download_pdf + code: | + return {"output": {"statement_pdf": "a.pdf"}} + - block_type: code + label: extract_total + code: | + return {"output": {"statement_total": "12.00"}} + """ + ) + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "" + assert evaluation.payload["output_owner_labels"] == ["download_pdf", "extract_total"] + assert "ambiguous_output_owner" in evaluation.shape_violations + + def test_runtime_output_owner_overrides_stale_pin(self) -> None: + ctx = _code_only_ctx() + ctx.turn_id = "runtime-owner-pin" + ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome( + phase="persisted_block_run", + attempted_tool="update_and_run_blocks", + verdict="repairable_failure", + reason_code="outcome_not_demonstrated", + workflow_run_id="wr_current", + structural_failure_identity="completion:runtime-output", + runtime_output_repair_facts=[ + { + "workflow_run_id": "wr_current", + "owner_labels": ["new_owner"], + "output_path": "output.statement_pdf", + "output_root": "output", + "criterion_id": "requested_statement_pdf", + "reason_code": "evidence_confirms", + "value_status": "satisfied", + } + ], + ) + workflow_yaml = _yaml( + """ + title: Statement download + workflow_definition: + blocks: + - block_type: code + label: stale_owner + code: | + return {"output": {"statement_pdf": "old.pdf"}} + - block_type: code + label: new_owner + code: | + return {"output": {"statement_pdf": "new.pdf"}} + """ + ) + pin_key = workflow_update_module._output_contract_pin_key(ctx, workflow_yaml, {"output.statement_pdf"}) + ctx.output_contract_pinned_block_label_by_signature = {pin_key: "stale_owner"} + + evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, []) + + assert evaluation is not None + assert evaluation.block_label == "new_owner" + assert evaluation.payload["output_owner_labels"] == ["new_owner"] + @pytest.mark.asyncio async def test_runtime_output_repair_facts_trigger_one_envelope_attempt_before_budget( self, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py index 5d83ae933..d40cc5890 100644 --- a/tests/unit/test_copilot_output_contract_actuation.py +++ b/tests/unit/test_copilot_output_contract_actuation.py @@ -525,6 +525,7 @@ def _advisory_ctx() -> SimpleNamespace: recorded_build_test_outcome_history=[], scouted_output_covered_paths=set(), composition_page_evidence=None, + recorded_outcome_binding_constraint=None, )