diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py index 090cfffc1..b4156db24 100644 --- a/skyvern/forge/sdk/copilot/agent.py +++ b/skyvern/forge/sdk/copilot/agent.py @@ -10,7 +10,7 @@ import contextlib import json import re import uuid -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import ( is_optional_dismissal_only_trajectory, render_synthesized_offer_text, synthesize_code_block, + trajectory_has_browser_fill_interaction, ) from skyvern.forge.sdk.copilot.completion_criteria_store import ( StoredCriteriaSnapshot, @@ -106,19 +107,33 @@ from skyvern.forge.sdk.copilot.outcome_verification_trace import ( record_criteria_lifecycle, record_gate_decision, ) +from skyvern.forge.sdk.copilot.output_contracts import ( + OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE, + OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, +) from skyvern.forge.sdk.copilot.output_policy import ( + ACTUATION_OBLIGATION_STEER_REASON_CODE, + ACTUATION_OBLIGATION_UNMET_REASON_CODE, UNVALIDATED_DISCLOSURE_PHRASES, WORKFLOW_PRESENT_SENTINEL, + ActuationObligationEvaluation, + ActuationObligationStatus, + CannotActReason, CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict, + actuation_obligation_key, build_output_policy_diagnostics, derive_output_kind, + evaluate_actuation_obligation, evaluate_output_policy, hard_block_output_policy_verdict, normalize_response_scaffolding, output_policy_verdict_from_trace_data, output_policy_verdict_to_trace_data, + prior_turn_satisfies_actuation_terminal_condition, + request_policy_requires_durable_fill, + turn_intent_requires_actuation, ) from skyvern.forge.sdk.copilot.output_utils import ( extract_final_text, @@ -481,16 +496,7 @@ def _store_request_policy_on_context( chat_history_text=policy_inputs.chat_history_text, previous_user_message=policy_inputs.previous_user_message, ) - _reconcile_completion_criteria_on_context(ctx, policy, policy_inputs) - ctx.request_policy = policy - ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test" - ctx.user_message = agent_user_message - ctx.block_goal_main_goal = _build_block_goal_main_goal( - user_message=agent_user_message, - chat_history_text=policy_chat_history_text, - global_llm_context=policy_inputs.global_llm_context, - ) - ctx.turn_intent = build_turn_intent( + turn_intent = build_turn_intent( user_message=policy_inputs.user_message, workflow_yaml=policy_inputs.workflow_yaml, chat_history=policy_inputs.chat_history_messages, @@ -503,6 +509,16 @@ def _store_request_policy_on_context( classifier_result=turn_intent_classifier_result, fix_origin=policy_inputs.fix_origin, ) + _reconcile_completion_criteria_on_context(ctx, policy, policy_inputs) + ctx.request_policy = policy + ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test" + ctx.user_message = agent_user_message + ctx.block_goal_main_goal = _build_block_goal_main_goal( + user_message=agent_user_message, + chat_history_text=policy_chat_history_text, + global_llm_context=policy_inputs.global_llm_context, + ) + ctx.turn_intent = turn_intent def _turn_intent_log_fields(intent: TurnIntent | None) -> dict[str, Any]: @@ -1237,6 +1253,18 @@ _DRAFT_ONLY_MCP_TOOL_ALLOWLIST = frozenset({"get_block_schema", "validate_block" _DRAFT_ONLY_NATIVE_TOOL_DENYLIST = frozenset( {"discover_workflow_entrypoint", "inspect_page_for_composition", "fill_credential_field"} ) +_MUTATING_BROWSER_SCOUT_TOOLS = frozenset({"click", "press_key", "type_text", "select_option", "fill_credential_field"}) +_STRUCTURAL_CANNOT_ACT_REASON_CODES = frozenset( + {OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE} +) +_FinalActionDataValue = str | int | float | bool | None +_ACTUATION_OBLIGATION_STEER_REPLY = ( + "I need to act in the browser for this request. Please let me try the browser action, or tell me what blocks it." +) +_ACTUATION_OBLIGATION_UNMET_REPLY = ( + "I still did not act in the browser for this request, so I cannot treat the task as completed. " + "Please retry or tell me what prevented the browser action." +) def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy | None) -> bool: @@ -1248,6 +1276,108 @@ def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy | ) +def _turn_intent_requires_browser_state(turn_intent: TurnIntent | None) -> bool: + return isinstance(turn_intent, TurnIntent) and RequiredContextKey.BROWSER_STATE in turn_intent.required_context + + +def _turn_intent_browser_only_tools(turn_intent: TurnIntent | None) -> bool: + return ( + isinstance(turn_intent, TurnIntent) + and turn_intent.mode == TurnIntentMode.BUILD + and RequiredContextKey.BROWSER_STATE in turn_intent.required_context + and not turn_intent.authority.may_update_workflow + and not turn_intent.authority.may_run_blocks + ) + + +def _successful_mutating_browser_action_count(ctx: CopilotContext) -> int: + return sum( + 1 + for interaction in ctx.scout_trajectory + if isinstance(interaction, Mapping) and interaction.get("tool_name") in _MUTATING_BROWSER_SCOUT_TOOLS + ) + + +def _successful_durable_fill_count(ctx: CopilotContext) -> int: + return int(trajectory_has_browser_fill_interaction(ctx.scout_trajectory)) + + +def _typed_cannot_act_reason(ctx: CopilotContext) -> CannotActReason | None: + signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else None + if signal is None: + return None + if signal.blocker_kind == "authority_denied": + return None + if signal.blocker_kind == "missing_required_context": + return CannotActReason.MISSING_FIELD_VALUE + if signal.recovery_hint == "ask_user_clarifying": + return CannotActReason.AMBIGUOUS_TARGET + if signal.internal_reason_code in _STRUCTURAL_CANNOT_ACT_REASON_CODES: + return CannotActReason.STRUCTURAL_BLOCKER + return None + + +def _evaluate_actuation_obligation_for_output( + ctx: CopilotContext, + action_data: Mapping[str, _FinalActionDataValue], + response_type: str, + output_kind: CopilotOutputKind, +) -> ActuationObligationEvaluation: + cannot_act_reason = _typed_cannot_act_reason(ctx) + evaluation = evaluate_actuation_obligation( + turn_intent=ctx.turn_intent, + response_type=response_type, + output_kind=output_kind, + successful_mutating_browser_actions=_successful_mutating_browser_action_count(ctx), + cannot_act_reason=cannot_act_reason, + prior_turn_outcome=ctx.prior_turn_outcome, + ) + if evaluation.status != ActuationObligationStatus.ALLOWED: + return evaluation + if ( + not _turn_intent_browser_only_tools(ctx.turn_intent) + or not request_policy_requires_durable_fill(ctx.request_policy) + or response_type != "REPLY" + or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER + ): + return evaluation + if _successful_durable_fill_count(ctx) > 0 or cannot_act_reason is not None: + return evaluation + obligation_key = actuation_obligation_key(ctx.turn_intent) + if prior_turn_satisfies_actuation_terminal_condition(ctx.prior_turn_outcome, obligation_key): + return ActuationObligationEvaluation( + status=ActuationObligationStatus.TERMINAL, + reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE, + obligation_key=obligation_key, + ) + return ActuationObligationEvaluation( + status=ActuationObligationStatus.STEER, + reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE, + obligation_key=obligation_key, + ) + + +def _actuation_obligation_diagnostics( + ctx: CopilotContext, + evaluation: ActuationObligationEvaluation, +) -> dict[str, str | int | bool | None]: + if ( + not turn_intent_requires_actuation(ctx.turn_intent) + and evaluation.status == ActuationObligationStatus.ALLOWED + and evaluation.cannot_act_reason is None + ): + return {} + return { + "actuation_obligation_status": evaluation.status.value, + "actuation_obligation_reason_code": evaluation.reason_code or None, + "actuation_obligation_key": evaluation.obligation_key or None, + "cannot_act_reason": evaluation.cannot_act_reason.value if evaluation.cannot_act_reason is not None else None, + "successful_mutating_browser_actions": _successful_mutating_browser_action_count(ctx), + "durable_fill_required": request_policy_requires_durable_fill(ctx.request_policy), + "successful_durable_browser_fills": _successful_durable_fill_count(ctx), + } + + def _mcp_tool_surface_for_turn( alias_map: dict[str, str], overlays: dict[str, Any], @@ -3303,6 +3433,15 @@ async def _translate_to_agent_result( unvalidated=unvalidated, output_kind=output_kind, ) + actuation_obligation = _evaluate_actuation_obligation_for_output(ctx, action_data, resp_type, output_kind) + if actuation_obligation.status == ActuationObligationStatus.STEER: + raw_output_policy_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER) + actuation_obligation_terminal = actuation_obligation.status == ActuationObligationStatus.TERMINAL + if actuation_obligation_terminal: + resp_type = "REPLY" + user_response = _ACTUATION_OBLIGATION_UNMET_REPLY + last_workflow = None + last_workflow_yaml = None output_policy_verdict = _copy_output_policy_verdict(raw_output_policy_verdict) soft_rewrite_reasons: list[OutputPolicyReason] = [] unbacked_workflow_delivery_rewritten = False @@ -3356,6 +3495,7 @@ async def _translate_to_agent_result( hard_block_reason_codes=list(output_policy_verdict.reason_codes), soft_rewrite_reason_codes=soft_rewrite_reasons, ) + output_policy_diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation)) trace_data = output_policy_verdict_to_trace_data( output_policy_verdict, surface="final_translation", @@ -3386,15 +3526,22 @@ async def _translate_to_agent_result( else [] ) reason_code = ",".join(reason_codes) + terminal_reason = None + if actuation_obligation_terminal: + reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE + terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE final_user_response, turn_outcome = apply_repeated_reply_guard( final_text=final_user_response, attempted_kind=attempted_kind, blocked_signatures=ctx.blocked_reply_signatures, reason_code=reason_code, + terminal_reason=terminal_reason, turn_intent=ctx.turn_intent, tool_calls=[name for name in tool_call_names if name], ) + if actuation_obligation.reason_code: + turn_outcome = turn_outcome.model_copy(update={"actuation_obligation_key": actuation_obligation.obligation_key}) return _finalize_result_with_blocker_override( ctx, @@ -3595,6 +3742,7 @@ def _blocked_final_output_kind(verdict: OutputPolicyVerdict) -> CopilotOutputKin OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS, OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE, OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED, + OutputPolicyReason.ACTUATION_OBLIGATION_STEER, } if any(reason in clarification_reasons for reason in verdict.reason_codes): return CopilotOutputKind.CLARIFICATION_REQUEST @@ -3661,6 +3809,16 @@ def _evaluate_copilot_final_output_policy( workflow_attempted=workflow_attempted, output_kind=output_kind, ) + actuation_obligation = _evaluate_actuation_obligation_for_output( + ctx, + action_data, + policy_response_type, + output_kind, + ) + if actuation_obligation.status == ActuationObligationStatus.STEER: + raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER) + elif actuation_obligation.status == ActuationObligationStatus.TERMINAL: + raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_UNMET) hard_verdict = hard_block_output_policy_verdict(raw_verdict) deferred_reason_codes = _defer_avoidable_ask_to_recycle(ctx, hard_verdict, response_type) if deferred_reason_codes is not None: @@ -3674,6 +3832,7 @@ def _evaluate_copilot_final_output_policy( hard_block_reason_codes=list(hard_verdict.reason_codes), soft_rewrite_reason_codes=[], ) + diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation)) if deferred_reason_codes is not None: diagnostics["deferred_to_recycle"] = True diagnostics["deferred_reason_codes"] = [reason.value for reason in deferred_reason_codes] @@ -3878,6 +4037,10 @@ def _build_output_policy_blocked_result( "workflow without a tracked URL, re-select it so Copilot can confirm its URL scope." ) add_saved_draft_copy = True + elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes: + user_response = _ACTUATION_OBLIGATION_UNMET_REPLY + elif OutputPolicyReason.ACTUATION_OBLIGATION_STEER in verdict.reason_codes: + user_response = _ACTUATION_OBLIGATION_STEER_REPLY elif preserved_workflow is not None: user_response = ( "I could not safely return that chat reply, but the workflow draft is still saved. " @@ -3908,13 +4071,37 @@ def _build_output_policy_blocked_result( composed_from_recorded_evidence = True if preserved_workflow is not None and add_saved_draft_copy: user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}" + has_non_actuation_hard_block = any( + reason + not in { + OutputPolicyReason.ACTUATION_OBLIGATION_STEER, + OutputPolicyReason.ACTUATION_OBLIGATION_UNMET, + } + for reason in verdict.reason_codes + ) + if has_non_actuation_hard_block: + blocked_reason_code = "output_policy_block" + blocked_terminal_reason: str | None = "output_policy_block" + elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes: + blocked_reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE + blocked_terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE + else: + blocked_reason_code = ACTUATION_OBLIGATION_STEER_REASON_CODE + blocked_terminal_reason = None final_user_response, output_policy_outcome = apply_repeated_reply_guard( final_text=user_response, attempted_kind=ResponseKind.CLARIFY, blocked_signatures=ctx.blocked_reply_signatures, - reason_code="output_policy_block", - terminal_reason="output_policy_block", + reason_code=blocked_reason_code, + terminal_reason=blocked_terminal_reason, ) + if blocked_reason_code in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}: + key = "" + if output_policy_diagnostics is not None: + key = str(output_policy_diagnostics.get("actuation_obligation_key") or "") + output_policy_outcome = output_policy_outcome.model_copy( + update={"actuation_obligation_key": key or actuation_obligation_key(ctx.turn_intent)} + ) if composed_from_recorded_evidence and fallback_user_response is not None: composed_verdict = evaluate_output_policy( request_policy=ctx.request_policy, @@ -3997,6 +4184,7 @@ async def run_copilot_agent( prior_copilot_workflow_yaml: str | None = None, prior_block_count: int | None = None, stored_completion_criteria: StoredCriteriaSnapshot | None = None, + prior_turn_outcome: TurnOutcome | None = None, ) -> AgentResult: # One id per turn — passed to every downstream AgentResult and # CopilotContext so the envelope and terminal frames correlate. The @@ -4034,6 +4222,7 @@ async def run_copilot_agent( prior_block_count=prior_block_count, ctx_sink=ctx_sink, stored_completion_criteria=stored_completion_criteria, + prior_turn_outcome=prior_turn_outcome, ) except Exception as exc: LOG.error( @@ -4102,6 +4291,7 @@ async def _run_copilot_turn_impl( prior_block_count: int | None = None, ctx_sink: list[CopilotContext] | None = None, stored_completion_criteria: StoredCriteriaSnapshot | None = None, + prior_turn_outcome: TurnOutcome | None = None, ) -> AgentResult: copilot_config = config or CopilotConfig(security_rules=security_rules) chat_history_text = _format_chat_history(chat_history) @@ -4166,6 +4356,7 @@ async def _run_copilot_turn_impl( turn_index=turn_index, prior_block_count=prior_block_count, prior_copilot_workflow_yaml=prior_copilot_workflow_yaml, + prior_turn_outcome=prior_turn_outcome, block_authoring_policy=copilot_config.block_authoring_policy, impose_synthesized_code_block=copilot_config.impose_synthesized_code_block, copilot_config=copilot_config, diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py index 4ba8d5c5e..567a0d1de 100644 --- a/skyvern/forge/sdk/copilot/code_block_synthesis.py +++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py @@ -1101,6 +1101,21 @@ def trajectory_has_credential_fill(trajectory: Sequence[Mapping[str, Any]]) -> b return False +def trajectory_has_browser_fill_interaction(trajectory: Sequence[Mapping[str, Any]]) -> bool: + for interaction in trajectory: + tool_name = str(interaction.get("tool_name") or "") + typed_length = interaction.get("typed_length") + if tool_name == "type_text" and ( + (isinstance(typed_length, int) and typed_length > 0) or str(interaction.get("typed_value") or "").strip() + ): + return True + if tool_name == "select_option" and str(interaction.get("value") or "").strip(): + return True + if tool_name == CREDENTIAL_FILL_TOOL_NAME and str(interaction.get("credential_field") or "").strip(): + return True + return False + + def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]: return build_artifact_metadata_skeleton(trajectory, block_label=_SYNTHESIZED_BLOCK_LABEL) diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py index bfced6eae..b518762c3 100644 --- a/skyvern/forge/sdk/copilot/context.py +++ b/skyvern/forge/sdk/copilot/context.py @@ -643,6 +643,7 @@ class CopilotContext(AgentContext): target_block_label: str | None = None turn_intent: TurnIntent | None = None turn_context_packet: TurnContextPacket | None = None + prior_turn_outcome: TurnOutcome | None = None latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None blocked_reply_signatures: list[str] = field(default_factory=list) diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py index 8ce3dfa31..3376e3e31 100644 --- a/skyvern/forge/sdk/copilot/enforcement.py +++ b/skyvern/forge/sdk/copilot/enforcement.py @@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import ( is_optional_dismissal_only_trajectory, render_synthesized_offer_text, synthesize_code_block, + trajectory_has_browser_fill_interaction, ) from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_paths from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions @@ -76,7 +77,10 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import ( from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason from skyvern.forge.sdk.copilot.narration import TransitionKind from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState -from skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding +from skyvern.forge.sdk.copilot.output_policy import ( + completion_criterion_requires_browser_fill_delivery, + normalize_response_scaffolding, +) from skyvern.forge.sdk.copilot.output_utils import ( extract_final_text, looks_like_workflow_delivery_claim, @@ -116,7 +120,7 @@ from skyvern.forge.sdk.copilot.turn_halt import ( stash_repair_ceiling_turn_halt, stash_turn_halt_from_blocker_signal, ) -from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode +from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode from skyvern.utils.token_counter import count_tokens if TYPE_CHECKING: @@ -188,6 +192,7 @@ SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks" _SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS = frozenset( {SYNTHESIZED_BLOCK_PERSISTENCE_TOOL, "fill_credential_field", "update_workflow"} ) +_ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL = "type_text" _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS = frozenset( {"click", "press_key", "type_text", "select_option", "navigate_browser"} ) @@ -2394,6 +2399,39 @@ def _uncovered_output_reject_admits_evaluate(ctx: CopilotContext, tool_name: str return tool_name == "evaluate" and bool(_active_uncovered_output_reject_paths(ctx)) +def _actuation_obligation_live_fill_delivery_required(ctx: CopilotContext) -> bool: + turn_intent = getattr(ctx, "turn_intent", None) + if ( + not isinstance(turn_intent, TurnIntent) + or turn_intent.mode != TurnIntentMode.BUILD + or RequiredContextKey.BROWSER_STATE not in turn_intent.required_context + ): + return False + if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER: + return False + request_policy = getattr(ctx, "request_policy", None) + criteria: list[CompletionCriterion] = [] + if isinstance(request_policy, RequestPolicy): + criteria.extend(request_policy.completion_criteria) + turn_state = getattr(ctx, "completion_criteria_turn_state", None) + if turn_state is not None and turn_state.decision is not None: + criteria.extend(turn_state.decision.criteria) + if any(completion_criterion_requires_browser_fill_delivery(criterion) for criterion in criteria): + return True + trajectory = getattr(ctx, "scout_trajectory", None) + return trajectory_has_browser_fill_interaction(trajectory) if isinstance(trajectory, list) else False + + +def _actuation_obligation_required_fill_tool(ctx: CopilotContext) -> str | None: + if _actuation_obligation_live_fill_delivery_required(ctx): + return _ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL + return None + + +def _actuation_obligation_admits_required_fill_tool(ctx: CopilotContext, tool_name: str) -> bool: + return tool_name == _actuation_obligation_required_fill_tool(ctx) + + def consume_uncovered_output_reopen_event(ctx: CopilotContext) -> bool: """Arm a one-shot scout-window reopen for the first author-time reject citing an uncovered requested-output path. Returns True only on that first reject per structural identity; a @@ -2501,6 +2539,8 @@ def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToo return None if _uncovered_output_reject_admits_evaluate(ctx, tool_name): return None + if _actuation_obligation_admits_required_fill_tool(ctx, tool_name): + return None if ( ambiguous_selector_rescout_state != "block" and not _should_force_synthesized_block_persistence(ctx) diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py index 08581034f..caa8b04ea 100644 --- a/skyvern/forge/sdk/copilot/output_policy.py +++ b/skyvern/forge/sdk/copilot/output_policy.py @@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.output_utils import ( looks_like_workflow_yaml_in_chat, ) from skyvern.forge.sdk.copilot.request_policy import ( + CompletionCriterion, RequestPolicy, contains_email_password_pair, request_policy_has_present_completion_contract, @@ -21,6 +22,7 @@ from skyvern.forge.sdk.copilot.secret_redaction import ( RAW_SECRET_PATTERNS, SECRET_KEYWORD_ASSIGNMENT_PATTERN, ) +from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode from skyvern.forge.sdk.copilot.workflow_credential_utils import ( block_credential_ids, credential_param_ids, @@ -30,8 +32,12 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import ( workflow_credential_ids_from_parsed, workflow_credential_origins_from_parsed, ) +from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome WORKFLOW_PRESENT_SENTINEL = object() +ACTUATION_OBLIGATION_STEER_REASON_CODE = "actuation_obligation_steer" +ACTUATION_OBLIGATION_UNMET_REASON_CODE = "actuation_obligation_unmet" +ACTUATION_OBLIGATION_BROWSER_ACTION_KEY = "browser_state:build:no_update:no_run" _CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b") _PLACEHOLDER_MARKERS = ("{{", "{%", "[REDACTED_SECRET]") # RHS of a secret-keyword assignment that references a bound value instead of carrying one: @@ -183,6 +189,26 @@ class CopilotOutputKind(StrEnum): WORKFLOW_RUN_RESULT = "workflow_run_result" +class CannotActReason(StrEnum): + MISSING_FIELD_VALUE = "missing_field_value" + AMBIGUOUS_TARGET = "ambiguous_target" + STRUCTURAL_BLOCKER = "structural_blocker" + + +class ActuationObligationStatus(StrEnum): + ALLOWED = "allowed" + STEER = "steer" + TERMINAL = "terminal" + + +@dataclass(frozen=True) +class ActuationObligationEvaluation: + status: ActuationObligationStatus = ActuationObligationStatus.ALLOWED + reason_code: str = "" + cannot_act_reason: CannotActReason | None = None + obligation_key: str = "" + + class OutputPolicyReason(StrEnum): RAW_SECRET_LEAK = "raw_secret_leak" REQUEST_POLICY_CLARIFICATION_BYPASS = "request_policy_clarification_bypass" @@ -199,6 +225,8 @@ class OutputPolicyReason(StrEnum): SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak" WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply" AVOIDABLE_OUTPUT_FIELD_CONFIRMATION = "avoidable_output_field_confirmation" + ACTUATION_OBLIGATION_STEER = ACTUATION_OBLIGATION_STEER_REASON_CODE + ACTUATION_OBLIGATION_UNMET = ACTUATION_OBLIGATION_UNMET_REASON_CODE @dataclass @@ -232,10 +260,97 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset( OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK, OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING, OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION, + OutputPolicyReason.ACTUATION_OBLIGATION_STEER, + OutputPolicyReason.ACTUATION_OBLIGATION_UNMET, } ) +def coerce_cannot_act_reason(value: str | None) -> CannotActReason | None: + if value is None: + return None + try: + return CannotActReason(value) + except ValueError: + return None + + +def turn_intent_requires_actuation(turn_intent: TurnIntent | None) -> bool: + return ( + isinstance(turn_intent, TurnIntent) + and turn_intent.mode in {TurnIntentMode.BUILD, TurnIntentMode.UNKNOWN} + and RequiredContextKey.BROWSER_STATE in turn_intent.required_context + and not turn_intent.authority.may_update_workflow + and not turn_intent.authority.may_run_blocks + ) + + +def actuation_obligation_key(turn_intent: TurnIntent | None) -> str: + if not turn_intent_requires_actuation(turn_intent): + return "" + if turn_intent is not None and turn_intent.mode == TurnIntentMode.UNKNOWN: + return "browser_state:unknown:no_update:no_run" + return ACTUATION_OBLIGATION_BROWSER_ACTION_KEY + + +def completion_criterion_requires_browser_fill_delivery(criterion: CompletionCriterion) -> bool: + if criterion.kind == "terminal_action" and criterion.terminal_action_family == "form": + return True + return criterion.kind == "outcome" and criterion.level == "run" and criterion.method_mandated + + +def request_policy_requires_durable_fill(request_policy: RequestPolicy | None) -> bool: + return isinstance(request_policy, RequestPolicy) and any( + completion_criterion_requires_browser_fill_delivery(criterion) + for criterion in request_policy.completion_criteria + ) + + +def prior_turn_satisfies_actuation_terminal_condition( + prior_turn_outcome: TurnOutcome | None, + obligation_key: str, +) -> bool: + # One key covers the turn's actuation obligation; a later partial action + # still failed the same browser-fill contract. + return ( + prior_turn_outcome is not None + and prior_turn_outcome.reason_code + in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE} + and prior_turn_outcome.actuation_obligation_key == obligation_key + ) + + +def evaluate_actuation_obligation( + *, + turn_intent: TurnIntent | None, + response_type: str, + output_kind: CopilotOutputKind, + successful_mutating_browser_actions: int, + cannot_act_reason: CannotActReason | None, + prior_turn_outcome: TurnOutcome | None, +) -> ActuationObligationEvaluation: + obligation_key = actuation_obligation_key(turn_intent) + if ( + not turn_intent_requires_actuation(turn_intent) + or response_type != "REPLY" + or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER + ): + return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason) + if successful_mutating_browser_actions > 0 or cannot_act_reason is not None: + return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason, obligation_key=obligation_key) + if prior_turn_satisfies_actuation_terminal_condition(prior_turn_outcome, obligation_key): + return ActuationObligationEvaluation( + status=ActuationObligationStatus.TERMINAL, + reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE, + obligation_key=obligation_key, + ) + return ActuationObligationEvaluation( + status=ActuationObligationStatus.STEER, + reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE, + obligation_key=obligation_key, + ) + + def hard_block_output_policy_verdict(verdict: OutputPolicyVerdict) -> OutputPolicyVerdict: hard_reasons = [reason for reason in verdict.reason_codes if reason in _FINAL_OUTPUT_HARD_BLOCK_REASONS] return OutputPolicyVerdict( diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py index 960a333ff..72fc572b7 100644 --- a/skyvern/forge/sdk/routes/workflow_copilot.py +++ b/skyvern/forge/sdk/routes/workflow_copilot.py @@ -2069,6 +2069,7 @@ async def _new_copilot_chat_post( prior_copilot_workflow_yaml=prior_copilot_workflow_yaml, prior_block_count=prior_block_count, stored_completion_criteria=stored_completion_criteria, + prior_turn_outcome=prior_turn_outcome, ) agent_result.turn_outcome = _with_current_copilot_code_mode_metadata( diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py index 23b55a413..8e4d50a48 100644 --- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py +++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py @@ -29,6 +29,7 @@ class TurnOutcome(BaseModel): turn_intent_summary: dict[str, Any] = Field(default_factory=dict) response_kind: ResponseKind reason_code: str = "" + actuation_obligation_key: str = "" normalized_reply_signature: str = "" tool_calls: list[str] = Field(default_factory=list) terminal_reason: str | None = None diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.py index 399ccfc98..fc306f105 100644 --- a/tests/unit/test_copilot_agent_blocker_renderer.py +++ b/tests/unit/test_copilot_agent_blocker_renderer.py @@ -27,7 +27,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import ( ) from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext -from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict +from skyvern.forge.sdk.copilot.output_policy import ( + ACTUATION_OBLIGATION_STEER_REASON_CODE, + CopilotOutputKind, + OutputPolicyReason, + OutputPolicyVerdict, +) from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome from skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKind @@ -338,6 +343,18 @@ def test_output_policy_specific_branches_bypass_recorded_terminal_fallback() -> assert result.narrative_payload["responseType"] == "ASK_QUESTION" +def test_output_policy_hard_block_priority_over_actuation_steer() -> None: + ctx = _ctx() + + result = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK, OutputPolicyReason.ACTUATION_OBLIGATION_STEER) + + assert result.user_response == _RAW_SECRET_LEAK_REFUSAL + assert result.turn_outcome is not None + assert result.turn_outcome.reason_code == "output_policy_block" + assert result.turn_outcome.terminal_reason == "output_policy_block" + assert result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE + + def test_shim_preserves_workflow_draft_when_signal_opts_in() -> None: """Late-block-running blockers say 'wrap up with what you have' — they surface the saved draft alongside the rendered chat reply.""" diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py index 070c962b0..f9fd3a836 100644 --- a/tests/unit/test_copilot_agent_helpers.py +++ b/tests/unit/test_copilot_agent_helpers.py @@ -60,6 +60,11 @@ from skyvern.forge.sdk.copilot.enforcement import ( verified_goal_satisfied_context, ) from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE +from skyvern.forge.sdk.copilot.output_policy import ( + ACTUATION_OBLIGATION_BROWSER_ACTION_KEY, + ACTUATION_OBLIGATION_STEER_REASON_CODE, + ACTUATION_OBLIGATION_UNMET_REASON_CODE, +) from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure from skyvern.forge.sdk.copilot.request_policy import ( _MAX_COMPLETION_CRITERIA, @@ -90,12 +95,14 @@ from skyvern.forge.sdk.copilot.turn_halt import ( stash_repair_ceiling_turn_halt, ) from skyvern.forge.sdk.copilot.turn_intent import ( + RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode, TurnIntentReasonCode, ) from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence +from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome from skyvern.forge.sdk.schemas.workflow_copilot import ( WorkflowCopilotChatHistoryMessage, WorkflowCopilotChatSender, @@ -1640,7 +1647,6 @@ class TestRequestPolicyInputGuardrail: request_policy_handler=request_policy_handler, turn_intent_handler=turn_intent_handler, ) - guardrails = agent_module._build_copilot_input_guardrails( InputGuardrail, GuardrailFunctionOutput, @@ -1886,6 +1892,30 @@ def _chat_request() -> SimpleNamespace: ) +def _browser_actuation_intent() -> TurnIntent: + return TurnIntent( + mode=TurnIntentMode.BUILD, + required_context=[RequiredContextKey.BROWSER_STATE], + authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False), + ) + + +def _unknown_browser_actuation_intent() -> TurnIntent: + return TurnIntent( + mode=TurnIntentMode.UNKNOWN, + required_context=[RequiredContextKey.BROWSER_STATE], + authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False), + ) + + +def _browser_actuation_intent_without_browser_state(*, mode: TurnIntentMode) -> TurnIntent: + return TurnIntent( + mode=mode, + required_context=[], + authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False), + ) + + class TestBlockGoalMainGoal: def test_empty_message_returns_empty(self) -> None: assert agent_module._build_block_goal_main_goal("", chat_history_text="", global_llm_context=None) == "" @@ -2268,6 +2298,69 @@ class TestTranslateToAgentResultGating: assert agent_result.response_type == "ASK_QUESTION" assert agent_result.user_response == "Which account should I use?" + def test_actuation_required_reply_without_browser_mutation_returns_steer(self) -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent()) + result = _fake_run_result({"type": "REPLY", "user_response": "I can fill those fields for you."}) + + agent_result = asyncio.run( + agent_module._translate_to_agent_result( + result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1" + ) + ) + + assert agent_result.response_type == "ASK_QUESTION" + assert agent_result.turn_outcome is not None + assert agent_result.turn_outcome.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE + assert agent_result.turn_outcome.actuation_obligation_key == ACTUATION_OBLIGATION_BROWSER_ACTION_KEY + + def test_repeated_actuation_required_reply_without_browser_mutation_returns_terminal(self) -> None: + ctx = _ctx( + turn_intent=_browser_actuation_intent(), + prior_turn_outcome=TurnOutcome( + response_kind=ResponseKind.CLARIFY, + reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE, + actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY, + ), + ) + result = _fake_run_result({"type": "REPLY", "user_response": "I can fill those fields for you."}) + + agent_result = asyncio.run( + agent_module._translate_to_agent_result( + result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1" + ) + ) + + assert agent_result.response_type == "REPLY" + assert agent_result.updated_workflow is None + assert agent_result.turn_outcome is not None + assert agent_result.turn_outcome.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE + assert agent_result.turn_outcome.terminal_reason == ACTUATION_OBLIGATION_UNMET_REASON_CODE + assert agent_result.turn_outcome.actuation_obligation_key == ACTUATION_OBLIGATION_BROWSER_ACTION_KEY + + def test_unknown_click_with_authority_denied_blocker_returns_reply(self) -> None: + ctx = _ctx(turn_intent=_unknown_browser_actuation_intent()) + ctx.scout_trajectory.append({"tool_name": "click"}) + ctx.blocker_signal = CopilotToolBlockerSignal( + blocker_kind="authority_denied", + agent_steering_text="Use browser tools.", + user_facing_reason="I'll respond with the information I already have.", + recovery_hint="report_blocker_to_user", + internal_reason_code="turn_intent_no_mutation_run_blocked", + blocked_tool="update_and_run_blocks", + classifier_mode="unknown", + ) + result = _fake_run_result({"type": "REPLY", "user_response": "I'll respond with the information I have."}) + + agent_result = asyncio.run( + agent_module._translate_to_agent_result( + result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1" + ) + ) + + assert agent_result.response_type == "REPLY" + assert agent_result.turn_outcome is not None + assert agent_result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE + def test_verified_terminal_state_surfaces_workflow_despite_weak_final_reply(self) -> None: workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[])) ctx = _ctx( diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py index fb7f5d4dc..8c227dee8 100644 --- a/tests/unit/test_copilot_enforcement_pruning.py +++ b/tests/unit/test_copilot_enforcement_pruning.py @@ -65,7 +65,7 @@ from skyvern.forge.sdk.copilot.output_contracts import ( OutputContractAdvisoryState, ) from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget -from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion +from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy from skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_tool from skyvern.forge.sdk.copilot.tools import ( _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY, @@ -75,7 +75,7 @@ from skyvern.forge.sdk.copilot.tools import ( _record_workflow_update_result, ) from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal -from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode +from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence from tests.unit.conftest import make_copilot_context @@ -128,6 +128,7 @@ class _Ctx: self.completion_criteria_turn_state = None self.reached_download_target: ReachedDownloadTarget | None = None self.author_time_gate_ablation_events = [] + self.request_policy = None class TestSynthesizedOfferPersistenceGate: @@ -340,6 +341,185 @@ class TestSynthesizedOfferPersistenceGate: assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"}) assert signal.renders_final_reply is False + def test_actuation_obligation_admits_required_fill_tool_during_persistence_offer(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy( + completion_criteria=[ + CompletionCriterion( + id="form-submit", + outcome="form fields are filled", + kind="terminal_action", + terminal_action_family="form", + ) + ], + ) + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 1 + ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}] + + assert synthesized_block_persistence_signal(ctx, "type_text") is None + click_signal = synthesized_block_persistence_signal(ctx, "click") + assert isinstance(click_signal, CopilotToolBlockerSignal) + assert click_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE + + def test_actuation_obligation_admits_required_fill_tool_for_method_mandated_run_contract(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy( + completion_criteria=[ + CompletionCriterion( + id="visible-fill", + outcome="fields are visibly filled on the live page", + method_mandated=True, + level="run", + ) + ], + ) + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 1 + ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}] + + assert synthesized_block_persistence_signal(ctx, "type_text") is None + click_signal = synthesized_block_persistence_signal(ctx, "click") + assert isinstance(click_signal, CopilotToolBlockerSignal) + assert click_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE + + def test_actuation_obligation_blocks_type_text_for_definition_method_contract(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy( + completion_criteria=[ + CompletionCriterion( + id="definition-contract", + outcome="inputs are reusable", + method_mandated=True, + level="definition", + ) + ], + ) + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 1 + ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}] + + signal = synthesized_block_persistence_signal(ctx, "type_text") + + assert isinstance(signal, CopilotToolBlockerSignal) + assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE + + def test_actuation_obligation_admits_required_fill_tool_after_turn_state_reconcile(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy( + completion_criteria=[ + CompletionCriterion( + id="form-submit", + outcome="form fields are filled", + kind="terminal_action", + terminal_action_family="form", + ) + ], + ) + ctx.completion_criteria_turn_state = SimpleNamespace( + decision=SimpleNamespace( + criteria=( + CompletionCriterion( + id="workflow-run", + outcome="workflow has been tested", + kind="terminal_action", + terminal_action_family="workflow_run", + ), + ) + ) + ) + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 2 + ctx.scout_trajectory = [ + {"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}, + {"tool_name": "type_text", "selector": "#company", "accessible_name": "Company"}, + ] + + assert synthesized_block_persistence_signal(ctx, "type_text") is None + + def test_persistence_offer_blocks_type_text_without_actuation_obligation(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy() + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 1 + ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}] + + signal = synthesized_block_persistence_signal(ctx, "type_text") + + assert isinstance(signal, CopilotToolBlockerSignal) + assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE + assert signal.blocked_tool == "type_text" + + def test_actuation_obligation_admits_required_fill_tool_from_fill_trajectory(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy() + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 1 + ctx.scout_trajectory = [ + { + "tool_name": "type_text", + "selector": "#company", + "typed_value": "Example Realty Labs Inc", + "accessible_name": "Company", + } + ] + + assert synthesized_block_persistence_signal(ctx, "type_text") is None + + def test_fill_less_trajectory_does_not_arm_actuation_obligation(self) -> None: + ctx = _Ctx() + ctx.turn_intent = TurnIntent( + mode=TurnIntentMode.BUILD, + authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True), + required_context={RequiredContextKey.BROWSER_STATE}, + ) + ctx.request_policy = RequestPolicy() + ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER + ctx.synthesized_block_offered = True + ctx.synthesized_block_offered_trajectory_len = 1 + ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}] + + signal = synthesized_block_persistence_signal(ctx, "type_text") + + assert isinstance(signal, CopilotToolBlockerSignal) + assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE + def test_prerun_ambiguous_bare_selector_repair_allows_one_evaluate(self) -> None: ctx = _Ctx() ctx.turn_intent = TurnIntent( diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py index befb08ec0..898ea3e3a 100644 --- a/tests/unit/test_copilot_output_policy.py +++ b/tests/unit/test_copilot_output_policy.py @@ -21,12 +21,19 @@ from skyvern.forge.sdk.copilot.enforcement import ( PRESENT_COMPLETION_CONTRACT_ASK_RETRY, _response_coverage_nudge, ) +from skyvern.forge.sdk.copilot.output_contracts import OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE from skyvern.forge.sdk.copilot.output_policy import ( + ACTUATION_OBLIGATION_BROWSER_ACTION_KEY, + ACTUATION_OBLIGATION_STEER_REASON_CODE, + ACTUATION_OBLIGATION_UNMET_REASON_CODE, + ActuationObligationStatus, + CannotActReason, CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict, _contains_internal_tool_vocab_leak, derive_output_kind, + evaluate_actuation_obligation, evaluate_output_policy, hard_block_output_policy_verdict, normalize_response_scaffolding, @@ -36,7 +43,8 @@ from skyvern.forge.sdk.copilot.tools import ( _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL, NATIVE_TOOLS, ) -from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode +from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode +from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome def _credential(credential_id: str = "cred_safe", tested_url: str = "https://login.example.test/login") -> object: @@ -63,6 +71,43 @@ def _ctx(**overrides: object) -> CopilotContext: return CopilotContext(**defaults) +def _browser_actuation_intent() -> TurnIntent: + return TurnIntent( + mode=TurnIntentMode.BUILD, + required_context=[RequiredContextKey.BROWSER_STATE], + authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False), + ) + + +def _live_fill_policy() -> RequestPolicy: + return _policy( + completion_criteria=[ + CompletionCriterion( + id="c0", + outcome="The form is completed.", + kind="terminal_action", + terminal_action_family="form", + ) + ] + ) + + +def _unknown_browser_actuation_intent() -> TurnIntent: + return TurnIntent( + mode=TurnIntentMode.UNKNOWN, + required_context=[RequiredContextKey.BROWSER_STATE], + authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False), + ) + + +def _browser_actuation_intent_without_browser_state(*, mode: TurnIntentMode) -> TurnIntent: + return TurnIntent( + mode=mode, + required_context=[], + authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False), + ) + + def _fake_run_result(payload: dict) -> SimpleNamespace: return SimpleNamespace(final_output=json.dumps(payload), new_items=[]) @@ -1384,6 +1429,384 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None: } +def test_actuation_obligation_steers_zero_action_informational_reply() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=None, + ) + + assert evaluation.status == ActuationObligationStatus.STEER + assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE + + +def test_actuation_obligation_steers_unknown_browser_only_intent() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_unknown_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=None, + ) + + assert evaluation.status == ActuationObligationStatus.STEER + assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE + + +def test_actuation_obligation_allows_typed_cannot_act_reason() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=CannotActReason.MISSING_FIELD_VALUE, + prior_turn_outcome=None, + ) + + assert evaluation.status == ActuationObligationStatus.ALLOWED + assert evaluation.cannot_act_reason == CannotActReason.MISSING_FIELD_VALUE + + +def test_actuation_obligation_repeats_as_terminal() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=TurnOutcome( + response_kind=ResponseKind.CLARIFY, + reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE, + actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY, + ), + ) + + assert evaluation.status == ActuationObligationStatus.TERMINAL + assert evaluation.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE + + +def test_actuation_obligation_prior_unmet_stays_terminal() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=TurnOutcome( + response_kind=ResponseKind.CLARIFY, + reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE, + terminal_reason=ACTUATION_OBLIGATION_UNMET_REASON_CODE, + actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY, + ), + ) + + assert evaluation.status == ActuationObligationStatus.TERMINAL + assert evaluation.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE + + +def test_actuation_obligation_unrelated_prior_steer_stays_recoverable() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=TurnOutcome( + response_kind=ResponseKind.CLARIFY, + reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE, + actuation_obligation_key="other", + ), + ) + + assert evaluation.status == ActuationObligationStatus.STEER + assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE + + +@pytest.mark.parametrize("mode", [TurnIntentMode.DOCS_ANSWER, TurnIntentMode.DIAGNOSE]) +def test_actuation_obligation_allows_non_actuation_intents(mode: TurnIntentMode) -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=TurnIntent(mode=mode, authority=TurnIntentAuthority(may_update_workflow=False)), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=None, + ) + + assert evaluation.status == ActuationObligationStatus.ALLOWED + + +def test_sdk_output_guardrail_steers_actuation_required_zero_action_reply() -> None: + evaluation = evaluate_actuation_obligation( + turn_intent=_browser_actuation_intent(), + response_type="REPLY", + output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER, + successful_mutating_browser_actions=0, + cannot_act_reason=None, + prior_turn_outcome=None, + ) + + assert evaluation.status == ActuationObligationStatus.STEER + assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE + + +def test_sdk_output_guardrail_steers_actuation_required_reply() -> None: + ctx = _ctx( + turn_intent=_browser_actuation_intent(), + tool_activity=[{"tool": "evaluate"}, {"tool": "get_browser_screenshot"}], + ) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "I can fill those fields for you."}, + ) + + assert response_type == "REPLY" + assert not verdict.allowed + assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER] + assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE + assert diagnostics["successful_mutating_browser_actions"] == 0 + assert diagnostics.get("deferred_to_recycle", False) is False + + +@pytest.mark.asyncio +async def test_sdk_output_guardrail_trips_actuation_steer() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent()) + output_guardrails = agent_module._build_copilot_output_guardrails(OutputGuardrail, GuardrailFunctionOutput) + result = await output_guardrails[0].run( + RunContextWrapper(context=ctx), + SimpleNamespace(), + {"type": "REPLY", "user_response": "I can fill those fields for you."}, + ) + + assert result.output.tripwire_triggered is True + assert result.output.output_info["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE + + +def test_sdk_output_guardrail_repeated_actuation_steer_terminalizes() -> None: + ctx = _ctx( + turn_intent=_browser_actuation_intent(), + prior_turn_outcome=TurnOutcome( + response_kind=ResponseKind.CLARIFY, + reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE, + actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY, + ), + ) + output = {"type": "REPLY", "user_response": "I can fill those fields for you."} + + verdict, _, diagnostics = agent_module._evaluate_copilot_final_output_policy(ctx, output) + + assert not verdict.allowed + assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_UNMET] + assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.TERMINAL.value + assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_UNMET_REASON_CODE + + +def test_sdk_output_guardrail_rejects_uncorroborated_model_typed_cannot_act_reason() -> None: + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + _ctx(turn_intent=_browser_actuation_intent()), + { + "type": "REPLY", + "user_response": "I need the field value before I can continue.", + "cannot_act_reason": "missing_field_value", + }, + ) + + assert response_type == "REPLY" + assert not verdict.allowed + assert diagnostics["cannot_act_reason"] is None + assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE + + +def test_sdk_output_guardrail_allows_structural_blocker_reason() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent()) + ctx.blocker_signal = CopilotToolBlockerSignal( + blocker_kind="missing_required_context", + agent_steering_text="Ask which field value to use.", + user_facing_reason="I need the field value before I can continue.", + recovery_hint="ask_user_clarifying", + ) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "I need the field value before I can continue."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics["cannot_act_reason"] == "missing_field_value" + + +def test_sdk_output_guardrail_allows_explicit_structural_blocker_reason_code() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent()) + ctx.blocker_signal = CopilotToolBlockerSignal( + blocker_kind="tool_error", + agent_steering_text="Report the unavailable output source.", + user_facing_reason="The values are not observable on this page.", + recovery_hint="report_blocker_to_user", + internal_reason_code=OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, + ) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "The values are not observable on this page."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics["cannot_act_reason"] == "structural_blocker" + + +def test_sdk_output_guardrail_rejects_generic_report_blocker_reason() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent()) + ctx.blocker_signal = CopilotToolBlockerSignal( + blocker_kind="tool_error", + agent_steering_text="Report the blocker.", + user_facing_reason="The page is not ready.", + recovery_hint="report_blocker_to_user", + internal_reason_code="tool_error_late_block_running", + ) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "The page is not ready."}, + ) + + assert response_type == "REPLY" + assert not verdict.allowed + assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER] + assert diagnostics["cannot_act_reason"] is None + + +def test_sdk_output_guardrail_allows_successful_mutating_browser_action() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent()) + ctx.scout_trajectory.append({"tool_name": "click"}) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "Clicked the field."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics["successful_mutating_browser_actions"] == 1 + + +def test_sdk_output_guardrail_rejects_form_fill_click_only_reply() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy()) + ctx.scout_trajectory.append({"tool_name": "click"}) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "The requested form interaction was completed."}, + ) + + assert response_type == "REPLY" + assert not verdict.allowed + assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER] + assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE + assert diagnostics["successful_mutating_browser_actions"] == 1 + assert diagnostics["successful_durable_browser_fills"] == 0 + assert diagnostics["durable_fill_required"] is True + + +def test_sdk_output_guardrail_requires_browser_state_for_form_fill_obligation() -> None: + ctx = _ctx( + turn_intent=_browser_actuation_intent_without_browser_state(mode=TurnIntentMode.BUILD), + request_policy=_live_fill_policy(), + ) + ctx.scout_trajectory.append({"tool_name": "click"}) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "The requested form interaction was completed."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics.get("deferred_to_recycle", False) is False + assert "durable_fill_required" not in diagnostics + + +def test_sdk_output_guardrail_allows_non_form_click_reply() -> None: + ctx = _ctx( + turn_intent=_browser_actuation_intent_without_browser_state(mode=TurnIntentMode.UNKNOWN), + ) + ctx.scout_trajectory.append({"tool_name": "click"}) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "Clicked the requested target."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics.get("deferred_to_recycle", False) is False + assert "durable_fill_required" not in diagnostics + + +def test_sdk_output_guardrail_allows_live_fill_after_durable_fill() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy()) + ctx.scout_trajectory.append({"tool_name": "type_text", "typed_length": 3}) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "The requested form interaction was completed."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics.get("deferred_to_recycle", False) is False + assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.ALLOWED.value + assert diagnostics["successful_durable_browser_fills"] == 1 + + +def test_sdk_output_guardrail_rejects_live_fill_after_noop_type_text() -> None: + ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy()) + ctx.scout_trajectory.append({"tool_name": "type_text", "typed_length": 0, "typed_value": ""}) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "The requested form interaction was completed."}, + ) + + assert response_type == "REPLY" + assert not verdict.allowed + assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE + assert diagnostics["successful_durable_browser_fills"] == 0 + assert diagnostics["durable_fill_required"] is True + + +def test_sdk_output_guardrail_allows_unknown_click_with_authority_denied_blocker() -> None: + ctx = _ctx(turn_intent=_unknown_browser_actuation_intent()) + ctx.scout_trajectory.append({"tool_name": "click"}) + ctx.blocker_signal = CopilotToolBlockerSignal( + blocker_kind="authority_denied", + agent_steering_text="Use browser tools.", + user_facing_reason="I'll respond with the information I already have.", + recovery_hint="report_blocker_to_user", + internal_reason_code="turn_intent_no_mutation_run_blocked", + blocked_tool="update_and_run_blocks", + classifier_mode="unknown", + ) + + verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy( + ctx, + {"type": "REPLY", "user_response": "I'll respond with the information I already have."}, + ) + + assert response_type == "REPLY" + assert verdict.allowed + assert diagnostics.get("deferred_to_recycle", False) is False + assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.ALLOWED.value + assert diagnostics["cannot_act_reason"] is None + assert diagnostics["successful_mutating_browser_actions"] == 1 + + @pytest.mark.parametrize( ("reason", "expected_terms"), [