mirror of
https://github.com/Skyvern-AI/skyvern.git
synced 2026-07-09 16:09:13 +00:00
This commit is contained in:
Shuchang Zheng
GitHub
parent
d79359d55c
commit
bf3624d788
11 changed files with 1097 additions and 20 deletions
|
|
@ -10,7 +10,7 @@ import contextlib
|
|||
import json
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
|
@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
|
|||
is_optional_dismissal_only_trajectory,
|
||||
render_synthesized_offer_text,
|
||||
synthesize_code_block,
|
||||
trajectory_has_browser_fill_interaction,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.completion_criteria_store import (
|
||||
StoredCriteriaSnapshot,
|
||||
|
|
@ -106,19 +107,33 @@ from skyvern.forge.sdk.copilot.outcome_verification_trace import (
|
|||
record_criteria_lifecycle,
|
||||
record_gate_decision,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.output_contracts import (
|
||||
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
|
||||
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.output_policy import (
|
||||
ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
UNVALIDATED_DISCLOSURE_PHRASES,
|
||||
WORKFLOW_PRESENT_SENTINEL,
|
||||
ActuationObligationEvaluation,
|
||||
ActuationObligationStatus,
|
||||
CannotActReason,
|
||||
CopilotOutputKind,
|
||||
OutputPolicyReason,
|
||||
OutputPolicyVerdict,
|
||||
actuation_obligation_key,
|
||||
build_output_policy_diagnostics,
|
||||
derive_output_kind,
|
||||
evaluate_actuation_obligation,
|
||||
evaluate_output_policy,
|
||||
hard_block_output_policy_verdict,
|
||||
normalize_response_scaffolding,
|
||||
output_policy_verdict_from_trace_data,
|
||||
output_policy_verdict_to_trace_data,
|
||||
prior_turn_satisfies_actuation_terminal_condition,
|
||||
request_policy_requires_durable_fill,
|
||||
turn_intent_requires_actuation,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.output_utils import (
|
||||
extract_final_text,
|
||||
|
|
@ -481,16 +496,7 @@ def _store_request_policy_on_context(
|
|||
chat_history_text=policy_inputs.chat_history_text,
|
||||
previous_user_message=policy_inputs.previous_user_message,
|
||||
)
|
||||
_reconcile_completion_criteria_on_context(ctx, policy, policy_inputs)
|
||||
ctx.request_policy = policy
|
||||
ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test"
|
||||
ctx.user_message = agent_user_message
|
||||
ctx.block_goal_main_goal = _build_block_goal_main_goal(
|
||||
user_message=agent_user_message,
|
||||
chat_history_text=policy_chat_history_text,
|
||||
global_llm_context=policy_inputs.global_llm_context,
|
||||
)
|
||||
ctx.turn_intent = build_turn_intent(
|
||||
turn_intent = build_turn_intent(
|
||||
user_message=policy_inputs.user_message,
|
||||
workflow_yaml=policy_inputs.workflow_yaml,
|
||||
chat_history=policy_inputs.chat_history_messages,
|
||||
|
|
@ -503,6 +509,16 @@ def _store_request_policy_on_context(
|
|||
classifier_result=turn_intent_classifier_result,
|
||||
fix_origin=policy_inputs.fix_origin,
|
||||
)
|
||||
_reconcile_completion_criteria_on_context(ctx, policy, policy_inputs)
|
||||
ctx.request_policy = policy
|
||||
ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test"
|
||||
ctx.user_message = agent_user_message
|
||||
ctx.block_goal_main_goal = _build_block_goal_main_goal(
|
||||
user_message=agent_user_message,
|
||||
chat_history_text=policy_chat_history_text,
|
||||
global_llm_context=policy_inputs.global_llm_context,
|
||||
)
|
||||
ctx.turn_intent = turn_intent
|
||||
|
||||
|
||||
def _turn_intent_log_fields(intent: TurnIntent | None) -> dict[str, Any]:
|
||||
|
|
@ -1237,6 +1253,18 @@ _DRAFT_ONLY_MCP_TOOL_ALLOWLIST = frozenset({"get_block_schema", "validate_block"
|
|||
_DRAFT_ONLY_NATIVE_TOOL_DENYLIST = frozenset(
|
||||
{"discover_workflow_entrypoint", "inspect_page_for_composition", "fill_credential_field"}
|
||||
)
|
||||
_MUTATING_BROWSER_SCOUT_TOOLS = frozenset({"click", "press_key", "type_text", "select_option", "fill_credential_field"})
|
||||
_STRUCTURAL_CANNOT_ACT_REASON_CODES = frozenset(
|
||||
{OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE}
|
||||
)
|
||||
_FinalActionDataValue = str | int | float | bool | None
|
||||
_ACTUATION_OBLIGATION_STEER_REPLY = (
|
||||
"I need to act in the browser for this request. Please let me try the browser action, or tell me what blocks it."
|
||||
)
|
||||
_ACTUATION_OBLIGATION_UNMET_REPLY = (
|
||||
"I still did not act in the browser for this request, so I cannot treat the task as completed. "
|
||||
"Please retry or tell me what prevented the browser action."
|
||||
)
|
||||
|
||||
|
||||
def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy | None) -> bool:
|
||||
|
|
@ -1248,6 +1276,108 @@ def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy |
|
|||
)
|
||||
|
||||
|
||||
def _turn_intent_requires_browser_state(turn_intent: TurnIntent | None) -> bool:
|
||||
return isinstance(turn_intent, TurnIntent) and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
|
||||
|
||||
|
||||
def _turn_intent_browser_only_tools(turn_intent: TurnIntent | None) -> bool:
|
||||
return (
|
||||
isinstance(turn_intent, TurnIntent)
|
||||
and turn_intent.mode == TurnIntentMode.BUILD
|
||||
and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
|
||||
and not turn_intent.authority.may_update_workflow
|
||||
and not turn_intent.authority.may_run_blocks
|
||||
)
|
||||
|
||||
|
||||
def _successful_mutating_browser_action_count(ctx: CopilotContext) -> int:
|
||||
return sum(
|
||||
1
|
||||
for interaction in ctx.scout_trajectory
|
||||
if isinstance(interaction, Mapping) and interaction.get("tool_name") in _MUTATING_BROWSER_SCOUT_TOOLS
|
||||
)
|
||||
|
||||
|
||||
def _successful_durable_fill_count(ctx: CopilotContext) -> int:
|
||||
return int(trajectory_has_browser_fill_interaction(ctx.scout_trajectory))
|
||||
|
||||
|
||||
def _typed_cannot_act_reason(ctx: CopilotContext) -> CannotActReason | None:
|
||||
signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else None
|
||||
if signal is None:
|
||||
return None
|
||||
if signal.blocker_kind == "authority_denied":
|
||||
return None
|
||||
if signal.blocker_kind == "missing_required_context":
|
||||
return CannotActReason.MISSING_FIELD_VALUE
|
||||
if signal.recovery_hint == "ask_user_clarifying":
|
||||
return CannotActReason.AMBIGUOUS_TARGET
|
||||
if signal.internal_reason_code in _STRUCTURAL_CANNOT_ACT_REASON_CODES:
|
||||
return CannotActReason.STRUCTURAL_BLOCKER
|
||||
return None
|
||||
|
||||
|
||||
def _evaluate_actuation_obligation_for_output(
|
||||
ctx: CopilotContext,
|
||||
action_data: Mapping[str, _FinalActionDataValue],
|
||||
response_type: str,
|
||||
output_kind: CopilotOutputKind,
|
||||
) -> ActuationObligationEvaluation:
|
||||
cannot_act_reason = _typed_cannot_act_reason(ctx)
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=ctx.turn_intent,
|
||||
response_type=response_type,
|
||||
output_kind=output_kind,
|
||||
successful_mutating_browser_actions=_successful_mutating_browser_action_count(ctx),
|
||||
cannot_act_reason=cannot_act_reason,
|
||||
prior_turn_outcome=ctx.prior_turn_outcome,
|
||||
)
|
||||
if evaluation.status != ActuationObligationStatus.ALLOWED:
|
||||
return evaluation
|
||||
if (
|
||||
not _turn_intent_browser_only_tools(ctx.turn_intent)
|
||||
or not request_policy_requires_durable_fill(ctx.request_policy)
|
||||
or response_type != "REPLY"
|
||||
or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER
|
||||
):
|
||||
return evaluation
|
||||
if _successful_durable_fill_count(ctx) > 0 or cannot_act_reason is not None:
|
||||
return evaluation
|
||||
obligation_key = actuation_obligation_key(ctx.turn_intent)
|
||||
if prior_turn_satisfies_actuation_terminal_condition(ctx.prior_turn_outcome, obligation_key):
|
||||
return ActuationObligationEvaluation(
|
||||
status=ActuationObligationStatus.TERMINAL,
|
||||
reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
obligation_key=obligation_key,
|
||||
)
|
||||
return ActuationObligationEvaluation(
|
||||
status=ActuationObligationStatus.STEER,
|
||||
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
obligation_key=obligation_key,
|
||||
)
|
||||
|
||||
|
||||
def _actuation_obligation_diagnostics(
|
||||
ctx: CopilotContext,
|
||||
evaluation: ActuationObligationEvaluation,
|
||||
) -> dict[str, str | int | bool | None]:
|
||||
if (
|
||||
not turn_intent_requires_actuation(ctx.turn_intent)
|
||||
and evaluation.status == ActuationObligationStatus.ALLOWED
|
||||
and evaluation.cannot_act_reason is None
|
||||
):
|
||||
return {}
|
||||
return {
|
||||
"actuation_obligation_status": evaluation.status.value,
|
||||
"actuation_obligation_reason_code": evaluation.reason_code or None,
|
||||
"actuation_obligation_key": evaluation.obligation_key or None,
|
||||
"cannot_act_reason": evaluation.cannot_act_reason.value if evaluation.cannot_act_reason is not None else None,
|
||||
"successful_mutating_browser_actions": _successful_mutating_browser_action_count(ctx),
|
||||
"durable_fill_required": request_policy_requires_durable_fill(ctx.request_policy),
|
||||
"successful_durable_browser_fills": _successful_durable_fill_count(ctx),
|
||||
}
|
||||
|
||||
|
||||
def _mcp_tool_surface_for_turn(
|
||||
alias_map: dict[str, str],
|
||||
overlays: dict[str, Any],
|
||||
|
|
@ -3303,6 +3433,15 @@ async def _translate_to_agent_result(
|
|||
unvalidated=unvalidated,
|
||||
output_kind=output_kind,
|
||||
)
|
||||
actuation_obligation = _evaluate_actuation_obligation_for_output(ctx, action_data, resp_type, output_kind)
|
||||
if actuation_obligation.status == ActuationObligationStatus.STEER:
|
||||
raw_output_policy_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
|
||||
actuation_obligation_terminal = actuation_obligation.status == ActuationObligationStatus.TERMINAL
|
||||
if actuation_obligation_terminal:
|
||||
resp_type = "REPLY"
|
||||
user_response = _ACTUATION_OBLIGATION_UNMET_REPLY
|
||||
last_workflow = None
|
||||
last_workflow_yaml = None
|
||||
output_policy_verdict = _copy_output_policy_verdict(raw_output_policy_verdict)
|
||||
soft_rewrite_reasons: list[OutputPolicyReason] = []
|
||||
unbacked_workflow_delivery_rewritten = False
|
||||
|
|
@ -3356,6 +3495,7 @@ async def _translate_to_agent_result(
|
|||
hard_block_reason_codes=list(output_policy_verdict.reason_codes),
|
||||
soft_rewrite_reason_codes=soft_rewrite_reasons,
|
||||
)
|
||||
output_policy_diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation))
|
||||
trace_data = output_policy_verdict_to_trace_data(
|
||||
output_policy_verdict,
|
||||
surface="final_translation",
|
||||
|
|
@ -3386,15 +3526,22 @@ async def _translate_to_agent_result(
|
|||
else []
|
||||
)
|
||||
reason_code = ",".join(reason_codes)
|
||||
terminal_reason = None
|
||||
if actuation_obligation_terminal:
|
||||
reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
|
||||
final_user_response, turn_outcome = apply_repeated_reply_guard(
|
||||
final_text=final_user_response,
|
||||
attempted_kind=attempted_kind,
|
||||
blocked_signatures=ctx.blocked_reply_signatures,
|
||||
reason_code=reason_code,
|
||||
terminal_reason=terminal_reason,
|
||||
turn_intent=ctx.turn_intent,
|
||||
tool_calls=[name for name in tool_call_names if name],
|
||||
)
|
||||
if actuation_obligation.reason_code:
|
||||
turn_outcome = turn_outcome.model_copy(update={"actuation_obligation_key": actuation_obligation.obligation_key})
|
||||
|
||||
return _finalize_result_with_blocker_override(
|
||||
ctx,
|
||||
|
|
@ -3595,6 +3742,7 @@ def _blocked_final_output_kind(verdict: OutputPolicyVerdict) -> CopilotOutputKin
|
|||
OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,
|
||||
OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
|
||||
OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
|
||||
OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
|
||||
}
|
||||
if any(reason in clarification_reasons for reason in verdict.reason_codes):
|
||||
return CopilotOutputKind.CLARIFICATION_REQUEST
|
||||
|
|
@ -3661,6 +3809,16 @@ def _evaluate_copilot_final_output_policy(
|
|||
workflow_attempted=workflow_attempted,
|
||||
output_kind=output_kind,
|
||||
)
|
||||
actuation_obligation = _evaluate_actuation_obligation_for_output(
|
||||
ctx,
|
||||
action_data,
|
||||
policy_response_type,
|
||||
output_kind,
|
||||
)
|
||||
if actuation_obligation.status == ActuationObligationStatus.STEER:
|
||||
raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
|
||||
elif actuation_obligation.status == ActuationObligationStatus.TERMINAL:
|
||||
raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_UNMET)
|
||||
hard_verdict = hard_block_output_policy_verdict(raw_verdict)
|
||||
deferred_reason_codes = _defer_avoidable_ask_to_recycle(ctx, hard_verdict, response_type)
|
||||
if deferred_reason_codes is not None:
|
||||
|
|
@ -3674,6 +3832,7 @@ def _evaluate_copilot_final_output_policy(
|
|||
hard_block_reason_codes=list(hard_verdict.reason_codes),
|
||||
soft_rewrite_reason_codes=[],
|
||||
)
|
||||
diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation))
|
||||
if deferred_reason_codes is not None:
|
||||
diagnostics["deferred_to_recycle"] = True
|
||||
diagnostics["deferred_reason_codes"] = [reason.value for reason in deferred_reason_codes]
|
||||
|
|
@ -3878,6 +4037,10 @@ def _build_output_policy_blocked_result(
|
|||
"workflow without a tracked URL, re-select it so Copilot can confirm its URL scope."
|
||||
)
|
||||
add_saved_draft_copy = True
|
||||
elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes:
|
||||
user_response = _ACTUATION_OBLIGATION_UNMET_REPLY
|
||||
elif OutputPolicyReason.ACTUATION_OBLIGATION_STEER in verdict.reason_codes:
|
||||
user_response = _ACTUATION_OBLIGATION_STEER_REPLY
|
||||
elif preserved_workflow is not None:
|
||||
user_response = (
|
||||
"I could not safely return that chat reply, but the workflow draft is still saved. "
|
||||
|
|
@ -3908,13 +4071,37 @@ def _build_output_policy_blocked_result(
|
|||
composed_from_recorded_evidence = True
|
||||
if preserved_workflow is not None and add_saved_draft_copy:
|
||||
user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
|
||||
has_non_actuation_hard_block = any(
|
||||
reason
|
||||
not in {
|
||||
OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
|
||||
OutputPolicyReason.ACTUATION_OBLIGATION_UNMET,
|
||||
}
|
||||
for reason in verdict.reason_codes
|
||||
)
|
||||
if has_non_actuation_hard_block:
|
||||
blocked_reason_code = "output_policy_block"
|
||||
blocked_terminal_reason: str | None = "output_policy_block"
|
||||
elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes:
|
||||
blocked_reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
blocked_terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
else:
|
||||
blocked_reason_code = ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
blocked_terminal_reason = None
|
||||
final_user_response, output_policy_outcome = apply_repeated_reply_guard(
|
||||
final_text=user_response,
|
||||
attempted_kind=ResponseKind.CLARIFY,
|
||||
blocked_signatures=ctx.blocked_reply_signatures,
|
||||
reason_code="output_policy_block",
|
||||
terminal_reason="output_policy_block",
|
||||
reason_code=blocked_reason_code,
|
||||
terminal_reason=blocked_terminal_reason,
|
||||
)
|
||||
if blocked_reason_code in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}:
|
||||
key = ""
|
||||
if output_policy_diagnostics is not None:
|
||||
key = str(output_policy_diagnostics.get("actuation_obligation_key") or "")
|
||||
output_policy_outcome = output_policy_outcome.model_copy(
|
||||
update={"actuation_obligation_key": key or actuation_obligation_key(ctx.turn_intent)}
|
||||
)
|
||||
if composed_from_recorded_evidence and fallback_user_response is not None:
|
||||
composed_verdict = evaluate_output_policy(
|
||||
request_policy=ctx.request_policy,
|
||||
|
|
@ -3997,6 +4184,7 @@ async def run_copilot_agent(
|
|||
prior_copilot_workflow_yaml: str | None = None,
|
||||
prior_block_count: int | None = None,
|
||||
stored_completion_criteria: StoredCriteriaSnapshot | None = None,
|
||||
prior_turn_outcome: TurnOutcome | None = None,
|
||||
) -> AgentResult:
|
||||
# One id per turn — passed to every downstream AgentResult and
|
||||
# CopilotContext so the envelope and terminal frames correlate. The
|
||||
|
|
@ -4034,6 +4222,7 @@ async def run_copilot_agent(
|
|||
prior_block_count=prior_block_count,
|
||||
ctx_sink=ctx_sink,
|
||||
stored_completion_criteria=stored_completion_criteria,
|
||||
prior_turn_outcome=prior_turn_outcome,
|
||||
)
|
||||
except Exception as exc:
|
||||
LOG.error(
|
||||
|
|
@ -4102,6 +4291,7 @@ async def _run_copilot_turn_impl(
|
|||
prior_block_count: int | None = None,
|
||||
ctx_sink: list[CopilotContext] | None = None,
|
||||
stored_completion_criteria: StoredCriteriaSnapshot | None = None,
|
||||
prior_turn_outcome: TurnOutcome | None = None,
|
||||
) -> AgentResult:
|
||||
copilot_config = config or CopilotConfig(security_rules=security_rules)
|
||||
chat_history_text = _format_chat_history(chat_history)
|
||||
|
|
@ -4166,6 +4356,7 @@ async def _run_copilot_turn_impl(
|
|||
turn_index=turn_index,
|
||||
prior_block_count=prior_block_count,
|
||||
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
|
||||
prior_turn_outcome=prior_turn_outcome,
|
||||
block_authoring_policy=copilot_config.block_authoring_policy,
|
||||
impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
|
||||
copilot_config=copilot_config,
|
||||
|
|
|
|||
|
|
@ -1101,6 +1101,21 @@ def trajectory_has_credential_fill(trajectory: Sequence[Mapping[str, Any]]) -> b
|
|||
return False
|
||||
|
||||
|
||||
def trajectory_has_browser_fill_interaction(trajectory: Sequence[Mapping[str, Any]]) -> bool:
|
||||
for interaction in trajectory:
|
||||
tool_name = str(interaction.get("tool_name") or "")
|
||||
typed_length = interaction.get("typed_length")
|
||||
if tool_name == "type_text" and (
|
||||
(isinstance(typed_length, int) and typed_length > 0) or str(interaction.get("typed_value") or "").strip()
|
||||
):
|
||||
return True
|
||||
if tool_name == "select_option" and str(interaction.get("value") or "").strip():
|
||||
return True
|
||||
if tool_name == CREDENTIAL_FILL_TOOL_NAME and str(interaction.get("credential_field") or "").strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
return build_artifact_metadata_skeleton(trajectory, block_label=_SYNTHESIZED_BLOCK_LABEL)
|
||||
|
||||
|
|
|
|||
|
|
@ -643,6 +643,7 @@ class CopilotContext(AgentContext):
|
|||
target_block_label: str | None = None
|
||||
turn_intent: TurnIntent | None = None
|
||||
turn_context_packet: TurnContextPacket | None = None
|
||||
prior_turn_outcome: TurnOutcome | None = None
|
||||
latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
|
||||
blocked_reply_signatures: list[str] = field(default_factory=list)
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
|
|||
is_optional_dismissal_only_trajectory,
|
||||
render_synthesized_offer_text,
|
||||
synthesize_code_block,
|
||||
trajectory_has_browser_fill_interaction,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_paths
|
||||
from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
|
||||
|
|
@ -76,7 +77,10 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
|
|||
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
|
||||
from skyvern.forge.sdk.copilot.narration import TransitionKind
|
||||
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
|
||||
from skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding
|
||||
from skyvern.forge.sdk.copilot.output_policy import (
|
||||
completion_criterion_requires_browser_fill_delivery,
|
||||
normalize_response_scaffolding,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.output_utils import (
|
||||
extract_final_text,
|
||||
looks_like_workflow_delivery_claim,
|
||||
|
|
@ -116,7 +120,7 @@ from skyvern.forge.sdk.copilot.turn_halt import (
|
|||
stash_repair_ceiling_turn_halt,
|
||||
stash_turn_halt_from_blocker_signal,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
|
||||
from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode
|
||||
from skyvern.utils.token_counter import count_tokens
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -188,6 +192,7 @@ SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"
|
|||
_SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS = frozenset(
|
||||
{SYNTHESIZED_BLOCK_PERSISTENCE_TOOL, "fill_credential_field", "update_workflow"}
|
||||
)
|
||||
_ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL = "type_text"
|
||||
_SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS = frozenset(
|
||||
{"click", "press_key", "type_text", "select_option", "navigate_browser"}
|
||||
)
|
||||
|
|
@ -2394,6 +2399,39 @@ def _uncovered_output_reject_admits_evaluate(ctx: CopilotContext, tool_name: str
|
|||
return tool_name == "evaluate" and bool(_active_uncovered_output_reject_paths(ctx))
|
||||
|
||||
|
||||
def _actuation_obligation_live_fill_delivery_required(ctx: CopilotContext) -> bool:
|
||||
turn_intent = getattr(ctx, "turn_intent", None)
|
||||
if (
|
||||
not isinstance(turn_intent, TurnIntent)
|
||||
or turn_intent.mode != TurnIntentMode.BUILD
|
||||
or RequiredContextKey.BROWSER_STATE not in turn_intent.required_context
|
||||
):
|
||||
return False
|
||||
if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
|
||||
return False
|
||||
request_policy = getattr(ctx, "request_policy", None)
|
||||
criteria: list[CompletionCriterion] = []
|
||||
if isinstance(request_policy, RequestPolicy):
|
||||
criteria.extend(request_policy.completion_criteria)
|
||||
turn_state = getattr(ctx, "completion_criteria_turn_state", None)
|
||||
if turn_state is not None and turn_state.decision is not None:
|
||||
criteria.extend(turn_state.decision.criteria)
|
||||
if any(completion_criterion_requires_browser_fill_delivery(criterion) for criterion in criteria):
|
||||
return True
|
||||
trajectory = getattr(ctx, "scout_trajectory", None)
|
||||
return trajectory_has_browser_fill_interaction(trajectory) if isinstance(trajectory, list) else False
|
||||
|
||||
|
||||
def _actuation_obligation_required_fill_tool(ctx: CopilotContext) -> str | None:
|
||||
if _actuation_obligation_live_fill_delivery_required(ctx):
|
||||
return _ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL
|
||||
return None
|
||||
|
||||
|
||||
def _actuation_obligation_admits_required_fill_tool(ctx: CopilotContext, tool_name: str) -> bool:
|
||||
return tool_name == _actuation_obligation_required_fill_tool(ctx)
|
||||
|
||||
|
||||
def consume_uncovered_output_reopen_event(ctx: CopilotContext) -> bool:
|
||||
"""Arm a one-shot scout-window reopen for the first author-time reject citing an uncovered
|
||||
requested-output path. Returns True only on that first reject per structural identity; a
|
||||
|
|
@ -2501,6 +2539,8 @@ def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToo
|
|||
return None
|
||||
if _uncovered_output_reject_admits_evaluate(ctx, tool_name):
|
||||
return None
|
||||
if _actuation_obligation_admits_required_fill_tool(ctx, tool_name):
|
||||
return None
|
||||
if (
|
||||
ambiguous_selector_rescout_state != "block"
|
||||
and not _should_force_synthesized_block_persistence(ctx)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
|
|||
looks_like_workflow_yaml_in_chat,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.request_policy import (
|
||||
CompletionCriterion,
|
||||
RequestPolicy,
|
||||
contains_email_password_pair,
|
||||
request_policy_has_present_completion_contract,
|
||||
|
|
@ -21,6 +22,7 @@ from skyvern.forge.sdk.copilot.secret_redaction import (
|
|||
RAW_SECRET_PATTERNS,
|
||||
SECRET_KEYWORD_ASSIGNMENT_PATTERN,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode
|
||||
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
|
||||
block_credential_ids,
|
||||
credential_param_ids,
|
||||
|
|
@ -30,8 +32,12 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
|
|||
workflow_credential_ids_from_parsed,
|
||||
workflow_credential_origins_from_parsed,
|
||||
)
|
||||
from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
|
||||
|
||||
WORKFLOW_PRESENT_SENTINEL = object()
|
||||
ACTUATION_OBLIGATION_STEER_REASON_CODE = "actuation_obligation_steer"
|
||||
ACTUATION_OBLIGATION_UNMET_REASON_CODE = "actuation_obligation_unmet"
|
||||
ACTUATION_OBLIGATION_BROWSER_ACTION_KEY = "browser_state:build:no_update:no_run"
|
||||
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
|
||||
_PLACEHOLDER_MARKERS = ("{{", "{%", "[REDACTED_SECRET]")
|
||||
# RHS of a secret-keyword assignment that references a bound value instead of carrying one:
|
||||
|
|
@ -183,6 +189,26 @@ class CopilotOutputKind(StrEnum):
|
|||
WORKFLOW_RUN_RESULT = "workflow_run_result"
|
||||
|
||||
|
||||
class CannotActReason(StrEnum):
|
||||
MISSING_FIELD_VALUE = "missing_field_value"
|
||||
AMBIGUOUS_TARGET = "ambiguous_target"
|
||||
STRUCTURAL_BLOCKER = "structural_blocker"
|
||||
|
||||
|
||||
class ActuationObligationStatus(StrEnum):
|
||||
ALLOWED = "allowed"
|
||||
STEER = "steer"
|
||||
TERMINAL = "terminal"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActuationObligationEvaluation:
|
||||
status: ActuationObligationStatus = ActuationObligationStatus.ALLOWED
|
||||
reason_code: str = ""
|
||||
cannot_act_reason: CannotActReason | None = None
|
||||
obligation_key: str = ""
|
||||
|
||||
|
||||
class OutputPolicyReason(StrEnum):
|
||||
RAW_SECRET_LEAK = "raw_secret_leak"
|
||||
REQUEST_POLICY_CLARIFICATION_BYPASS = "request_policy_clarification_bypass"
|
||||
|
|
@ -199,6 +225,8 @@ class OutputPolicyReason(StrEnum):
|
|||
SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak"
|
||||
WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply"
|
||||
AVOIDABLE_OUTPUT_FIELD_CONFIRMATION = "avoidable_output_field_confirmation"
|
||||
ACTUATION_OBLIGATION_STEER = ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
ACTUATION_OBLIGATION_UNMET = ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -232,10 +260,97 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset(
|
|||
OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK,
|
||||
OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING,
|
||||
OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION,
|
||||
OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
|
||||
OutputPolicyReason.ACTUATION_OBLIGATION_UNMET,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def coerce_cannot_act_reason(value: str | None) -> CannotActReason | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return CannotActReason(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def turn_intent_requires_actuation(turn_intent: TurnIntent | None) -> bool:
|
||||
return (
|
||||
isinstance(turn_intent, TurnIntent)
|
||||
and turn_intent.mode in {TurnIntentMode.BUILD, TurnIntentMode.UNKNOWN}
|
||||
and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
|
||||
and not turn_intent.authority.may_update_workflow
|
||||
and not turn_intent.authority.may_run_blocks
|
||||
)
|
||||
|
||||
|
||||
def actuation_obligation_key(turn_intent: TurnIntent | None) -> str:
|
||||
if not turn_intent_requires_actuation(turn_intent):
|
||||
return ""
|
||||
if turn_intent is not None and turn_intent.mode == TurnIntentMode.UNKNOWN:
|
||||
return "browser_state:unknown:no_update:no_run"
|
||||
return ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
|
||||
|
||||
|
||||
def completion_criterion_requires_browser_fill_delivery(criterion: CompletionCriterion) -> bool:
|
||||
if criterion.kind == "terminal_action" and criterion.terminal_action_family == "form":
|
||||
return True
|
||||
return criterion.kind == "outcome" and criterion.level == "run" and criterion.method_mandated
|
||||
|
||||
|
||||
def request_policy_requires_durable_fill(request_policy: RequestPolicy | None) -> bool:
|
||||
return isinstance(request_policy, RequestPolicy) and any(
|
||||
completion_criterion_requires_browser_fill_delivery(criterion)
|
||||
for criterion in request_policy.completion_criteria
|
||||
)
|
||||
|
||||
|
||||
def prior_turn_satisfies_actuation_terminal_condition(
|
||||
prior_turn_outcome: TurnOutcome | None,
|
||||
obligation_key: str,
|
||||
) -> bool:
|
||||
# One key covers the turn's actuation obligation; a later partial action
|
||||
# still failed the same browser-fill contract.
|
||||
return (
|
||||
prior_turn_outcome is not None
|
||||
and prior_turn_outcome.reason_code
|
||||
in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}
|
||||
and prior_turn_outcome.actuation_obligation_key == obligation_key
|
||||
)
|
||||
|
||||
|
||||
def evaluate_actuation_obligation(
|
||||
*,
|
||||
turn_intent: TurnIntent | None,
|
||||
response_type: str,
|
||||
output_kind: CopilotOutputKind,
|
||||
successful_mutating_browser_actions: int,
|
||||
cannot_act_reason: CannotActReason | None,
|
||||
prior_turn_outcome: TurnOutcome | None,
|
||||
) -> ActuationObligationEvaluation:
|
||||
obligation_key = actuation_obligation_key(turn_intent)
|
||||
if (
|
||||
not turn_intent_requires_actuation(turn_intent)
|
||||
or response_type != "REPLY"
|
||||
or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER
|
||||
):
|
||||
return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason)
|
||||
if successful_mutating_browser_actions > 0 or cannot_act_reason is not None:
|
||||
return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason, obligation_key=obligation_key)
|
||||
if prior_turn_satisfies_actuation_terminal_condition(prior_turn_outcome, obligation_key):
|
||||
return ActuationObligationEvaluation(
|
||||
status=ActuationObligationStatus.TERMINAL,
|
||||
reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
obligation_key=obligation_key,
|
||||
)
|
||||
return ActuationObligationEvaluation(
|
||||
status=ActuationObligationStatus.STEER,
|
||||
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
obligation_key=obligation_key,
|
||||
)
|
||||
|
||||
|
||||
def hard_block_output_policy_verdict(verdict: OutputPolicyVerdict) -> OutputPolicyVerdict:
|
||||
hard_reasons = [reason for reason in verdict.reason_codes if reason in _FINAL_OUTPUT_HARD_BLOCK_REASONS]
|
||||
return OutputPolicyVerdict(
|
||||
|
|
|
|||
|
|
@ -2069,6 +2069,7 @@ async def _new_copilot_chat_post(
|
|||
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
|
||||
prior_block_count=prior_block_count,
|
||||
stored_completion_criteria=stored_completion_criteria,
|
||||
prior_turn_outcome=prior_turn_outcome,
|
||||
)
|
||||
|
||||
agent_result.turn_outcome = _with_current_copilot_code_mode_metadata(
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ class TurnOutcome(BaseModel):
|
|||
turn_intent_summary: dict[str, Any] = Field(default_factory=dict)
|
||||
response_kind: ResponseKind
|
||||
reason_code: str = ""
|
||||
actuation_obligation_key: str = ""
|
||||
normalized_reply_signature: str = ""
|
||||
tool_calls: list[str] = Field(default_factory=list)
|
||||
terminal_reason: str | None = None
|
||||
|
|
|
|||
|
|
@ -27,7 +27,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
|
|||
)
|
||||
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
|
||||
from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
|
||||
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict
|
||||
from skyvern.forge.sdk.copilot.output_policy import (
|
||||
ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
CopilotOutputKind,
|
||||
OutputPolicyReason,
|
||||
OutputPolicyVerdict,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
|
||||
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
|
||||
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKind
|
||||
|
|
@ -338,6 +343,18 @@ def test_output_policy_specific_branches_bypass_recorded_terminal_fallback() ->
|
|||
assert result.narrative_payload["responseType"] == "ASK_QUESTION"
|
||||
|
||||
|
||||
def test_output_policy_hard_block_priority_over_actuation_steer() -> None:
|
||||
ctx = _ctx()
|
||||
|
||||
result = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK, OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
|
||||
|
||||
assert result.user_response == _RAW_SECRET_LEAK_REFUSAL
|
||||
assert result.turn_outcome is not None
|
||||
assert result.turn_outcome.reason_code == "output_policy_block"
|
||||
assert result.turn_outcome.terminal_reason == "output_policy_block"
|
||||
assert result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
def test_shim_preserves_workflow_draft_when_signal_opts_in() -> None:
|
||||
"""Late-block-running blockers say 'wrap up with what you have' — they
|
||||
surface the saved draft alongside the rendered chat reply."""
|
||||
|
|
|
|||
|
|
@ -60,6 +60,11 @@ from skyvern.forge.sdk.copilot.enforcement import (
|
|||
verified_goal_satisfied_context,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
|
||||
from skyvern.forge.sdk.copilot.output_policy import (
|
||||
ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
|
||||
ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
|
||||
from skyvern.forge.sdk.copilot.request_policy import (
|
||||
_MAX_COMPLETION_CRITERIA,
|
||||
|
|
@ -90,12 +95,14 @@ from skyvern.forge.sdk.copilot.turn_halt import (
|
|||
stash_repair_ceiling_turn_halt,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.turn_intent import (
|
||||
RequiredContextKey,
|
||||
TurnIntent,
|
||||
TurnIntentAuthority,
|
||||
TurnIntentMode,
|
||||
TurnIntentReasonCode,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
|
||||
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
|
||||
from skyvern.forge.sdk.schemas.workflow_copilot import (
|
||||
WorkflowCopilotChatHistoryMessage,
|
||||
WorkflowCopilotChatSender,
|
||||
|
|
@ -1640,7 +1647,6 @@ class TestRequestPolicyInputGuardrail:
|
|||
request_policy_handler=request_policy_handler,
|
||||
turn_intent_handler=turn_intent_handler,
|
||||
)
|
||||
|
||||
guardrails = agent_module._build_copilot_input_guardrails(
|
||||
InputGuardrail,
|
||||
GuardrailFunctionOutput,
|
||||
|
|
@ -1886,6 +1892,30 @@ def _chat_request() -> SimpleNamespace:
|
|||
)
|
||||
|
||||
|
||||
def _browser_actuation_intent() -> TurnIntent:
|
||||
return TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
required_context=[RequiredContextKey.BROWSER_STATE],
|
||||
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
|
||||
)
|
||||
|
||||
|
||||
def _unknown_browser_actuation_intent() -> TurnIntent:
|
||||
return TurnIntent(
|
||||
mode=TurnIntentMode.UNKNOWN,
|
||||
required_context=[RequiredContextKey.BROWSER_STATE],
|
||||
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
|
||||
)
|
||||
|
||||
|
||||
def _browser_actuation_intent_without_browser_state(*, mode: TurnIntentMode) -> TurnIntent:
|
||||
return TurnIntent(
|
||||
mode=mode,
|
||||
required_context=[],
|
||||
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
|
||||
)
|
||||
|
||||
|
||||
class TestBlockGoalMainGoal:
|
||||
def test_empty_message_returns_empty(self) -> None:
|
||||
assert agent_module._build_block_goal_main_goal("", chat_history_text="", global_llm_context=None) == ""
|
||||
|
|
@ -2268,6 +2298,69 @@ class TestTranslateToAgentResultGating:
|
|||
assert agent_result.response_type == "ASK_QUESTION"
|
||||
assert agent_result.user_response == "Which account should I use?"
|
||||
|
||||
def test_actuation_required_reply_without_browser_mutation_returns_steer(self) -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent())
|
||||
result = _fake_run_result({"type": "REPLY", "user_response": "I can fill those fields for you."})
|
||||
|
||||
agent_result = asyncio.run(
|
||||
agent_module._translate_to_agent_result(
|
||||
result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
|
||||
)
|
||||
)
|
||||
|
||||
assert agent_result.response_type == "ASK_QUESTION"
|
||||
assert agent_result.turn_outcome is not None
|
||||
assert agent_result.turn_outcome.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
assert agent_result.turn_outcome.actuation_obligation_key == ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
|
||||
|
||||
def test_repeated_actuation_required_reply_without_browser_mutation_returns_terminal(self) -> None:
|
||||
ctx = _ctx(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
prior_turn_outcome=TurnOutcome(
|
||||
response_kind=ResponseKind.CLARIFY,
|
||||
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
|
||||
),
|
||||
)
|
||||
result = _fake_run_result({"type": "REPLY", "user_response": "I can fill those fields for you."})
|
||||
|
||||
agent_result = asyncio.run(
|
||||
agent_module._translate_to_agent_result(
|
||||
result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
|
||||
)
|
||||
)
|
||||
|
||||
assert agent_result.response_type == "REPLY"
|
||||
assert agent_result.updated_workflow is None
|
||||
assert agent_result.turn_outcome is not None
|
||||
assert agent_result.turn_outcome.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
assert agent_result.turn_outcome.terminal_reason == ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
assert agent_result.turn_outcome.actuation_obligation_key == ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
|
||||
|
||||
def test_unknown_click_with_authority_denied_blocker_returns_reply(self) -> None:
|
||||
ctx = _ctx(turn_intent=_unknown_browser_actuation_intent())
|
||||
ctx.scout_trajectory.append({"tool_name": "click"})
|
||||
ctx.blocker_signal = CopilotToolBlockerSignal(
|
||||
blocker_kind="authority_denied",
|
||||
agent_steering_text="Use browser tools.",
|
||||
user_facing_reason="I'll respond with the information I already have.",
|
||||
recovery_hint="report_blocker_to_user",
|
||||
internal_reason_code="turn_intent_no_mutation_run_blocked",
|
||||
blocked_tool="update_and_run_blocks",
|
||||
classifier_mode="unknown",
|
||||
)
|
||||
result = _fake_run_result({"type": "REPLY", "user_response": "I'll respond with the information I have."})
|
||||
|
||||
agent_result = asyncio.run(
|
||||
agent_module._translate_to_agent_result(
|
||||
result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
|
||||
)
|
||||
)
|
||||
|
||||
assert agent_result.response_type == "REPLY"
|
||||
assert agent_result.turn_outcome is not None
|
||||
assert agent_result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
def test_verified_terminal_state_surfaces_workflow_despite_weak_final_reply(self) -> None:
|
||||
workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
|
||||
ctx = _ctx(
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ from skyvern.forge.sdk.copilot.output_contracts import (
|
|||
OutputContractAdvisoryState,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
|
||||
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
|
||||
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
|
||||
from skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_tool
|
||||
from skyvern.forge.sdk.copilot.tools import (
|
||||
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
|
||||
|
|
@ -75,7 +75,7 @@ from skyvern.forge.sdk.copilot.tools import (
|
|||
_record_workflow_update_result,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
|
||||
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
|
||||
from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode
|
||||
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
|
||||
from tests.unit.conftest import make_copilot_context
|
||||
|
||||
|
|
@ -128,6 +128,7 @@ class _Ctx:
|
|||
self.completion_criteria_turn_state = None
|
||||
self.reached_download_target: ReachedDownloadTarget | None = None
|
||||
self.author_time_gate_ablation_events = []
|
||||
self.request_policy = None
|
||||
|
||||
|
||||
class TestSynthesizedOfferPersistenceGate:
|
||||
|
|
@ -340,6 +341,185 @@ class TestSynthesizedOfferPersistenceGate:
|
|||
assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})
|
||||
assert signal.renders_final_reply is False
|
||||
|
||||
def test_actuation_obligation_admits_required_fill_tool_during_persistence_offer(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy(
|
||||
completion_criteria=[
|
||||
CompletionCriterion(
|
||||
id="form-submit",
|
||||
outcome="form fields are filled",
|
||||
kind="terminal_action",
|
||||
terminal_action_family="form",
|
||||
)
|
||||
],
|
||||
)
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 1
|
||||
ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
|
||||
|
||||
assert synthesized_block_persistence_signal(ctx, "type_text") is None
|
||||
click_signal = synthesized_block_persistence_signal(ctx, "click")
|
||||
assert isinstance(click_signal, CopilotToolBlockerSignal)
|
||||
assert click_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
|
||||
|
||||
def test_actuation_obligation_admits_required_fill_tool_for_method_mandated_run_contract(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy(
|
||||
completion_criteria=[
|
||||
CompletionCriterion(
|
||||
id="visible-fill",
|
||||
outcome="fields are visibly filled on the live page",
|
||||
method_mandated=True,
|
||||
level="run",
|
||||
)
|
||||
],
|
||||
)
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 1
|
||||
ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
|
||||
|
||||
assert synthesized_block_persistence_signal(ctx, "type_text") is None
|
||||
click_signal = synthesized_block_persistence_signal(ctx, "click")
|
||||
assert isinstance(click_signal, CopilotToolBlockerSignal)
|
||||
assert click_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
|
||||
|
||||
def test_actuation_obligation_blocks_type_text_for_definition_method_contract(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy(
|
||||
completion_criteria=[
|
||||
CompletionCriterion(
|
||||
id="definition-contract",
|
||||
outcome="inputs are reusable",
|
||||
method_mandated=True,
|
||||
level="definition",
|
||||
)
|
||||
],
|
||||
)
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 1
|
||||
ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
|
||||
|
||||
signal = synthesized_block_persistence_signal(ctx, "type_text")
|
||||
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
|
||||
|
||||
def test_actuation_obligation_admits_required_fill_tool_after_turn_state_reconcile(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy(
|
||||
completion_criteria=[
|
||||
CompletionCriterion(
|
||||
id="form-submit",
|
||||
outcome="form fields are filled",
|
||||
kind="terminal_action",
|
||||
terminal_action_family="form",
|
||||
)
|
||||
],
|
||||
)
|
||||
ctx.completion_criteria_turn_state = SimpleNamespace(
|
||||
decision=SimpleNamespace(
|
||||
criteria=(
|
||||
CompletionCriterion(
|
||||
id="workflow-run",
|
||||
outcome="workflow has been tested",
|
||||
kind="terminal_action",
|
||||
terminal_action_family="workflow_run",
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 2
|
||||
ctx.scout_trajectory = [
|
||||
{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"},
|
||||
{"tool_name": "type_text", "selector": "#company", "accessible_name": "Company"},
|
||||
]
|
||||
|
||||
assert synthesized_block_persistence_signal(ctx, "type_text") is None
|
||||
|
||||
def test_persistence_offer_blocks_type_text_without_actuation_obligation(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy()
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 1
|
||||
ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
|
||||
|
||||
signal = synthesized_block_persistence_signal(ctx, "type_text")
|
||||
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
|
||||
assert signal.blocked_tool == "type_text"
|
||||
|
||||
def test_actuation_obligation_admits_required_fill_tool_from_fill_trajectory(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy()
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 1
|
||||
ctx.scout_trajectory = [
|
||||
{
|
||||
"tool_name": "type_text",
|
||||
"selector": "#company",
|
||||
"typed_value": "Example Realty Labs Inc",
|
||||
"accessible_name": "Company",
|
||||
}
|
||||
]
|
||||
|
||||
assert synthesized_block_persistence_signal(ctx, "type_text") is None
|
||||
|
||||
def test_fill_less_trajectory_does_not_arm_actuation_obligation(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
|
||||
required_context={RequiredContextKey.BROWSER_STATE},
|
||||
)
|
||||
ctx.request_policy = RequestPolicy()
|
||||
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
|
||||
ctx.synthesized_block_offered = True
|
||||
ctx.synthesized_block_offered_trajectory_len = 1
|
||||
ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
|
||||
|
||||
signal = synthesized_block_persistence_signal(ctx, "type_text")
|
||||
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
|
||||
|
||||
def test_prerun_ambiguous_bare_selector_repair_allows_one_evaluate(self) -> None:
|
||||
ctx = _Ctx()
|
||||
ctx.turn_intent = TurnIntent(
|
||||
|
|
|
|||
|
|
@ -21,12 +21,19 @@ from skyvern.forge.sdk.copilot.enforcement import (
|
|||
PRESENT_COMPLETION_CONTRACT_ASK_RETRY,
|
||||
_response_coverage_nudge,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.output_contracts import OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE
|
||||
from skyvern.forge.sdk.copilot.output_policy import (
|
||||
ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
|
||||
ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
ActuationObligationStatus,
|
||||
CannotActReason,
|
||||
CopilotOutputKind,
|
||||
OutputPolicyReason,
|
||||
OutputPolicyVerdict,
|
||||
_contains_internal_tool_vocab_leak,
|
||||
derive_output_kind,
|
||||
evaluate_actuation_obligation,
|
||||
evaluate_output_policy,
|
||||
hard_block_output_policy_verdict,
|
||||
normalize_response_scaffolding,
|
||||
|
|
@ -36,7 +43,8 @@ from skyvern.forge.sdk.copilot.tools import (
|
|||
_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL,
|
||||
NATIVE_TOOLS,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
|
||||
from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode
|
||||
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
|
||||
|
||||
|
||||
def _credential(credential_id: str = "cred_safe", tested_url: str = "https://login.example.test/login") -> object:
|
||||
|
|
@ -63,6 +71,43 @@ def _ctx(**overrides: object) -> CopilotContext:
|
|||
return CopilotContext(**defaults)
|
||||
|
||||
|
||||
def _browser_actuation_intent() -> TurnIntent:
|
||||
return TurnIntent(
|
||||
mode=TurnIntentMode.BUILD,
|
||||
required_context=[RequiredContextKey.BROWSER_STATE],
|
||||
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
|
||||
)
|
||||
|
||||
|
||||
def _live_fill_policy() -> RequestPolicy:
|
||||
return _policy(
|
||||
completion_criteria=[
|
||||
CompletionCriterion(
|
||||
id="c0",
|
||||
outcome="The form is completed.",
|
||||
kind="terminal_action",
|
||||
terminal_action_family="form",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _unknown_browser_actuation_intent() -> TurnIntent:
|
||||
return TurnIntent(
|
||||
mode=TurnIntentMode.UNKNOWN,
|
||||
required_context=[RequiredContextKey.BROWSER_STATE],
|
||||
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
|
||||
)
|
||||
|
||||
|
||||
def _browser_actuation_intent_without_browser_state(*, mode: TurnIntentMode) -> TurnIntent:
|
||||
return TurnIntent(
|
||||
mode=mode,
|
||||
required_context=[],
|
||||
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
|
||||
)
|
||||
|
||||
|
||||
def _fake_run_result(payload: dict) -> SimpleNamespace:
|
||||
return SimpleNamespace(final_output=json.dumps(payload), new_items=[])
|
||||
|
||||
|
|
@ -1384,6 +1429,384 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:
|
|||
}
|
||||
|
||||
|
||||
def test_actuation_obligation_steers_zero_action_informational_reply() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=None,
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.STEER
|
||||
assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
def test_actuation_obligation_steers_unknown_browser_only_intent() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_unknown_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=None,
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.STEER
|
||||
assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
def test_actuation_obligation_allows_typed_cannot_act_reason() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=CannotActReason.MISSING_FIELD_VALUE,
|
||||
prior_turn_outcome=None,
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.ALLOWED
|
||||
assert evaluation.cannot_act_reason == CannotActReason.MISSING_FIELD_VALUE
|
||||
|
||||
|
||||
def test_actuation_obligation_repeats_as_terminal() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=TurnOutcome(
|
||||
response_kind=ResponseKind.CLARIFY,
|
||||
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
|
||||
),
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.TERMINAL
|
||||
assert evaluation.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
|
||||
|
||||
def test_actuation_obligation_prior_unmet_stays_terminal() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=TurnOutcome(
|
||||
response_kind=ResponseKind.CLARIFY,
|
||||
reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
terminal_reason=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
|
||||
actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
|
||||
),
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.TERMINAL
|
||||
assert evaluation.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
|
||||
|
||||
def test_actuation_obligation_unrelated_prior_steer_stays_recoverable() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=TurnOutcome(
|
||||
response_kind=ResponseKind.CLARIFY,
|
||||
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
actuation_obligation_key="other",
|
||||
),
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.STEER
|
||||
assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [TurnIntentMode.DOCS_ANSWER, TurnIntentMode.DIAGNOSE])
|
||||
def test_actuation_obligation_allows_non_actuation_intents(mode: TurnIntentMode) -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=TurnIntent(mode=mode, authority=TurnIntentAuthority(may_update_workflow=False)),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=None,
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.ALLOWED
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_steers_actuation_required_zero_action_reply() -> None:
|
||||
evaluation = evaluate_actuation_obligation(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
response_type="REPLY",
|
||||
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
|
||||
successful_mutating_browser_actions=0,
|
||||
cannot_act_reason=None,
|
||||
prior_turn_outcome=None,
|
||||
)
|
||||
|
||||
assert evaluation.status == ActuationObligationStatus.STEER
|
||||
assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_steers_actuation_required_reply() -> None:
|
||||
ctx = _ctx(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
tool_activity=[{"tool": "evaluate"}, {"tool": "get_browser_screenshot"}],
|
||||
)
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "I can fill those fields for you."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert not verdict.allowed
|
||||
assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER]
|
||||
assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
assert diagnostics["successful_mutating_browser_actions"] == 0
|
||||
assert diagnostics.get("deferred_to_recycle", False) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_output_guardrail_trips_actuation_steer() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent())
|
||||
output_guardrails = agent_module._build_copilot_output_guardrails(OutputGuardrail, GuardrailFunctionOutput)
|
||||
result = await output_guardrails[0].run(
|
||||
RunContextWrapper(context=ctx),
|
||||
SimpleNamespace(),
|
||||
{"type": "REPLY", "user_response": "I can fill those fields for you."},
|
||||
)
|
||||
|
||||
assert result.output.tripwire_triggered is True
|
||||
assert result.output.output_info["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_repeated_actuation_steer_terminalizes() -> None:
|
||||
ctx = _ctx(
|
||||
turn_intent=_browser_actuation_intent(),
|
||||
prior_turn_outcome=TurnOutcome(
|
||||
response_kind=ResponseKind.CLARIFY,
|
||||
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
|
||||
actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
|
||||
),
|
||||
)
|
||||
output = {"type": "REPLY", "user_response": "I can fill those fields for you."}
|
||||
|
||||
verdict, _, diagnostics = agent_module._evaluate_copilot_final_output_policy(ctx, output)
|
||||
|
||||
assert not verdict.allowed
|
||||
assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_UNMET]
|
||||
assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.TERMINAL.value
|
||||
assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_UNMET_REASON_CODE
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_rejects_uncorroborated_model_typed_cannot_act_reason() -> None:
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
_ctx(turn_intent=_browser_actuation_intent()),
|
||||
{
|
||||
"type": "REPLY",
|
||||
"user_response": "I need the field value before I can continue.",
|
||||
"cannot_act_reason": "missing_field_value",
|
||||
},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert not verdict.allowed
|
||||
assert diagnostics["cannot_act_reason"] is None
|
||||
assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_allows_structural_blocker_reason() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent())
|
||||
ctx.blocker_signal = CopilotToolBlockerSignal(
|
||||
blocker_kind="missing_required_context",
|
||||
agent_steering_text="Ask which field value to use.",
|
||||
user_facing_reason="I need the field value before I can continue.",
|
||||
recovery_hint="ask_user_clarifying",
|
||||
)
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "I need the field value before I can continue."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics["cannot_act_reason"] == "missing_field_value"
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_allows_explicit_structural_blocker_reason_code() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent())
|
||||
ctx.blocker_signal = CopilotToolBlockerSignal(
|
||||
blocker_kind="tool_error",
|
||||
agent_steering_text="Report the unavailable output source.",
|
||||
user_facing_reason="The values are not observable on this page.",
|
||||
recovery_hint="report_blocker_to_user",
|
||||
internal_reason_code=OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
|
||||
)
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "The values are not observable on this page."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics["cannot_act_reason"] == "structural_blocker"
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_rejects_generic_report_blocker_reason() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent())
|
||||
ctx.blocker_signal = CopilotToolBlockerSignal(
|
||||
blocker_kind="tool_error",
|
||||
agent_steering_text="Report the blocker.",
|
||||
user_facing_reason="The page is not ready.",
|
||||
recovery_hint="report_blocker_to_user",
|
||||
internal_reason_code="tool_error_late_block_running",
|
||||
)
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "The page is not ready."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert not verdict.allowed
|
||||
assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER]
|
||||
assert diagnostics["cannot_act_reason"] is None
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_allows_successful_mutating_browser_action() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent())
|
||||
ctx.scout_trajectory.append({"tool_name": "click"})
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "Clicked the field."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics["successful_mutating_browser_actions"] == 1
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_rejects_form_fill_click_only_reply() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy())
|
||||
ctx.scout_trajectory.append({"tool_name": "click"})
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "The requested form interaction was completed."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert not verdict.allowed
|
||||
assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER]
|
||||
assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
assert diagnostics["successful_mutating_browser_actions"] == 1
|
||||
assert diagnostics["successful_durable_browser_fills"] == 0
|
||||
assert diagnostics["durable_fill_required"] is True
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_requires_browser_state_for_form_fill_obligation() -> None:
|
||||
ctx = _ctx(
|
||||
turn_intent=_browser_actuation_intent_without_browser_state(mode=TurnIntentMode.BUILD),
|
||||
request_policy=_live_fill_policy(),
|
||||
)
|
||||
ctx.scout_trajectory.append({"tool_name": "click"})
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "The requested form interaction was completed."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics.get("deferred_to_recycle", False) is False
|
||||
assert "durable_fill_required" not in diagnostics
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_allows_non_form_click_reply() -> None:
|
||||
ctx = _ctx(
|
||||
turn_intent=_browser_actuation_intent_without_browser_state(mode=TurnIntentMode.UNKNOWN),
|
||||
)
|
||||
ctx.scout_trajectory.append({"tool_name": "click"})
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "Clicked the requested target."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics.get("deferred_to_recycle", False) is False
|
||||
assert "durable_fill_required" not in diagnostics
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_allows_live_fill_after_durable_fill() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy())
|
||||
ctx.scout_trajectory.append({"tool_name": "type_text", "typed_length": 3})
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "The requested form interaction was completed."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics.get("deferred_to_recycle", False) is False
|
||||
assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.ALLOWED.value
|
||||
assert diagnostics["successful_durable_browser_fills"] == 1
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_rejects_live_fill_after_noop_type_text() -> None:
|
||||
ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy())
|
||||
ctx.scout_trajectory.append({"tool_name": "type_text", "typed_length": 0, "typed_value": ""})
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "The requested form interaction was completed."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert not verdict.allowed
|
||||
assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
|
||||
assert diagnostics["successful_durable_browser_fills"] == 0
|
||||
assert diagnostics["durable_fill_required"] is True
|
||||
|
||||
|
||||
def test_sdk_output_guardrail_allows_unknown_click_with_authority_denied_blocker() -> None:
|
||||
ctx = _ctx(turn_intent=_unknown_browser_actuation_intent())
|
||||
ctx.scout_trajectory.append({"tool_name": "click"})
|
||||
ctx.blocker_signal = CopilotToolBlockerSignal(
|
||||
blocker_kind="authority_denied",
|
||||
agent_steering_text="Use browser tools.",
|
||||
user_facing_reason="I'll respond with the information I already have.",
|
||||
recovery_hint="report_blocker_to_user",
|
||||
internal_reason_code="turn_intent_no_mutation_run_blocked",
|
||||
blocked_tool="update_and_run_blocks",
|
||||
classifier_mode="unknown",
|
||||
)
|
||||
|
||||
verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
|
||||
ctx,
|
||||
{"type": "REPLY", "user_response": "I'll respond with the information I already have."},
|
||||
)
|
||||
|
||||
assert response_type == "REPLY"
|
||||
assert verdict.allowed
|
||||
assert diagnostics.get("deferred_to_recycle", False) is False
|
||||
assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.ALLOWED.value
|
||||
assert diagnostics["cannot_act_reason"] is None
|
||||
assert diagnostics["successful_mutating_browser_actions"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "expected_terms"),
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue