vikarti.anatra/Skyvern
1
0
Fork 0
mirror of https://github.com/Skyvern-AI/skyvern.git synced 2026-07-09 16:09:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

fix(SKY-11996): restore copilot actuation-obligation feature reverted by #13210 (#7230)

Browse source
This commit is contained in:
Shuchang Zheng 2026-07-08 23:00:59 -07:00 • committed by GitHub
parent d79359d55c
commit bf3624d788
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1097 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

217
skyvern/forge/sdk/copilot/agent.py
View file

@ -10,7 +10,7 @@ import contextlib
import json
import re
import uuid
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
trajectory_has_browser_fill_interaction,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSnapshot,
@ -106,19 +107,33 @@ from skyvern.forge.sdk.copilot.outcome_verification_trace import (
record_criteria_lifecycle,
record_gate_decision,
)
from skyvern.forge.sdk.copilot.output_contracts import (
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
)
from skyvern.forge.sdk.copilot.output_policy import (
ACTUATION_OBLIGATION_STEER_REASON_CODE,
ACTUATION_OBLIGATION_UNMET_REASON_CODE,
UNVALIDATED_DISCLOSURE_PHRASES,
WORKFLOW_PRESENT_SENTINEL,
ActuationObligationEvaluation,
ActuationObligationStatus,
CannotActReason,
CopilotOutputKind,
OutputPolicyReason,
OutputPolicyVerdict,
actuation_obligation_key,
build_output_policy_diagnostics,
derive_output_kind,
evaluate_actuation_obligation,
evaluate_output_policy,
hard_block_output_policy_verdict,
normalize_response_scaffolding,
output_policy_verdict_from_trace_data,
output_policy_verdict_to_trace_data,
prior_turn_satisfies_actuation_terminal_condition,
request_policy_requires_durable_fill,
turn_intent_requires_actuation,
)
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
@ -481,16 +496,7 @@ def _store_request_policy_on_context(
chat_history_text=policy_inputs.chat_history_text,
previous_user_message=policy_inputs.previous_user_message,
)
_reconcile_completion_criteria_on_context(ctx, policy, policy_inputs)
ctx.request_policy = policy
ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test"
ctx.user_message = agent_user_message
ctx.block_goal_main_goal = _build_block_goal_main_goal(
user_message=agent_user_message,
chat_history_text=policy_chat_history_text,
global_llm_context=policy_inputs.global_llm_context,
)
ctx.turn_intent = build_turn_intent(
turn_intent = build_turn_intent(
user_message=policy_inputs.user_message,
workflow_yaml=policy_inputs.workflow_yaml,
chat_history=policy_inputs.chat_history_messages,
@ -503,6 +509,16 @@ def _store_request_policy_on_context(
classifier_result=turn_intent_classifier_result,
fix_origin=policy_inputs.fix_origin,
)
_reconcile_completion_criteria_on_context(ctx, policy, policy_inputs)
ctx.request_policy = policy
ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test"
ctx.user_message = agent_user_message
ctx.block_goal_main_goal = _build_block_goal_main_goal(
user_message=agent_user_message,
chat_history_text=policy_chat_history_text,
global_llm_context=policy_inputs.global_llm_context,
)
ctx.turn_intent = turn_intent
def _turn_intent_log_fields(intent: TurnIntent | None) -> dict[str, Any]:
@ -1237,6 +1253,18 @@ _DRAFT_ONLY_MCP_TOOL_ALLOWLIST = frozenset({"get_block_schema", "validate_block"
_DRAFT_ONLY_NATIVE_TOOL_DENYLIST = frozenset(
{"discover_workflow_entrypoint", "inspect_page_for_composition", "fill_credential_field"}
)
_MUTATING_BROWSER_SCOUT_TOOLS = frozenset({"click", "press_key", "type_text", "select_option", "fill_credential_field"})
_STRUCTURAL_CANNOT_ACT_REASON_CODES = frozenset(
{OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE}
)
_FinalActionDataValue = str | int | float | bool | None
_ACTUATION_OBLIGATION_STEER_REPLY = (
"I need to act in the browser for this request. Please let me try the browser action, or tell me what blocks it."
)
_ACTUATION_OBLIGATION_UNMET_REPLY = (
"I still did not act in the browser for this request, so I cannot treat the task as completed. "
"Please retry or tell me what prevented the browser action."
)
def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy | None) -> bool:
@ -1248,6 +1276,108 @@ def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy |
)
def _turn_intent_requires_browser_state(turn_intent: TurnIntent | None) -> bool:
return isinstance(turn_intent, TurnIntent) and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
def _turn_intent_browser_only_tools(turn_intent: TurnIntent | None) -> bool:
return (
isinstance(turn_intent, TurnIntent)
and turn_intent.mode == TurnIntentMode.BUILD
and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
and not turn_intent.authority.may_update_workflow
and not turn_intent.authority.may_run_blocks
)
def _successful_mutating_browser_action_count(ctx: CopilotContext) -> int:
return sum(
1
for interaction in ctx.scout_trajectory
if isinstance(interaction, Mapping) and interaction.get("tool_name") in _MUTATING_BROWSER_SCOUT_TOOLS
)
def _successful_durable_fill_count(ctx: CopilotContext) -> int:
return int(trajectory_has_browser_fill_interaction(ctx.scout_trajectory))
def _typed_cannot_act_reason(ctx: CopilotContext) -> CannotActReason | None:
signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else None
if signal is None:
return None
if signal.blocker_kind == "authority_denied":
return None
if signal.blocker_kind == "missing_required_context":
return CannotActReason.MISSING_FIELD_VALUE
if signal.recovery_hint == "ask_user_clarifying":
return CannotActReason.AMBIGUOUS_TARGET
if signal.internal_reason_code in _STRUCTURAL_CANNOT_ACT_REASON_CODES:
return CannotActReason.STRUCTURAL_BLOCKER
return None
def _evaluate_actuation_obligation_for_output(
ctx: CopilotContext,
action_data: Mapping[str, _FinalActionDataValue],
response_type: str,
output_kind: CopilotOutputKind,
) -> ActuationObligationEvaluation:
cannot_act_reason = _typed_cannot_act_reason(ctx)
evaluation = evaluate_actuation_obligation(
turn_intent=ctx.turn_intent,
response_type=response_type,
output_kind=output_kind,
successful_mutating_browser_actions=_successful_mutating_browser_action_count(ctx),
cannot_act_reason=cannot_act_reason,
prior_turn_outcome=ctx.prior_turn_outcome,
)
if evaluation.status != ActuationObligationStatus.ALLOWED:
return evaluation
if (
not _turn_intent_browser_only_tools(ctx.turn_intent)
or not request_policy_requires_durable_fill(ctx.request_policy)
or response_type != "REPLY"
or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER
):
return evaluation
if _successful_durable_fill_count(ctx) > 0 or cannot_act_reason is not None:
return evaluation
obligation_key = actuation_obligation_key(ctx.turn_intent)
if prior_turn_satisfies_actuation_terminal_condition(ctx.prior_turn_outcome, obligation_key):
return ActuationObligationEvaluation(
status=ActuationObligationStatus.TERMINAL,
reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
obligation_key=obligation_key,
)
return ActuationObligationEvaluation(
status=ActuationObligationStatus.STEER,
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
obligation_key=obligation_key,
)
def _actuation_obligation_diagnostics(
ctx: CopilotContext,
evaluation: ActuationObligationEvaluation,
) -> dict[str, str | int | bool | None]:
if (
not turn_intent_requires_actuation(ctx.turn_intent)
and evaluation.status == ActuationObligationStatus.ALLOWED
and evaluation.cannot_act_reason is None
):
return {}
return {
"actuation_obligation_status": evaluation.status.value,
"actuation_obligation_reason_code": evaluation.reason_code or None,
"actuation_obligation_key": evaluation.obligation_key or None,
"cannot_act_reason": evaluation.cannot_act_reason.value if evaluation.cannot_act_reason is not None else None,
"successful_mutating_browser_actions": _successful_mutating_browser_action_count(ctx),
"durable_fill_required": request_policy_requires_durable_fill(ctx.request_policy),
"successful_durable_browser_fills": _successful_durable_fill_count(ctx),
}
def _mcp_tool_surface_for_turn(
alias_map: dict[str, str],
overlays: dict[str, Any],
@ -3303,6 +3433,15 @@ async def _translate_to_agent_result(
unvalidated=unvalidated,
output_kind=output_kind,
)
actuation_obligation = _evaluate_actuation_obligation_for_output(ctx, action_data, resp_type, output_kind)
if actuation_obligation.status == ActuationObligationStatus.STEER:
raw_output_policy_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
actuation_obligation_terminal = actuation_obligation.status == ActuationObligationStatus.TERMINAL
if actuation_obligation_terminal:
resp_type = "REPLY"
user_response = _ACTUATION_OBLIGATION_UNMET_REPLY
last_workflow = None
last_workflow_yaml = None
output_policy_verdict = _copy_output_policy_verdict(raw_output_policy_verdict)
soft_rewrite_reasons: list[OutputPolicyReason] = []
unbacked_workflow_delivery_rewritten = False
@ -3356,6 +3495,7 @@ async def _translate_to_agent_result(
hard_block_reason_codes=list(output_policy_verdict.reason_codes),
soft_rewrite_reason_codes=soft_rewrite_reasons,
)
output_policy_diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation))
trace_data = output_policy_verdict_to_trace_data(
output_policy_verdict,
surface="final_translation",
@ -3386,15 +3526,22 @@ async def _translate_to_agent_result(
else []
)
reason_code = ",".join(reason_codes)
terminal_reason = None
if actuation_obligation_terminal:
reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE
terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE
final_user_response, turn_outcome = apply_repeated_reply_guard(
final_text=final_user_response,
attempted_kind=attempted_kind,
blocked_signatures=ctx.blocked_reply_signatures,
reason_code=reason_code,
terminal_reason=terminal_reason,
turn_intent=ctx.turn_intent,
tool_calls=[name for name in tool_call_names if name],
)
if actuation_obligation.reason_code:
turn_outcome = turn_outcome.model_copy(update={"actuation_obligation_key": actuation_obligation.obligation_key})
return _finalize_result_with_blocker_override(
ctx,
@ -3595,6 +3742,7 @@ def _blocked_final_output_kind(verdict: OutputPolicyVerdict) -> CopilotOutputKin
OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,
OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
}
if any(reason in clarification_reasons for reason in verdict.reason_codes):
return CopilotOutputKind.CLARIFICATION_REQUEST
@ -3661,6 +3809,16 @@ def _evaluate_copilot_final_output_policy(
workflow_attempted=workflow_attempted,
output_kind=output_kind,
)
actuation_obligation = _evaluate_actuation_obligation_for_output(
ctx,
action_data,
policy_response_type,
output_kind,
)
if actuation_obligation.status == ActuationObligationStatus.STEER:
raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
elif actuation_obligation.status == ActuationObligationStatus.TERMINAL:
raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_UNMET)
hard_verdict = hard_block_output_policy_verdict(raw_verdict)
deferred_reason_codes = _defer_avoidable_ask_to_recycle(ctx, hard_verdict, response_type)
if deferred_reason_codes is not None:
@ -3674,6 +3832,7 @@ def _evaluate_copilot_final_output_policy(
hard_block_reason_codes=list(hard_verdict.reason_codes),
soft_rewrite_reason_codes=[],
)
diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation))
if deferred_reason_codes is not None:
diagnostics["deferred_to_recycle"] = True
diagnostics["deferred_reason_codes"] = [reason.value for reason in deferred_reason_codes]
@ -3878,6 +4037,10 @@ def _build_output_policy_blocked_result(
"workflow without a tracked URL, re-select it so Copilot can confirm its URL scope."
)
add_saved_draft_copy = True
elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes:
user_response = _ACTUATION_OBLIGATION_UNMET_REPLY
elif OutputPolicyReason.ACTUATION_OBLIGATION_STEER in verdict.reason_codes:
user_response = _ACTUATION_OBLIGATION_STEER_REPLY
elif preserved_workflow is not None:
user_response = (
"I could not safely return that chat reply, but the workflow draft is still saved. "
@ -3908,13 +4071,37 @@ def _build_output_policy_blocked_result(
composed_from_recorded_evidence = True
if preserved_workflow is not None and add_saved_draft_copy:
user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
has_non_actuation_hard_block = any(
reason
not in {
OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
OutputPolicyReason.ACTUATION_OBLIGATION_UNMET,
}
for reason in verdict.reason_codes
)
if has_non_actuation_hard_block:
blocked_reason_code = "output_policy_block"
blocked_terminal_reason: str | None = "output_policy_block"
elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes:
blocked_reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE
blocked_terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE
else:
blocked_reason_code = ACTUATION_OBLIGATION_STEER_REASON_CODE
blocked_terminal_reason = None
final_user_response, output_policy_outcome = apply_repeated_reply_guard(
final_text=user_response,
attempted_kind=ResponseKind.CLARIFY,
blocked_signatures=ctx.blocked_reply_signatures,
reason_code="output_policy_block",
terminal_reason="output_policy_block",
reason_code=blocked_reason_code,
terminal_reason=blocked_terminal_reason,
)
if blocked_reason_code in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}:
key = ""
if output_policy_diagnostics is not None:
key = str(output_policy_diagnostics.get("actuation_obligation_key") or "")
output_policy_outcome = output_policy_outcome.model_copy(
update={"actuation_obligation_key": key or actuation_obligation_key(ctx.turn_intent)}
)
if composed_from_recorded_evidence and fallback_user_response is not None:
composed_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
@ -3997,6 +4184,7 @@ async def run_copilot_agent(
prior_copilot_workflow_yaml: str | None = None,
prior_block_count: int | None = None,
stored_completion_criteria: StoredCriteriaSnapshot | None = None,
prior_turn_outcome: TurnOutcome | None = None,
) -> AgentResult:
# One id per turn — passed to every downstream AgentResult and
# CopilotContext so the envelope and terminal frames correlate. The
@ -4034,6 +4222,7 @@ async def run_copilot_agent(
prior_block_count=prior_block_count,
ctx_sink=ctx_sink,
stored_completion_criteria=stored_completion_criteria,
prior_turn_outcome=prior_turn_outcome,
)
except Exception as exc:
LOG.error(
@ -4102,6 +4291,7 @@ async def _run_copilot_turn_impl(
prior_block_count: int | None = None,
ctx_sink: list[CopilotContext] | None = None,
stored_completion_criteria: StoredCriteriaSnapshot | None = None,
prior_turn_outcome: TurnOutcome | None = None,
) -> AgentResult:
copilot_config = config or CopilotConfig(security_rules=security_rules)
chat_history_text = _format_chat_history(chat_history)
@ -4166,6 +4356,7 @@ async def _run_copilot_turn_impl(
turn_index=turn_index,
prior_block_count=prior_block_count,
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
prior_turn_outcome=prior_turn_outcome,
block_authoring_policy=copilot_config.block_authoring_policy,
impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
copilot_config=copilot_config,

15
skyvern/forge/sdk/copilot/code_block_synthesis.py
View file

@ -1101,6 +1101,21 @@ def trajectory_has_credential_fill(trajectory: Sequence[Mapping[str, Any]]) -> b
return False
def trajectory_has_browser_fill_interaction(trajectory: Sequence[Mapping[str, Any]]) -> bool:
for interaction in trajectory:
tool_name = str(interaction.get("tool_name") or "")
typed_length = interaction.get("typed_length")
if tool_name == "type_text" and (
(isinstance(typed_length, int) and typed_length > 0) or str(interaction.get("typed_value") or "").strip()
):
return True
if tool_name == "select_option" and str(interaction.get("value") or "").strip():
return True
if tool_name == CREDENTIAL_FILL_TOOL_NAME and str(interaction.get("credential_field") or "").strip():
return True
return False
def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
return build_artifact_metadata_skeleton(trajectory, block_label=_SYNTHESIZED_BLOCK_LABEL)

1
skyvern/forge/sdk/copilot/context.py
View file

@ -643,6 +643,7 @@ class CopilotContext(AgentContext):
target_block_label: str | None = None
turn_intent: TurnIntent | None = None
turn_context_packet: TurnContextPacket | None = None
prior_turn_outcome: TurnOutcome | None = None
latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
blocked_reply_signatures: list[str] = field(default_factory=list)

44
skyvern/forge/sdk/copilot/enforcement.py
View file

@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
trajectory_has_browser_fill_interaction,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_paths
from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
@ -76,7 +77,10 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
from skyvern.forge.sdk.copilot.narration import TransitionKind
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
from skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding
from skyvern.forge.sdk.copilot.output_policy import (
completion_criterion_requires_browser_fill_delivery,
normalize_response_scaffolding,
)
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
looks_like_workflow_delivery_claim,
@ -116,7 +120,7 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
)
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode
from skyvern.utils.token_counter import count_tokens
if TYPE_CHECKING:
@ -188,6 +192,7 @@ SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"
_SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS = frozenset(
{SYNTHESIZED_BLOCK_PERSISTENCE_TOOL, "fill_credential_field", "update_workflow"}
)
_ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL = "type_text"
_SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS = frozenset(
{"click", "press_key", "type_text", "select_option", "navigate_browser"}
)
@ -2394,6 +2399,39 @@ def _uncovered_output_reject_admits_evaluate(ctx: CopilotContext, tool_name: str
return tool_name == "evaluate" and bool(_active_uncovered_output_reject_paths(ctx))
def _actuation_obligation_live_fill_delivery_required(ctx: CopilotContext) -> bool:
turn_intent = getattr(ctx, "turn_intent", None)
if (
not isinstance(turn_intent, TurnIntent)
or turn_intent.mode != TurnIntentMode.BUILD
or RequiredContextKey.BROWSER_STATE not in turn_intent.required_context
):
return False
if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
return False
request_policy = getattr(ctx, "request_policy", None)
criteria: list[CompletionCriterion] = []
if isinstance(request_policy, RequestPolicy):
criteria.extend(request_policy.completion_criteria)
turn_state = getattr(ctx, "completion_criteria_turn_state", None)
if turn_state is not None and turn_state.decision is not None:
criteria.extend(turn_state.decision.criteria)
if any(completion_criterion_requires_browser_fill_delivery(criterion) for criterion in criteria):
return True
trajectory = getattr(ctx, "scout_trajectory", None)
return trajectory_has_browser_fill_interaction(trajectory) if isinstance(trajectory, list) else False
def _actuation_obligation_required_fill_tool(ctx: CopilotContext) -> str | None:
if _actuation_obligation_live_fill_delivery_required(ctx):
return _ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL
return None
def _actuation_obligation_admits_required_fill_tool(ctx: CopilotContext, tool_name: str) -> bool:
return tool_name == _actuation_obligation_required_fill_tool(ctx)
def consume_uncovered_output_reopen_event(ctx: CopilotContext) -> bool:
"""Arm a one-shot scout-window reopen for the first author-time reject citing an uncovered
requested-output path. Returns True only on that first reject per structural identity; a
@ -2501,6 +2539,8 @@ def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToo
return None
if _uncovered_output_reject_admits_evaluate(ctx, tool_name):
return None
if _actuation_obligation_admits_required_fill_tool(ctx, tool_name):
return None
if (
ambiguous_selector_rescout_state != "block"
and not _should_force_synthesized_block_persistence(ctx)

115
skyvern/forge/sdk/copilot/output_policy.py
View file

@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
looks_like_workflow_yaml_in_chat,
)
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
RequestPolicy,
contains_email_password_pair,
request_policy_has_present_completion_contract,
@ -21,6 +22,7 @@ from skyvern.forge.sdk.copilot.secret_redaction import (
RAW_SECRET_PATTERNS,
SECRET_KEYWORD_ASSIGNMENT_PATTERN,
)
from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
block_credential_ids,
credential_param_ids,
@ -30,8 +32,12 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
workflow_credential_ids_from_parsed,
workflow_credential_origins_from_parsed,
)
from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
WORKFLOW_PRESENT_SENTINEL = object()
ACTUATION_OBLIGATION_STEER_REASON_CODE = "actuation_obligation_steer"
ACTUATION_OBLIGATION_UNMET_REASON_CODE = "actuation_obligation_unmet"
ACTUATION_OBLIGATION_BROWSER_ACTION_KEY = "browser_state:build:no_update:no_run"
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
_PLACEHOLDER_MARKERS = ("{{", "{%", "[REDACTED_SECRET]")
# RHS of a secret-keyword assignment that references a bound value instead of carrying one:
@ -183,6 +189,26 @@ class CopilotOutputKind(StrEnum):
WORKFLOW_RUN_RESULT = "workflow_run_result"
class CannotActReason(StrEnum):
MISSING_FIELD_VALUE = "missing_field_value"
AMBIGUOUS_TARGET = "ambiguous_target"
STRUCTURAL_BLOCKER = "structural_blocker"
class ActuationObligationStatus(StrEnum):
ALLOWED = "allowed"
STEER = "steer"
TERMINAL = "terminal"
@dataclass(frozen=True)
class ActuationObligationEvaluation:
status: ActuationObligationStatus = ActuationObligationStatus.ALLOWED
reason_code: str = ""
cannot_act_reason: CannotActReason | None = None
obligation_key: str = ""
class OutputPolicyReason(StrEnum):
RAW_SECRET_LEAK = "raw_secret_leak"
REQUEST_POLICY_CLARIFICATION_BYPASS = "request_policy_clarification_bypass"
@ -199,6 +225,8 @@ class OutputPolicyReason(StrEnum):
SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak"
WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply"
AVOIDABLE_OUTPUT_FIELD_CONFIRMATION = "avoidable_output_field_confirmation"
ACTUATION_OBLIGATION_STEER = ACTUATION_OBLIGATION_STEER_REASON_CODE
ACTUATION_OBLIGATION_UNMET = ACTUATION_OBLIGATION_UNMET_REASON_CODE
@dataclass
@ -232,10 +260,97 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset(
OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK,
OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING,
OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION,
OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
OutputPolicyReason.ACTUATION_OBLIGATION_UNMET,
}
)
def coerce_cannot_act_reason(value: str | None) -> CannotActReason | None:
if value is None:
return None
try:
return CannotActReason(value)
except ValueError:
return None
def turn_intent_requires_actuation(turn_intent: TurnIntent | None) -> bool:
return (
isinstance(turn_intent, TurnIntent)
and turn_intent.mode in {TurnIntentMode.BUILD, TurnIntentMode.UNKNOWN}
and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
and not turn_intent.authority.may_update_workflow
and not turn_intent.authority.may_run_blocks
)
def actuation_obligation_key(turn_intent: TurnIntent | None) -> str:
if not turn_intent_requires_actuation(turn_intent):
return ""
if turn_intent is not None and turn_intent.mode == TurnIntentMode.UNKNOWN:
return "browser_state:unknown:no_update:no_run"
return ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
def completion_criterion_requires_browser_fill_delivery(criterion: CompletionCriterion) -> bool:
if criterion.kind == "terminal_action" and criterion.terminal_action_family == "form":
return True
return criterion.kind == "outcome" and criterion.level == "run" and criterion.method_mandated
def request_policy_requires_durable_fill(request_policy: RequestPolicy | None) -> bool:
return isinstance(request_policy, RequestPolicy) and any(
completion_criterion_requires_browser_fill_delivery(criterion)
for criterion in request_policy.completion_criteria
)
def prior_turn_satisfies_actuation_terminal_condition(
prior_turn_outcome: TurnOutcome | None,
obligation_key: str,
) -> bool:
# One key covers the turn's actuation obligation; a later partial action
# still failed the same browser-fill contract.
return (
prior_turn_outcome is not None
and prior_turn_outcome.reason_code
in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}
and prior_turn_outcome.actuation_obligation_key == obligation_key
)
def evaluate_actuation_obligation(
*,
turn_intent: TurnIntent | None,
response_type: str,
output_kind: CopilotOutputKind,
successful_mutating_browser_actions: int,
cannot_act_reason: CannotActReason | None,
prior_turn_outcome: TurnOutcome | None,
) -> ActuationObligationEvaluation:
obligation_key = actuation_obligation_key(turn_intent)
if (
not turn_intent_requires_actuation(turn_intent)
or response_type != "REPLY"
or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER
):
return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason)
if successful_mutating_browser_actions > 0 or cannot_act_reason is not None:
return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason, obligation_key=obligation_key)
if prior_turn_satisfies_actuation_terminal_condition(prior_turn_outcome, obligation_key):
return ActuationObligationEvaluation(
status=ActuationObligationStatus.TERMINAL,
reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
obligation_key=obligation_key,
)
return ActuationObligationEvaluation(
status=ActuationObligationStatus.STEER,
reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
obligation_key=obligation_key,
)
def hard_block_output_policy_verdict(verdict: OutputPolicyVerdict) -> OutputPolicyVerdict:
hard_reasons = [reason for reason in verdict.reason_codes if reason in _FINAL_OUTPUT_HARD_BLOCK_REASONS]
return OutputPolicyVerdict(

1
skyvern/forge/sdk/routes/workflow_copilot.py
View file

@ -2069,6 +2069,7 @@ async def _new_copilot_chat_post(
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
prior_block_count=prior_block_count,
stored_completion_criteria=stored_completion_criteria,
prior_turn_outcome=prior_turn_outcome,
)
agent_result.turn_outcome = _with_current_copilot_code_mode_metadata(

1
skyvern/forge/sdk/schemas/copilot_turn_outcome.py
View file

@ -29,6 +29,7 @@ class TurnOutcome(BaseModel):
turn_intent_summary: dict[str, Any] = Field(default_factory=dict)
response_kind: ResponseKind
reason_code: str = ""
actuation_obligation_key: str = ""
normalized_reply_signature: str = ""
tool_calls: list[str] = Field(default_factory=list)
terminal_reason: str | None = None