From 9949fb0dedb7f50775eaf4a011485c44cfc6fefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Celal=20Zamano=C4=9Flu?= Date: Wed, 8 Jul 2026 18:24:49 +0300 Subject: [PATCH] feat(SKY-11988): add credentialPrompt signal to copilot narrative payload (#7200) Co-authored-by: AronPerez --- skyvern/forge/sdk/copilot/agent.py | 6 ++ skyvern/forge/sdk/copilot/context.py | 2 + skyvern/forge/sdk/copilot/request_policy.py | 30 +++++++ ...test_copilot_make_agent_result_backfill.py | 47 ++++++++++- ...lot_request_policy_workflow_credentials.py | 84 ++++++++++++++++++- 5 files changed, 166 insertions(+), 3 deletions(-) diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py index f22d6da7d..9ff53af97 100644 --- a/skyvern/forge/sdk/copilot/agent.py +++ b/skyvern/forge/sdk/copilot/agent.py @@ -136,6 +136,7 @@ from skyvern.forge.sdk.copilot.request_policy import ( CompletionCriterion, RequestPolicy, build_request_policy, + credential_prompt_reason, redact_raw_secrets_for_prompt, ) from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason @@ -1524,6 +1525,11 @@ def _make_agent_result( payload_updates["proposalDisposition"] = proposal_disposition if turn_outcome is not None and "responseKind" not in narrative_payload: payload_updates["responseKind"] = turn_outcome.response_kind.value + if "credentialPrompt" not in narrative_payload: + policy = ctx.request_policy if ctx is not None else None + reason = credential_prompt_reason(policy, kwargs.get("user_response")) + if reason: + payload_updates["credentialPrompt"] = {"reason": reason} if ctx is not None and "verifiedSuccess" not in narrative_payload: payload_updates["verifiedSuccess"] = bool(verified_goal_claim_authorized(ctx)) if ctx is not None and "outcomeAdjudication" not in narrative_payload: diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py index 36f4566b6..ddf120e06 100644 --- a/skyvern/forge/sdk/copilot/context.py +++ b/skyvern/forge/sdk/copilot/context.py @@ -82,6 +82,8 @@ class TurnNarrativePayload(TypedDict): verifiedSuccess: NotRequired[bool] # Verdict-state summary from the turn's latest evaluated adjudication. outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication] + # {"reason": }, set when this turn surfaces a credential need. + credentialPrompt: NotRequired[dict[str, str]] designStarted: bool designEnded: bool draft: NarrativeDraft | None diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py index 38daae9d7..6fee80510 100644 --- a/skyvern/forge/sdk/copilot/request_policy.py +++ b/skyvern/forge/sdk/copilot/request_policy.py @@ -78,9 +78,14 @@ ClarificationReason = Literal[ ] RawSecretHandling = Literal["none", "block", "redacted_draft"] _VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_args(ClarificationReason)) +# Gates guardrails.py's deferred-draft tool authority — narrower than the prompt set below. CREDENTIAL_DEFERRED_DRAFT_REASONS: frozenset[ClarificationReason] = frozenset( {"workflow_credential_inputs_unbound", "credential_name_unresolved"} ) +# Broader: any reason credential_prompt_reason() should surface an add-credential CTA for. +CREDENTIAL_PROMPT_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset( + {"raw_secret", "credential_name_unresolved", "credential_invention_requested", "workflow_credential_inputs_unbound"} +) _PRE_RESOLUTION_CLARIFICATION_REASONS = { "credential_invention_requested", "ambiguous_loop_edit", @@ -97,6 +102,9 @@ _REASONS_OVERRIDDEN_BY_CREDENTIAL_REFS = { _CREDENTIALS_UI_DIRECTIONS = ( f"You can find or add saved credentials at {settings.SKYVERN_APP_URL.rstrip('/')}/credentials." ) +# Matches any final reply containing these substrings, not just credential-blocking +# ones; safe today because every such emitter routes through _CREDENTIALS_UI_DIRECTIONS. +_CREDENTIAL_PROMPT_TEXT_MARKERS = ("/credentials", "credentials ui") # Stable tail of every raw-secret refusal; transcript redaction keys off it, so all refusal emitters must keep it verbatim. RAW_SECRET_REFUSAL_SENTINEL = "DO NOT PROVIDE RAW LOGIN/PASSWORD" _RAW_SECRET_QUESTION = ( @@ -280,6 +288,10 @@ class RequestPolicy: allow_update_workflow: bool = True allow_run_blocks: bool = True allow_missing_credentials_in_draft: bool = False + # Narrower than the flag above: True only when a credential-specific path (an explicit + # code-block credential draft, or a redacted raw secret) set it, not the generic + # skip_test fallthrough that fires for any untested draft regardless of credentials. + credential_draft_deferred_explicitly: bool = False user_response_policy: str = "proceed" completion_contract: str | None = None completion_criteria: list[CompletionCriterion] = field(default_factory=list) @@ -313,6 +325,7 @@ class RequestPolicy: "allow_update_workflow": self.allow_update_workflow, "allow_run_blocks": self.allow_run_blocks, "allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft, + "credential_draft_deferred_explicitly": self.credential_draft_deferred_explicitly, "resolved_credential_count": len(self.resolved_credentials), "has_completion_contract": bool(self.completion_contract), "completion_criteria_count": len(self.graded_completion_criteria()), @@ -405,6 +418,21 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicy return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria) +def credential_prompt_reason(policy: RequestPolicy | None, final_text: str | None) -> str | None: + # Typed clarification_reason wins, then the explicit-defer flag — narrowly, since + # allow_missing_credentials_in_draft alone also covers the generic skip_test + # fallthrough with no credential involvement — then a text marker. + if isinstance(policy, RequestPolicy): + if policy.clarification_reason in CREDENTIAL_PROMPT_CLARIFICATION_REASONS: + return policy.clarification_reason + if policy.credential_draft_deferred_explicitly: + return "credential_deferred_draft" + normalized = " ".join((final_text or "").lower().split()) + if any(marker in normalized for marker in _CREDENTIAL_PROMPT_TEXT_MARKERS): + return "assistant_directed" + return None + + def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool: return ( isinstance(criterion.expected_output_value, bool) or criterion.expected_output_shape == "goal_judgment_boolean" @@ -1793,6 +1821,7 @@ def _apply_explicit_code_block_credential_draft_policy(policy: RequestPolicy, us policy.allow_update_workflow = True policy.allow_run_blocks = False policy.allow_missing_credentials_in_draft = True + policy.credential_draft_deferred_explicitly = True policy.requires_user_clarification = False policy.user_response_policy = "proceed" policy.clarification_reason = "none" @@ -2485,6 +2514,7 @@ async def build_request_policy( policy.allow_update_workflow = True policy.allow_run_blocks = False policy.allow_missing_credentials_in_draft = True + policy.credential_draft_deferred_explicitly = True policy.clarification_reason = "none" policy.clarification_question = None diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.py index 9dcedbdc6..813a8c731 100644 --- a/tests/unit/test_copilot_make_agent_result_backfill.py +++ b/tests/unit/test_copilot_make_agent_result_backfill.py @@ -6,8 +6,10 @@ from __future__ import annotations import pytest -from skyvern.forge.sdk.copilot.agent import _make_agent_result -from skyvern.forge.sdk.copilot.context import CopilotContext +from skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_result +from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal +from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext +from skyvern.forge.sdk.copilot.request_policy import RequestPolicy from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome from tests.unit.copilot_test_helpers import make_copilot_ctx as _ctx from tests.unit.copilot_test_helpers import make_verified_goal_contract @@ -117,3 +119,44 @@ def test_backfill_tolerates_missing_payload() -> None: def test_missing_payload_is_allowed_without_ctx() -> None: result = _result(None, turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None) assert result.narrative_payload is None + + +def test_backfill_adds_credential_prompt_for_typed_clarification_reason() -> None: + ctx = _ctx(request_policy=RequestPolicy(clarification_reason="credential_name_unresolved")) + result = _result(ctx, turn_outcome=_outcome(ResponseKind.CLARIFY), narrative_payload=_payload()) + assert result.narrative_payload is not None + assert result.narrative_payload["credentialPrompt"] == {"reason": "credential_name_unresolved"} + + +def test_blocker_override_path_adds_credential_prompt_from_request_policy() -> None: + ctx = _ctx(request_policy=RequestPolicy(clarification_reason="workflow_credential_inputs_unbound")) + ctx.blocker_signal = CopilotToolBlockerSignal( + blocker_kind="authority_denied", + agent_steering_text="Reply to the user without updating the workflow.", + user_facing_reason="I couldn't find the required credentials for the existing workflow.", + recovery_hint="report_blocker_to_user", + internal_reason_code="turn_intent_no_mutation_run_blocked", + blocked_tool="update_workflow", + ) + pre_override = AgentResult(user_response="agent reply", updated_workflow=None, global_llm_context=None) + + overridden = _finalize_result_with_blocker_override(ctx, pre_override) + + assert overridden.narrative_payload is not None + assert overridden.narrative_payload["credentialPrompt"] == {"reason": "workflow_credential_inputs_unbound"} + + +def test_backfill_adds_credential_prompt_from_text_marker_when_no_policy_signal() -> None: + result = _result( + _ctx(), + user_response="You can add one at https://app.skyvern.com/credentials.", + narrative_payload=_payload(), + ) + assert result.narrative_payload is not None + assert result.narrative_payload["credentialPrompt"] == {"reason": "assistant_directed"} + + +def test_backfill_omits_credential_prompt_when_no_signal_present() -> None: + result = _result(_ctx(), user_response="Done, the workflow is ready.", narrative_payload=_payload()) + assert result.narrative_payload is not None + assert "credentialPrompt" not in result.narrative_payload diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py index 8a2deba98..06417464c 100644 --- a/tests/unit/test_copilot_request_policy_workflow_credentials.py +++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py @@ -8,7 +8,14 @@ import pytest from skyvern.config import settings from skyvern.forge.sdk.copilot.context import CredentialCheck, StructuredContext -from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound, build_request_policy +from skyvern.forge.sdk.copilot.request_policy import ( + CREDENTIAL_DEFERRED_DRAFT_REASONS, + CREDENTIAL_PROMPT_CLARIFICATION_REASONS, + RequestPolicy, + _workflow_credential_inputs_unbound, + build_request_policy, + credential_prompt_reason, +) def _yaml(body: str) -> str: @@ -339,6 +346,81 @@ async def test_fallback_code_block_generic_one_time_code_does_not_skip_run() -> assert policy.testing_intent == "unspecified" +def test_credential_prompt_clarification_reasons_membership() -> None: + assert CREDENTIAL_PROMPT_CLARIFICATION_REASONS == { + "raw_secret", + "credential_name_unresolved", + "credential_invention_requested", + "workflow_credential_inputs_unbound", + } + + +def test_credential_deferred_draft_reasons_is_narrower_than_prompt_reasons() -> None: + assert CREDENTIAL_DEFERRED_DRAFT_REASONS < CREDENTIAL_PROMPT_CLARIFICATION_REASONS + + +def test_credential_prompt_reason_typed_reason_wins_over_deferred_draft_flag() -> None: + policy = RequestPolicy(clarification_reason="credential_name_unresolved", allow_missing_credentials_in_draft=True) + assert credential_prompt_reason(policy, None) == "credential_name_unresolved" + + +def test_credential_prompt_reason_deferred_draft_when_reason_cleared_but_flag_set() -> None: + # Mirrors the explicit-defer path (_apply_explicit_code_block_credential_draft_policy), + # which clears clarification_reason to "none" while leaving both flags set. + policy = RequestPolicy( + clarification_reason="none", + allow_missing_credentials_in_draft=True, + credential_draft_deferred_explicitly=True, + ) + assert credential_prompt_reason(policy, None) == "credential_deferred_draft" + + +def test_credential_prompt_reason_ignores_generic_skip_test_with_no_credential_signal() -> None: + # allow_missing_credentials_in_draft alone also fires for the generic skip_test + # fallthrough (any "draft only, don't test" turn), independent of credentials; + # only credential_draft_deferred_explicitly means a credential was really deferred. + policy = RequestPolicy( + testing_intent="skip_test", + clarification_reason="none", + allow_missing_credentials_in_draft=True, + credential_draft_deferred_explicitly=False, + ) + assert credential_prompt_reason(policy, None) is None + + +@pytest.mark.asyncio +async def test_explicit_code_block_login_block_ban_surfaces_credential_prompt_end_to_end() -> None: + policy = await build_request_policy( + user_message="Write this as a code block. Do not create a login block for this part.", + workflow_yaml="", + chat_history=[], + global_llm_context="", + organization_id="o_test", + handler=None, + ) + assert policy.credential_input_kind == "none" + assert policy.testing_intent == "skip_test" + assert credential_prompt_reason(policy, None) == "credential_deferred_draft" + + +@pytest.mark.parametrize( + "text", + [ + "Add it at https://app.skyvern.com/credentials.", + "ADD IT AT HTTPS://APP.SKYVERN.COM/CREDENTIALS.", + "Store it in the Credentials UI first.", + "store it in the CREDENTIALS UI first.", + ], +) +def test_credential_prompt_reason_marker_fallback_is_case_insensitive(text: str) -> None: + assert credential_prompt_reason(None, text) == "assistant_directed" + + +def test_credential_prompt_reason_policy_none_is_safe() -> None: + assert credential_prompt_reason(None, None) is None + assert credential_prompt_reason(None, "Everything is set, no action needed.") is None + + @pytest.mark.asyncio async def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_log_only( monkeypatch: pytest.MonkeyPatch,