feat(SKY-11988): add credentialPrompt signal to copilot narrative payload (#7200)

Co-authored-by: AronPerez <aperez0295@gmail.com>

skyvern/forge/sdk/copilot/agent.py

@@ -136,6 +136,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
     CompletionCriterion,
     RequestPolicy,
     build_request_policy,
+    credential_prompt_reason,
     redact_raw_secrets_for_prompt,
 )
 from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason
@@ -1524,6 +1525,11 @@ def _make_agent_result(
             payload_updates["proposalDisposition"] = proposal_disposition
         if turn_outcome is not None and "responseKind" not in narrative_payload:
             payload_updates["responseKind"] = turn_outcome.response_kind.value
+        if "credentialPrompt" not in narrative_payload:
+            policy = ctx.request_policy if ctx is not None else None
+            reason = credential_prompt_reason(policy, kwargs.get("user_response"))
+            if reason:
+                payload_updates["credentialPrompt"] = {"reason": reason}
         if ctx is not None and "verifiedSuccess" not in narrative_payload:
             payload_updates["verifiedSuccess"] = bool(verified_goal_claim_authorized(ctx))
         if ctx is not None and "outcomeAdjudication" not in narrative_payload:

skyvern/forge/sdk/copilot/context.py

@@ -82,6 +82,8 @@ class TurnNarrativePayload(TypedDict):
     verifiedSuccess: NotRequired[bool]
     # Verdict-state summary from the turn's latest evaluated adjudication.
     outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication]
+    # {"reason": <credential_prompt_reason() token>}, set when this turn surfaces a credential need.
+    credentialPrompt: NotRequired[dict[str, str]]
     designStarted: bool
     designEnded: bool
     draft: NarrativeDraft | None

skyvern/forge/sdk/copilot/request_policy.py

@@ -78,9 +78,14 @@ ClarificationReason = Literal[
 ]
 RawSecretHandling = Literal["none", "block", "redacted_draft"]
 _VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_args(ClarificationReason))
+# Gates guardrails.py's deferred-draft tool authority — narrower than the prompt set below.
 CREDENTIAL_DEFERRED_DRAFT_REASONS: frozenset[ClarificationReason] = frozenset(
     {"workflow_credential_inputs_unbound", "credential_name_unresolved"}
 )
+# Broader: any reason credential_prompt_reason() should surface an add-credential CTA for.
+CREDENTIAL_PROMPT_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(
+    {"raw_secret", "credential_name_unresolved", "credential_invention_requested", "workflow_credential_inputs_unbound"}
+)
 _PRE_RESOLUTION_CLARIFICATION_REASONS = {
     "credential_invention_requested",
     "ambiguous_loop_edit",
@@ -97,6 +102,9 @@ _REASONS_OVERRIDDEN_BY_CREDENTIAL_REFS = {
 _CREDENTIALS_UI_DIRECTIONS = (
     f"You can find or add saved credentials at {settings.SKYVERN_APP_URL.rstrip('/')}/credentials."
 )
+# Matches any final reply containing these substrings, not just credential-blocking
+# ones; safe today because every such emitter routes through _CREDENTIALS_UI_DIRECTIONS.
+_CREDENTIAL_PROMPT_TEXT_MARKERS = ("/credentials", "credentials ui")
 # Stable tail of every raw-secret refusal; transcript redaction keys off it, so all refusal emitters must keep it verbatim.
 RAW_SECRET_REFUSAL_SENTINEL = "DO NOT PROVIDE RAW LOGIN/PASSWORD"
 _RAW_SECRET_QUESTION = (
@@ -280,6 +288,10 @@ class RequestPolicy:
     allow_update_workflow: bool = True
     allow_run_blocks: bool = True
     allow_missing_credentials_in_draft: bool = False
+    # Narrower than the flag above: True only when a credential-specific path (an explicit
+    # code-block credential draft, or a redacted raw secret) set it, not the generic
+    # skip_test fallthrough that fires for any untested draft regardless of credentials.
+    credential_draft_deferred_explicitly: bool = False
     user_response_policy: str = "proceed"
     completion_contract: str | None = None
     completion_criteria: list[CompletionCriterion] = field(default_factory=list)
@@ -313,6 +325,7 @@ class RequestPolicy:
             "allow_update_workflow": self.allow_update_workflow,
             "allow_run_blocks": self.allow_run_blocks,
             "allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,
+            "credential_draft_deferred_explicitly": self.credential_draft_deferred_explicitly,
             "resolved_credential_count": len(self.resolved_credentials),
             "has_completion_contract": bool(self.completion_contract),
             "completion_criteria_count": len(self.graded_completion_criteria()),
@@ -405,6 +418,21 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicy
     return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
 
 
+def credential_prompt_reason(policy: RequestPolicy | None, final_text: str | None) -> str | None:
+    # Typed clarification_reason wins, then the explicit-defer flag — narrowly, since
+    # allow_missing_credentials_in_draft alone also covers the generic skip_test
+    # fallthrough with no credential involvement — then a text marker.
+    if isinstance(policy, RequestPolicy):
+        if policy.clarification_reason in CREDENTIAL_PROMPT_CLARIFICATION_REASONS:
+            return policy.clarification_reason
+        if policy.credential_draft_deferred_explicitly:
+            return "credential_deferred_draft"
+    normalized = " ".join((final_text or "").lower().split())
+    if any(marker in normalized for marker in _CREDENTIAL_PROMPT_TEXT_MARKERS):
+        return "assistant_directed"
+    return None
+
+
 def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool:
     return (
         isinstance(criterion.expected_output_value, bool) or criterion.expected_output_shape == "goal_judgment_boolean"
@@ -1793,6 +1821,7 @@ def _apply_explicit_code_block_credential_draft_policy(policy: RequestPolicy, us
     policy.allow_update_workflow = True
     policy.allow_run_blocks = False
     policy.allow_missing_credentials_in_draft = True
+    policy.credential_draft_deferred_explicitly = True
     policy.requires_user_clarification = False
     policy.user_response_policy = "proceed"
     policy.clarification_reason = "none"
@@ -2485,6 +2514,7 @@ async def build_request_policy(
         policy.allow_update_workflow = True
         policy.allow_run_blocks = False
         policy.allow_missing_credentials_in_draft = True
+        policy.credential_draft_deferred_explicitly = True
         policy.clarification_reason = "none"
         policy.clarification_question = None
 

tests/unit/test_copilot_make_agent_result_backfill.py

@@ -6,8 +6,10 @@ from __future__ import annotations
 
 import pytest
 
-from skyvern.forge.sdk.copilot.agent import _make_agent_result
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_result
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
 from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
 from tests.unit.copilot_test_helpers import make_copilot_ctx as _ctx
 from tests.unit.copilot_test_helpers import make_verified_goal_contract
@@ -117,3 +119,44 @@ def test_backfill_tolerates_missing_payload() -> None:
 def test_missing_payload_is_allowed_without_ctx() -> None:
     result = _result(None, turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)
     assert result.narrative_payload is None
+
+
+def test_backfill_adds_credential_prompt_for_typed_clarification_reason() -> None:
+    ctx = _ctx(request_policy=RequestPolicy(clarification_reason="credential_name_unresolved"))
+    result = _result(ctx, turn_outcome=_outcome(ResponseKind.CLARIFY), narrative_payload=_payload())
+    assert result.narrative_payload is not None
+    assert result.narrative_payload["credentialPrompt"] == {"reason": "credential_name_unresolved"}
+
+
+def test_blocker_override_path_adds_credential_prompt_from_request_policy() -> None:
+    ctx = _ctx(request_policy=RequestPolicy(clarification_reason="workflow_credential_inputs_unbound"))
+    ctx.blocker_signal = CopilotToolBlockerSignal(
+        blocker_kind="authority_denied",
+        agent_steering_text="Reply to the user without updating the workflow.",
+        user_facing_reason="I couldn't find the required credentials for the existing workflow.",
+        recovery_hint="report_blocker_to_user",
+        internal_reason_code="turn_intent_no_mutation_run_blocked",
+        blocked_tool="update_workflow",
+    )
+    pre_override = AgentResult(user_response="agent reply", updated_workflow=None, global_llm_context=None)
+
+    overridden = _finalize_result_with_blocker_override(ctx, pre_override)
+
+    assert overridden.narrative_payload is not None
+    assert overridden.narrative_payload["credentialPrompt"] == {"reason": "workflow_credential_inputs_unbound"}
+
+
+def test_backfill_adds_credential_prompt_from_text_marker_when_no_policy_signal() -> None:
+    result = _result(
+        _ctx(),
+        user_response="You can add one at https://app.skyvern.com/credentials.",
+        narrative_payload=_payload(),
+    )
+    assert result.narrative_payload is not None
+    assert result.narrative_payload["credentialPrompt"] == {"reason": "assistant_directed"}
+
+
+def test_backfill_omits_credential_prompt_when_no_signal_present() -> None:
+    result = _result(_ctx(), user_response="Done, the workflow is ready.", narrative_payload=_payload())
+    assert result.narrative_payload is not None
+    assert "credentialPrompt" not in result.narrative_payload

tests/unit/test_copilot_request_policy_workflow_credentials.py

@@ -8,7 +8,14 @@ import pytest
 
 from skyvern.config import settings
 from skyvern.forge.sdk.copilot.context import CredentialCheck, StructuredContext
-from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound, build_request_policy
+from skyvern.forge.sdk.copilot.request_policy import (
+    CREDENTIAL_DEFERRED_DRAFT_REASONS,
+    CREDENTIAL_PROMPT_CLARIFICATION_REASONS,
+    RequestPolicy,
+    _workflow_credential_inputs_unbound,
+    build_request_policy,
+    credential_prompt_reason,
+)
 
 
 def _yaml(body: str) -> str:
@@ -339,6 +346,81 @@ async def test_fallback_code_block_generic_one_time_code_does_not_skip_run() ->
     assert policy.testing_intent == "unspecified"
 
 
+def test_credential_prompt_clarification_reasons_membership() -> None:
+    assert CREDENTIAL_PROMPT_CLARIFICATION_REASONS == {
+        "raw_secret",
+        "credential_name_unresolved",
+        "credential_invention_requested",
+        "workflow_credential_inputs_unbound",
+    }
+
+
+def test_credential_deferred_draft_reasons_is_narrower_than_prompt_reasons() -> None:
+    assert CREDENTIAL_DEFERRED_DRAFT_REASONS < CREDENTIAL_PROMPT_CLARIFICATION_REASONS
+
+
+def test_credential_prompt_reason_typed_reason_wins_over_deferred_draft_flag() -> None:
+    policy = RequestPolicy(clarification_reason="credential_name_unresolved", allow_missing_credentials_in_draft=True)
+    assert credential_prompt_reason(policy, None) == "credential_name_unresolved"
+
+
+def test_credential_prompt_reason_deferred_draft_when_reason_cleared_but_flag_set() -> None:
+    # Mirrors the explicit-defer path (_apply_explicit_code_block_credential_draft_policy),
+    # which clears clarification_reason to "none" while leaving both flags set.
+    policy = RequestPolicy(
+        clarification_reason="none",
+        allow_missing_credentials_in_draft=True,
+        credential_draft_deferred_explicitly=True,
+    )
+    assert credential_prompt_reason(policy, None) == "credential_deferred_draft"
+
+
+def test_credential_prompt_reason_ignores_generic_skip_test_with_no_credential_signal() -> None:
+    # allow_missing_credentials_in_draft alone also fires for the generic skip_test
+    # fallthrough (any "draft only, don't test" turn), independent of credentials;
+    # only credential_draft_deferred_explicitly means a credential was really deferred.
+    policy = RequestPolicy(
+        testing_intent="skip_test",
+        clarification_reason="none",
+        allow_missing_credentials_in_draft=True,
+        credential_draft_deferred_explicitly=False,
+    )
+    assert credential_prompt_reason(policy, None) is None
+
+
+@pytest.mark.asyncio
+async def test_explicit_code_block_login_block_ban_surfaces_credential_prompt_end_to_end() -> None:
+    policy = await build_request_policy(
+        user_message="Write this as a code block. Do not create a login block for this part.",
+        workflow_yaml="",
+        chat_history=[],
+        global_llm_context="",
+        organization_id="o_test",
+        handler=None,
+    )
+    assert policy.credential_input_kind == "none"
+    assert policy.testing_intent == "skip_test"
+    assert credential_prompt_reason(policy, None) == "credential_deferred_draft"
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "Add it at https://app.skyvern.com/credentials.",
+        "ADD IT AT HTTPS://APP.SKYVERN.COM/CREDENTIALS.",
+        "Store it in the Credentials UI first.",
+        "store it in the CREDENTIALS UI first.",
+    ],
+)
+def test_credential_prompt_reason_marker_fallback_is_case_insensitive(text: str) -> None:
+    assert credential_prompt_reason(None, text) == "assistant_directed"
+
+
+def test_credential_prompt_reason_policy_none_is_safe() -> None:
+    assert credential_prompt_reason(None, None) is None
+    assert credential_prompt_reason(None, "Everything is set, no action needed.") is None
+
+
 @pytest.mark.asyncio
 async def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_log_only(
     monkeypatch: pytest.MonkeyPatch,