SKY-10823: nudge entry-point discovery before a pre-discovery 'provide the URL' ask (#6473)

skyvern/forge/sdk/copilot/config.py

@@ -271,6 +271,14 @@ POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE = (
     "and only when a separate required non-URL input is still missing."
 )
 
+PRE_DISCOVERY_URL_QUESTION_NUDGE = (
+    "STOP — you are asking the user for an entry-point URL before resolving it yourself. "
+    "discover_workflow_entrypoint has not run this turn. Call "
+    "discover_workflow_entrypoint(site_or_url, intent_hint) to resolve the entrypoint from the "
+    "site the user named, then compose from the resolved page. Only ask for a URL if discovery "
+    "runs and cannot resolve a site."
+)
+
 PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = (
     "STOP — the target site has failed to scrape on every attempt across "
     "multiple workflow shapes. Every run navigated successfully but the "
@@ -346,6 +354,7 @@ DEFAULT_ENFORCEMENT_NUDGES: dict[str, str] = {
     "post_per_tool_budget_stop": POST_PER_TOOL_BUDGET_STOP_NUDGE,
     "post_no_workflow_delivery": POST_NO_WORKFLOW_DELIVERY_NUDGE,
     "post_discovery_entrypoint_url_question": POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE,
+    "pre_discovery_url_question": PRE_DISCOVERY_URL_QUESTION_NUDGE,
     "post_probable_site_block_stop_prefix": PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX,
     "post_probable_site_block_stop": POST_PROBABLE_SITE_BLOCK_STOP_NUDGE,
     "post_anti_bot_failed_test": POST_ANTI_BOT_FAILED_TEST_NUDGE,

skyvern/forge/sdk/copilot/context.py

@@ -486,6 +486,7 @@ class CopilotContext(AgentContext):
     resolved_discovery_failure_reason: str | None = None
     resolved_discovery_entrypoint_inspection_baseline: int = 0
     discovery_entrypoint_url_question_nudge_count: int = 0
+    pre_discovery_url_question_nudge_count: int = 0
     # Set in `_run_attempt` after SkyvernOverlayMCPServer is constructed.
     # The discovery tool reaches the connected FastMCP client through this.
     discovery_mcp_server: Any | None = None

skyvern/forge/sdk/copilot/enforcement.py

@@ -15,6 +15,7 @@ from agents.run import Runner
 
 from skyvern.config import settings
 from skyvern.forge.sdk.copilot import config as copilot_config_defaults
+from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
 from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block
 from skyvern.forge.sdk.copilot.config import (
     DEFAULT_ENFORCEMENT_NUDGES,
@@ -37,6 +38,7 @@ from skyvern.forge.sdk.copilot.config import (
     POST_REPEATED_NULL_DATA_NUDGE,
     POST_SUSPICIOUS_SUCCESS_NUDGE,
     POST_UPDATE_NUDGE,
+    PRE_DISCOVERY_URL_QUESTION_NUDGE,
     SCREENSHOT_DROPPED_NUDGE,
     BlockAuthoringPolicy,
     CopilotConfig,
@@ -74,6 +76,7 @@ MAX_FAILED_TEST_NUDGES = 2
 MAX_FORMAT_NUDGES = 2
 MAX_NO_WORKFLOW_NUDGES = 2
 MAX_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGES = 2
+MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES = 2
 MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2
 # Stops the suspicious-success nudge from re-firing forever when the agent has
 # correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no
@@ -572,6 +575,46 @@ def _has_candidate_bound_page_evidence(ctx: Any, candidate_url: str) -> bool:
     return False
 
 
+def _pre_discovery_url_question_nudge(
+    ctx: Any,
+    parsed: dict[str, Any],
+    config: CopilotConfig | None = None,
+) -> str | None:
+    """Steer the model to discovery when it asks before discovery has run.
+
+    INITIAL/DISCOVERING phase with zero discovery calls means the model went
+    straight to asking instead of resolving the entrypoint itself. Credential,
+    loop, and conditional clarifications carry a non-default
+    request_policy.clarification_reason and are let through; the structural
+    triple (phase + zero discovery calls + default clarification_reason) already
+    excludes them. The post-discovery could-not-resolve ask happens after
+    discovery ran (discovery_calls_this_turn > 0) and so never reaches this gate.
+    Steering any remaining pre-discovery ASK to discovery is correct: discovery
+    is cheap, and if the site cannot resolve the model re-asks afterward.
+    """
+    if parsed.get("type") != "ASK_QUESTION":
+        return None
+    if getattr(ctx, "build_phase", None) not in DISCOVERY_PERMITTED_PHASES:
+        return None
+    if _get_int(ctx, "discovery_calls_this_turn") != 0:
+        return None
+    request_policy = getattr(ctx, "request_policy", None)
+    clarification_reason = getattr(request_policy, "clarification_reason", "none")
+    if clarification_reason not in (None, "none"):
+        return None
+    nudge_count = _get_int(ctx, "pre_discovery_url_question_nudge_count")
+    if nudge_count >= MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES:
+        return None
+    ctx.pre_discovery_url_question_nudge_count = nudge_count + 1
+    LOG.info(
+        "copilot.pre_discovery_url_question_nudge",
+        reason_code="pre_discovery_url_question_steer_to_discovery",
+        build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),
+        nudge_count=ctx.pre_discovery_url_question_nudge_count,
+    )
+    return _nudge(config, "pre_discovery_url_question")
+
+
 def _post_discovery_entrypoint_url_question_nudge(
     ctx: Any,
     parsed: dict[str, Any],
@@ -604,6 +647,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCo
     Returns the nudge string to inject, or None to let the response through.
     """
     response_type = parsed.get("type")
+    pre_discovery_nudge = _pre_discovery_url_question_nudge(ctx, parsed, config)
+    if pre_discovery_nudge is not None:
+        return pre_discovery_nudge
+
     discovery_entrypoint_nudge = _post_discovery_entrypoint_url_question_nudge(ctx, parsed, config)
     if discovery_entrypoint_nudge is not None:
         return discovery_entrypoint_nudge
@@ -1249,6 +1296,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {
     POST_PER_TOOL_BUDGET_STOP_NUDGE: "per_tool_budget_stop",
     POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",
     POST_DISCOVERY_ENTRYPOINT_URL_QUESTION_NUDGE: "discovery_entrypoint_url_question",
+    PRE_DISCOVERY_URL_QUESTION_NUDGE: "pre_discovery_url_question",
     POST_FAILED_TEST_NUDGE: "post_failed_test",
     POST_FAILED_TEST_INSPECT_FIRST_NUDGE: "post_failed_test_inspect_first",
     SCREENSHOT_DROPPED_NUDGE: "screenshot_dropped_on_recovery",
@@ -1273,6 +1321,7 @@ _NUDGE_TYPE_BY_KEY: dict[str, str] = {
     "post_per_tool_budget_stop": "per_tool_budget_stop",
     "post_no_workflow_delivery": "no_workflow_delivery",
     "post_discovery_entrypoint_url_question": "discovery_entrypoint_url_question",
+    "pre_discovery_url_question": "pre_discovery_url_question",
     "post_failed_test": "post_failed_test",
     "post_failed_test_inspect_first": "post_failed_test_inspect_first",
     "screenshot_dropped": "screenshot_dropped_on_recovery",

tests/unit/test_copilot_pre_discovery_url_nudge.py

@@ -0,0 +1,144 @@
+"""Tests for the pre-discovery entry-point-URL ASK gate in _check_enforcement.
+
+A fresh BUILD turn that names a site but no exact URL enters BuildPhase.INITIAL.
+The model is supposed to call discover_workflow_entrypoint, but at temperature=1
+it occasionally samples straight to an ASK_QUESTION demanding the exact URL. The
+gate fires on the structural triple (phase in DISCOVERY_PERMITTED_PHASES,
+discovery_calls_this_turn == 0, default clarification_reason) with no text
+matching, steering the model to discovery while leaving credential / loop /
+conditional clarifications and the post-discovery could-not-resolve ask
+untouched.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.enforcement import (
+    MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES,
+    PRE_DISCOVERY_URL_QUESTION_NUDGE,
+    _pre_discovery_url_question_nudge,
+    _response_coverage_nudge,
+)
+
+
+class _Ctx:
+    """Minimal stand-in for CopilotContext used in the pre-discovery gate."""
+
+    def __init__(self) -> None:
+        self.build_phase = BuildPhase.INITIAL
+        self.discovery_calls_this_turn = 0
+        self.pre_discovery_url_question_nudge_count = 0
+        self.request_policy = None
+        # Fields the post-discovery sibling reads when _response_coverage_nudge runs.
+        self.resolved_discovery_entrypoint_url = None
+        self.resolved_discovery_failure_reason = None
+        self.update_workflow_called = False
+        self.no_workflow_nudge_count = 0
+        self.coverage_nudge_count = 0
+        self.format_nudge_count = 0
+
+
+_URL_ASK = {
+    "type": "ASK_QUESTION",
+    "user_response": "I can build that, but I need the exact URL to use as the entry point. Please provide the URL.",
+}
+
+
+def test_pre_discovery_url_ask_in_initial_phase_steers_to_discovery() -> None:
+    ctx = _Ctx()
+    nudge = _pre_discovery_url_question_nudge(ctx, _URL_ASK)
+    assert nudge == PRE_DISCOVERY_URL_QUESTION_NUDGE
+    assert ctx.pre_discovery_url_question_nudge_count == 1
+
+
+def test_pre_discovery_url_ask_fires_through_response_coverage_gate() -> None:
+    ctx = _Ctx()
+    assert _response_coverage_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+
+
+def test_pre_discovery_url_ask_in_discovering_phase_steers_to_discovery() -> None:
+    ctx = _Ctx()
+    ctx.build_phase = BuildPhase.DISCOVERING
+    assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+
+
+def test_no_nudge_after_discovery_ran_this_turn() -> None:
+    ctx = _Ctx()
+    ctx.discovery_calls_this_turn = 1
+    assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+
+
+def test_no_nudge_outside_initial_or_discovering_phase() -> None:
+    ctx = _Ctx()
+    ctx.build_phase = BuildPhase.COMPOSING
+    assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+
+
+def test_no_nudge_for_non_ask_question() -> None:
+    ctx = _Ctx()
+    reply = {"type": "REPLY", "user_response": "Please provide the URL to start from."}
+    assert _pre_discovery_url_question_nudge(ctx, reply) is None
+
+
+# ---------------------------------------------------------------------------
+# No over-suppression — legitimate INITIAL asks are let through.
+# ---------------------------------------------------------------------------
+
+
+def test_credential_clarification_passes_through() -> None:
+    ctx = _Ctx()
+    ctx.request_policy = SimpleNamespace(clarification_reason="credential_name_unresolved")
+    ask = {
+        "type": "ASK_QUESTION",
+        "user_response": "Which saved credential should I use for the login page?",
+    }
+    assert _pre_discovery_url_question_nudge(ctx, ask) is None
+
+
+def test_loop_clarification_passes_through() -> None:
+    ctx = _Ctx()
+    ctx.request_policy = SimpleNamespace(clarification_reason="ambiguous_loop_edit")
+    ask = {"type": "ASK_QUESTION", "user_response": "Which loop block on the page should I edit?"}
+    assert _pre_discovery_url_question_nudge(ctx, ask) is None
+
+
+def test_conditional_clarification_passes_through() -> None:
+    ctx = _Ctx()
+    ctx.request_policy = SimpleNamespace(clarification_reason="missing_conditional_condition")
+    ask = {"type": "ASK_QUESTION", "user_response": "What condition should gate this site visit?"}
+    assert _pre_discovery_url_question_nudge(ctx, ask) is None
+
+
+def test_post_discovery_could_not_resolve_site_name_ask_passes_through() -> None:
+    # Discovery ran and failed to resolve a site → discovery_calls_this_turn > 0,
+    # so the genuine "which URL?" ask is let through to the user.
+    ctx = _Ctx()
+    ctx.discovery_calls_this_turn = 1
+    ctx.resolved_discovery_failure_reason = "could_not_resolve_site_name"
+    assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+
+
+def test_any_pre_discovery_ask_on_structural_triple_steers_to_discovery() -> None:
+    # No text matching: any ASK_QUESTION on the structural triple (INITIAL/0
+    # discovery / default clarification_reason) is steered to discovery, even
+    # when the phrasing names neither a URL nor a site.
+    ctx = _Ctx()
+    ask = {"type": "ASK_QUESTION", "user_response": "Where should the agent begin?"}
+    assert _pre_discovery_url_question_nudge(ctx, ask) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+    assert ctx.pre_discovery_url_question_nudge_count == 1
+
+
+# ---------------------------------------------------------------------------
+# Bounded — the (N+1)th ask is let through.
+# ---------------------------------------------------------------------------
+
+
+def test_pre_discovery_nudge_is_bounded() -> None:
+    ctx = _Ctx()
+    for expected_count in range(1, MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES + 1):
+        assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) == PRE_DISCOVERY_URL_QUESTION_NUDGE
+        assert ctx.pre_discovery_url_question_nudge_count == expected_count
+    assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
+    assert ctx.pre_discovery_url_question_nudge_count == MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES