mirror of
https://github.com/Skyvern-AI/skyvern.git
synced 2026-07-09 16:09:13 +00:00
fix(SKY-11533): honest turn-halt for no-forward-progress interaction-click churn (#6869)
Some checks are pending
Run tests and pre-commit / Run tests and pre-commit hooks (push) Waiting to run
Run tests and pre-commit / Frontend Lint and Build (push) Waiting to run
Run tests and pre-commit / pip Package Smoke Tests (3.11) (push) Waiting to run
Run tests and pre-commit / pip Package Smoke Tests (3.13) (push) Waiting to run
Publish Fern Docs / run (push) Waiting to run
Some checks are pending
Run tests and pre-commit / Run tests and pre-commit hooks (push) Waiting to run
Run tests and pre-commit / Frontend Lint and Build (push) Waiting to run
Run tests and pre-commit / pip Package Smoke Tests (3.11) (push) Waiting to run
Run tests and pre-commit / pip Package Smoke Tests (3.13) (push) Waiting to run
Publish Fern Docs / run (push) Waiting to run
This commit is contained in:
parent
89ffbccbdb
commit
d30575bb9a
12 changed files with 413 additions and 4 deletions
|
|
@ -391,6 +391,10 @@ _LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {
|
|||
"I kept rewriting the generated code, but the safety checks rejected each version.",
|
||||
"Tell me what to change and I'll try a different approach.",
|
||||
),
|
||||
"loop_detected_no_forward_progress_interaction": (
|
||||
"I couldn't get past this step.",
|
||||
"Tell me what to change and I'll try a different approach.",
|
||||
),
|
||||
}
|
||||
_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."
|
||||
_LOOP_RESULT_COMPOSITION_BLOCKER_COPY = (
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from agents.run import Runner
|
|||
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
|
||||
from skyvern.forge.sdk.copilot.blocker_signal import (
|
||||
CopilotToolBlockerSignal,
|
||||
clear_tool_blocker_signals_for_reason_codes,
|
||||
compose_loop_blocker_user_facing_reason,
|
||||
loop_blocker_evidence_from_ctx,
|
||||
stash_blocker_signal,
|
||||
|
|
@ -83,6 +84,7 @@ from skyvern.forge.sdk.copilot.terminal_predicates import (
|
|||
)
|
||||
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
|
||||
from skyvern.forge.sdk.copilot.turn_halt import (
|
||||
blocker_signal_is_genuinely_terminal,
|
||||
raise_if_turn_halt,
|
||||
stash_repair_ceiling_turn_halt,
|
||||
stash_turn_halt_from_blocker_signal,
|
||||
|
|
@ -142,6 +144,9 @@ MAX_CODE_AUTHORING_GUARDRAIL_REJECTS = 4
|
|||
# higher bound, which must stay above MAX_CODE_AUTHORING_GUARDRAIL_REJECTS so the
|
||||
# non-credential backstop and the low-count credential deferral are untouched.
|
||||
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS = 8
|
||||
# Far under the SDK max-turns cap so the halt arrives first.
|
||||
MAX_NO_PROGRESS_INTERACTION_ATTEMPTS = 4
|
||||
_NO_PROGRESS_INTERACTION_REASON_CODES = frozenset({"loop_detected_no_forward_progress_interaction"})
|
||||
MIN_BLOCKS_FOR_AUTO_COMPLETE = 10
|
||||
TOTAL_TIMEOUT_SECONDS = 900
|
||||
# Belt-and-braces cap alongside the elapsed-time budget. Per-nudge caps
|
||||
|
|
@ -375,6 +380,54 @@ def credential_priority_authoring_churn_stop_signal(ctx: Any) -> CopilotToolBloc
|
|||
)
|
||||
|
||||
|
||||
def no_forward_progress_interaction_stop_signal(ctx: Any) -> CopilotToolBlockerSignal:
|
||||
count = _get_int(ctx, "consecutive_no_progress_interaction_count")
|
||||
evidence = loop_blocker_evidence_from_ctx(ctx)
|
||||
user_facing, _tiers = compose_loop_blocker_user_facing_reason(
|
||||
"loop_detected_no_forward_progress_interaction", evidence, blocked_tool="click"
|
||||
)
|
||||
agent_steering = (
|
||||
f"Clicking has not advanced the page across {count} attempts; "
|
||||
"stop trying new selectors and report the recorded blocker from the preserved draft."
|
||||
)
|
||||
return CopilotToolBlockerSignal(
|
||||
blocker_kind="loop_detected",
|
||||
agent_steering_text=agent_steering,
|
||||
user_facing_reason=user_facing,
|
||||
recovery_hint="report_blocker_to_user",
|
||||
cleared_by_tools=frozenset(),
|
||||
preserves_workflow_draft=evidence.has_draft,
|
||||
renders_final_reply=True,
|
||||
internal_reason_code="loop_detected_no_forward_progress_interaction",
|
||||
blocked_tool="click",
|
||||
)
|
||||
|
||||
|
||||
def _needs_no_progress_interaction_halt(ctx: Any) -> bool:
|
||||
return _get_int(ctx, "consecutive_no_progress_interaction_count") >= MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
|
||||
|
||||
|
||||
def reset_no_progress_interaction_count(ctx: Any) -> None:
|
||||
if _get_int(ctx, "consecutive_no_progress_interaction_count") == 0:
|
||||
return
|
||||
ctx.consecutive_no_progress_interaction_count = 0
|
||||
clear_tool_blocker_signals_for_reason_codes(ctx, _NO_PROGRESS_INTERACTION_REASON_CODES)
|
||||
LOG.info("copilot_no_progress_interaction_reset")
|
||||
|
||||
|
||||
def register_no_progress_interaction_click(ctx: Any, *, outcome: str) -> None:
|
||||
count = _get_int(ctx, "consecutive_no_progress_interaction_count") + 1
|
||||
ctx.consecutive_no_progress_interaction_count = count
|
||||
LOG.info("copilot_no_progress_interaction_click", outcome=outcome, count=count)
|
||||
if count < MAX_NO_PROGRESS_INTERACTION_ATTEMPTS:
|
||||
return
|
||||
if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
|
||||
return
|
||||
signal = no_forward_progress_interaction_stop_signal(ctx)
|
||||
stash_blocker_signal(ctx, signal)
|
||||
ctx.blocker_signal = signal
|
||||
|
||||
|
||||
def _needs_code_authoring_churn_halt(ctx: Any) -> bool:
|
||||
count = _get_int(ctx, "code_authoring_guardrail_reject_count")
|
||||
if getattr(ctx, "last_code_authoring_reject_was_credential_priority", False) is True:
|
||||
|
|
@ -1420,6 +1473,11 @@ def _check_enforcement(
|
|||
stash_blocker_signal(ctx, churn_signal)
|
||||
stash_turn_halt_from_blocker_signal(ctx, churn_signal, source="enforcement_backstop")
|
||||
raise_if_turn_halt(ctx)
|
||||
if _needs_no_progress_interaction_halt(ctx):
|
||||
no_progress_signal = no_forward_progress_interaction_stop_signal(ctx)
|
||||
stash_blocker_signal(ctx, no_progress_signal)
|
||||
stash_turn_halt_from_blocker_signal(ctx, no_progress_signal, source="enforcement_backstop")
|
||||
raise_if_turn_halt(ctx)
|
||||
|
||||
# Response-time gate: peek at the model's final output to tell ASK_QUESTION
|
||||
# (always allowed) from a REPLY with a coverage gap or progress-narration.
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
|
|||
stash_blocker_signal,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
|
||||
from skyvern.forge.sdk.copilot.enforcement import terminal_challenge_blocker_signal_from_current_page_evidence
|
||||
from skyvern.forge.sdk.copilot.enforcement import (
|
||||
register_no_progress_interaction_click,
|
||||
terminal_challenge_blocker_signal_from_current_page_evidence,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.loop_detection import (
|
||||
detect_failed_tool_step_loop_for_ctx,
|
||||
detect_tool_loop,
|
||||
|
|
@ -72,6 +75,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
|
|||
"reached_download_target",
|
||||
"synthesized_block_offered",
|
||||
"synthesized_block_offered_trajectory_len",
|
||||
"consecutive_no_progress_interaction_count",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -359,6 +363,8 @@ class SkyvernOverlayMCPServer(MCPServer):
|
|||
exc_info=True,
|
||||
)
|
||||
err = scrub_secrets_from_structure(copilot_ctx, {"ok": False, "error": f"{tool_name} failed: {e}"})
|
||||
if tool_name == "click":
|
||||
register_no_progress_interaction_click(copilot_ctx, outcome="click_failed")
|
||||
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, err)
|
||||
return _copilot_to_call_tool_result(err)
|
||||
|
||||
|
|
|
|||
|
|
@ -204,6 +204,10 @@ class AgentContext:
|
|||
repeated_failure_nudge_emitted_at_streak: int = 0
|
||||
code_authoring_guardrail_reject_count: int = 0
|
||||
last_code_authoring_reject_was_credential_priority: bool = False
|
||||
# Climbs on each click that made no verified forward progress (failed/timed-out
|
||||
# click or a hollow post-click observe); resets on verified progress.
|
||||
consecutive_no_progress_interaction_count: int = 0
|
||||
last_scout_act_observe_outcome: str | None = None
|
||||
pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = None
|
||||
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
|
||||
challenge_gated_proxy_retry_count: int = 0
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from .scouting import (
|
|||
_reset_evaluate_tracker,
|
||||
_resolve_scout_role_name,
|
||||
_steer_evaluate_result,
|
||||
account_no_progress_interaction_click,
|
||||
)
|
||||
|
||||
LOG = structlog.get_logger()
|
||||
|
|
@ -407,6 +408,7 @@ async def _click_post_hook(
|
|||
raw: dict[str, Any],
|
||||
ctx: AgentContext,
|
||||
) -> dict[str, Any]:
|
||||
ctx.last_scout_act_observe_outcome = None
|
||||
_clear_pending_browser_interaction_observation(ctx)
|
||||
source_url = _consume_scout_source_url(ctx)
|
||||
pending_role_name = ctx.pending_scout_role_name
|
||||
|
|
@ -444,6 +446,7 @@ async def _click_post_hook(
|
|||
_attach_scout_page_summary(result, page_evidence)
|
||||
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
|
||||
await _maybe_attach_reached_download_target(ctx, result, url=url, page_evidence=page_evidence)
|
||||
account_no_progress_interaction_click(ctx, result)
|
||||
return result
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
|
|||
RepairNextAction,
|
||||
build_diagnosis_repair_contract,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.enforcement import repair_ceiling_stop_signal
|
||||
from skyvern.forge.sdk.copilot.enforcement import repair_ceiling_stop_signal, reset_no_progress_interaction_count
|
||||
from skyvern.forge.sdk.copilot.failure_tracking import (
|
||||
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
|
||||
PER_TOOL_BUDGET_FAILURE_CATEGORY,
|
||||
|
|
@ -2396,6 +2396,8 @@ def _update_repair_loop_state(copilot_ctx: Any, contract: DiagnosisRepairContrac
|
|||
copilot_ctx.verified_criteria_high_water = high_water | current
|
||||
copilot_ctx.verified_prefix_high_water_len = max(prefix_high, prefix_len)
|
||||
copilot_ctx.verified_full_pass_consumed = full_pass
|
||||
if progressed:
|
||||
reset_no_progress_interaction_count(copilot_ctx)
|
||||
|
||||
signature = _repair_non_convergence_signature(contract)
|
||||
if signature is None or progressed:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,11 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
|
|||
has_bounded_page_schema,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
|
||||
from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
|
||||
from skyvern.forge.sdk.copilot.enforcement import (
|
||||
_RECENT_TOOL_OUTPUT_CHAR_CAP,
|
||||
register_no_progress_interaction_click,
|
||||
reset_no_progress_interaction_count,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.reached_download_target import (
|
||||
ReachedDownloadTarget,
|
||||
)
|
||||
|
|
@ -90,6 +94,7 @@ def _consume_pending_browser_interaction_observation(
|
|||
current_url=current_url,
|
||||
)
|
||||
return False
|
||||
reset_no_progress_interaction_count(ctx)
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -349,6 +354,7 @@ async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> di
|
|||
except Exception:
|
||||
parsed = None
|
||||
outcome = "error"
|
||||
ctx.last_scout_act_observe_outcome = outcome
|
||||
LOG.info(
|
||||
"copilot_scout_act_observe",
|
||||
outcome=outcome,
|
||||
|
|
@ -391,6 +397,21 @@ async def _register_scout_interaction_observation(
|
|||
return step, page_evidence
|
||||
|
||||
|
||||
def account_no_progress_interaction_click(ctx: AgentContext, result: dict[str, Any]) -> None:
|
||||
"""Climb or reset the no-forward-progress counter from a click's outcome: a failed click or hollow
|
||||
observe is no progress, an attached observe is progress, a capture timeout/error is neutral."""
|
||||
if not result.get("ok"):
|
||||
register_no_progress_interaction_click(ctx, outcome="click_failed")
|
||||
return
|
||||
outcome = ctx.last_scout_act_observe_outcome
|
||||
if outcome == "attached":
|
||||
reset_no_progress_interaction_count(ctx)
|
||||
elif outcome == "hollow":
|
||||
register_no_progress_interaction_click(ctx, outcome="hollow")
|
||||
else:
|
||||
LOG.info("copilot_no_progress_interaction_neutral", outcome=outcome)
|
||||
|
||||
|
||||
_PAGE_SUMMARY_TEXT_CAP = 80
|
||||
_PAGE_SUMMARY_MAX_FIELDS = 8
|
||||
_PAGE_SUMMARY_MAX_SUBMITS = 4
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ _LOOP_TERMINAL_REASON_CODES = frozenset(
|
|||
"loop_detected_generic",
|
||||
"code_authoring_guardrail_churn",
|
||||
"credential_priority_authoring_churn",
|
||||
"loop_detected_no_forward_progress_interaction",
|
||||
}
|
||||
)
|
||||
_ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset(
|
||||
|
|
|
|||
|
|
@ -533,6 +533,23 @@ def test_turn_halt_exit_renders_code_authoring_churn_reason() -> None:
|
|||
assert "update_workflow" not in result.user_response
|
||||
|
||||
|
||||
def test_turn_halt_exit_renders_no_forward_progress_interaction_reason() -> None:
|
||||
ctx = _ctx()
|
||||
signal = _signal(
|
||||
kind="loop_detected",
|
||||
user_facing="I couldn't get past this step. Tell me what to change and I'll try a different approach.",
|
||||
internal_reason_code="loop_detected_no_forward_progress_interaction",
|
||||
blocked_tool="click",
|
||||
)
|
||||
ctx.blocker_signal = signal
|
||||
halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)
|
||||
|
||||
result = _build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
|
||||
|
||||
assert "couldn't get past this step" in result.user_response
|
||||
assert "click" not in result.user_response
|
||||
|
||||
|
||||
def test_turn_halt_exit_keeps_credential_scout_reply_through_refresh_with_draft() -> None:
|
||||
ctx = _ctx()
|
||||
ctx.last_workflow_yaml = "title: Draft\nworkflow_definition:\n blocks: []\n"
|
||||
|
|
|
|||
|
|
@ -22,7 +22,10 @@ from typing import Any
|
|||
|
||||
import pytest
|
||||
|
||||
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
|
||||
from skyvern.forge.sdk.copilot.blocker_signal import (
|
||||
CopilotToolBlockerSignal,
|
||||
maybe_clear_blocker_signal_on_tool_success,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.context import CopilotContext
|
||||
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
|
||||
DiagnosisInput,
|
||||
|
|
@ -35,6 +38,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
|
|||
from skyvern.forge.sdk.copilot.enforcement import (
|
||||
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
|
||||
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
|
||||
MAX_NO_PROGRESS_INTERACTION_ATTEMPTS,
|
||||
MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES,
|
||||
POST_FAILED_TEST_INSPECT_FIRST_NUDGE,
|
||||
POST_FAILED_TEST_NUDGE,
|
||||
|
|
@ -52,6 +56,8 @@ from skyvern.forge.sdk.copilot.enforcement import (
|
|||
_prune_input_list,
|
||||
_recover_from_context_overflow,
|
||||
_strip_input_images,
|
||||
register_no_progress_interaction_click,
|
||||
reset_no_progress_interaction_count,
|
||||
)
|
||||
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
|
||||
from skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_tool
|
||||
|
|
@ -511,3 +517,146 @@ def test_credential_priority_churn_defers_to_repair_ceiling() -> None:
|
|||
signal = ctx.blocker_signal
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == "repair_ceiling_reached"
|
||||
|
||||
|
||||
def test_no_progress_interaction_floor_raises_at_ceiling() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
|
||||
|
||||
with pytest.raises(CopilotTurnHalt) as excinfo:
|
||||
_check_enforcement(ctx)
|
||||
|
||||
assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
|
||||
signal = ctx.blocker_signal
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"
|
||||
assert signal.renders_final_reply is True
|
||||
assert signal.recovery_hint == "report_blocker_to_user"
|
||||
|
||||
|
||||
def test_no_progress_interaction_floor_does_not_raise_below_ceiling() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1
|
||||
|
||||
assert _check_enforcement(ctx) is None
|
||||
assert ctx.blocker_signal is None
|
||||
|
||||
|
||||
def test_no_progress_interaction_floor_defers_to_repair_ceiling() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
|
||||
ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()
|
||||
|
||||
with pytest.raises(CopilotTurnHalt) as excinfo:
|
||||
_check_enforcement(ctx)
|
||||
|
||||
assert excinfo.value.halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
|
||||
signal = ctx.blocker_signal
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == "repair_ceiling_reached"
|
||||
|
||||
|
||||
def test_no_progress_interaction_floor_yields_to_non_retriable_nav_error() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
|
||||
ctx.last_test_ok = False
|
||||
ctx.last_test_non_retriable_nav_error = (
|
||||
"Failed to navigate to url https://does-not-resolve.example. Error message: net::ERR_NAME_NOT_RESOLVED"
|
||||
)
|
||||
|
||||
nudge = _check_enforcement(ctx)
|
||||
|
||||
assert nudge is not None
|
||||
assert ctx.blocker_signal is None
|
||||
with pytest.raises(CopilotNonRetriableNavError):
|
||||
_maybe_raise_non_retriable_nav(ctx)
|
||||
|
||||
|
||||
def test_register_no_progress_interaction_click_stashes_blocker_at_cap() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1
|
||||
|
||||
register_no_progress_interaction_click(ctx, outcome="click_failed")
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
|
||||
signal = ctx.blocker_signal
|
||||
assert isinstance(signal, CopilotToolBlockerSignal)
|
||||
assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"
|
||||
|
||||
|
||||
def test_register_no_progress_interaction_click_below_cap_does_not_stash() -> None:
|
||||
ctx = _fresh_context()
|
||||
|
||||
register_no_progress_interaction_click(ctx, outcome="hollow")
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == 1
|
||||
assert ctx.blocker_signal is None
|
||||
|
||||
|
||||
def test_register_no_progress_interaction_click_defers_to_terminal_held_blocker() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1
|
||||
terminal = CopilotToolBlockerSignal(
|
||||
blocker_kind="loop_detected",
|
||||
agent_steering_text="The repair made no progress.",
|
||||
user_facing_reason="I couldn't get past the same problem after several attempts.",
|
||||
recovery_hint="report_blocker_to_user",
|
||||
internal_reason_code="repair_ceiling_reached",
|
||||
blocked_tool="update_and_run_blocks",
|
||||
)
|
||||
ctx.blocker_signal = terminal
|
||||
|
||||
register_no_progress_interaction_click(ctx, outcome="click_failed")
|
||||
|
||||
assert ctx.blocker_signal is terminal
|
||||
|
||||
|
||||
def test_reset_no_progress_interaction_count_clears_counter() -> None:
|
||||
ctx = _fresh_context()
|
||||
ctx.consecutive_no_progress_interaction_count = 3
|
||||
|
||||
reset_no_progress_interaction_count(ctx)
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == 0
|
||||
|
||||
|
||||
def _hit_no_progress_cap(ctx: CopilotContext) -> CopilotToolBlockerSignal:
|
||||
for _ in range(MAX_NO_PROGRESS_INTERACTION_ATTEMPTS):
|
||||
register_no_progress_interaction_click(ctx, outcome="hollow")
|
||||
held = ctx.blocker_signal
|
||||
assert isinstance(held, CopilotToolBlockerSignal)
|
||||
assert held.internal_reason_code == "loop_detected_no_forward_progress_interaction"
|
||||
assert held.renders_final_reply is True
|
||||
return held
|
||||
|
||||
|
||||
def test_no_progress_reset_clears_held_blocker_after_cap() -> None:
|
||||
ctx = _fresh_context()
|
||||
_hit_no_progress_cap(ctx)
|
||||
|
||||
reset_no_progress_interaction_count(ctx)
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == 0
|
||||
assert ctx.blocker_signal is None
|
||||
assert ctx.latest_tool_blocker_signal is None
|
||||
|
||||
|
||||
def test_no_progress_reset_at_progress_seam_stops_re_halt() -> None:
|
||||
ctx = _fresh_context()
|
||||
_hit_no_progress_cap(ctx)
|
||||
|
||||
reset_no_progress_interaction_count(ctx)
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == 0
|
||||
assert ctx.blocker_signal is None
|
||||
assert _check_enforcement(ctx) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("recovery_tool", ["evaluate", "inspect_page_for_composition"])
|
||||
def test_no_progress_held_blocker_survives_progress_tool_success(recovery_tool: str) -> None:
|
||||
ctx = _fresh_context()
|
||||
held = _hit_no_progress_cap(ctx)
|
||||
|
||||
maybe_clear_blocker_signal_on_tool_success(ctx, recovery_tool)
|
||||
|
||||
assert ctx.blocker_signal is held
|
||||
|
|
|
|||
64
tests/unit/test_copilot_mcp_adapter_no_progress.py
Normal file
64
tests/unit/test_copilot_mcp_adapter_no_progress.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
"""A raised ``client.call_tool`` for the click tool returns before the post-hook, so the shared
|
||||
no-progress helper must fire from the adapter exception handler exactly once, with no double-count
|
||||
against the post-hook path. OSS-synced fixture references use example.* only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, NoReturn
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer
|
||||
from skyvern.forge.sdk.copilot.runtime import AgentContext
|
||||
|
||||
|
||||
class _RaisingClient:
|
||||
async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> NoReturn:
|
||||
raise RuntimeError("Timeout 5000ms exceeded")
|
||||
|
||||
|
||||
def _agent_ctx() -> AgentContext:
|
||||
return AgentContext(
|
||||
organization_id="o_1",
|
||||
workflow_id="w_1",
|
||||
workflow_permanent_id="wpid_1",
|
||||
workflow_yaml="",
|
||||
browser_session_id="pbs_1",
|
||||
stream=MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
def _make_server(ctx: AgentContext, tool_name: str) -> SkyvernOverlayMCPServer:
|
||||
server = SkyvernOverlayMCPServer(
|
||||
transport=MagicMock(),
|
||||
overlays={tool_name: SchemaOverlay()},
|
||||
alias_map={},
|
||||
allowlist=frozenset(),
|
||||
context_provider=lambda: ctx,
|
||||
)
|
||||
server._client = _RaisingClient()
|
||||
return server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raised_click_increments_no_progress_counter_exactly_once() -> None:
|
||||
ctx = _agent_ctx()
|
||||
server = _make_server(ctx, "click")
|
||||
|
||||
result = await server.call_tool("click", {"selector": "#submit"})
|
||||
|
||||
assert result.isError is True
|
||||
assert ctx.consecutive_no_progress_interaction_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raised_non_click_tool_leaves_no_progress_counter_untouched() -> None:
|
||||
ctx = _agent_ctx()
|
||||
server = _make_server(ctx, "evaluate")
|
||||
|
||||
result = await server.call_tool("evaluate", {"expression": "scan()"})
|
||||
|
||||
assert result.isError is True
|
||||
assert ctx.consecutive_no_progress_interaction_count == 0
|
||||
|
|
@ -23,6 +23,8 @@ from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
|
|||
from skyvern.forge.sdk.copilot.tools import _click_post_hook
|
||||
from skyvern.forge.sdk.copilot.tools.scouting import (
|
||||
_consume_pending_browser_interaction_observation,
|
||||
_mark_pending_browser_interaction_observation,
|
||||
account_no_progress_interaction_click,
|
||||
)
|
||||
|
||||
_SOURCE_URL = "https://example.com/product"
|
||||
|
|
@ -348,3 +350,81 @@ class TestActObserveToolGate:
|
|||
assert result["ok"] is True
|
||||
assert "page" not in result["data"]
|
||||
assert ctx.flow_evidence[0]["had_bounded_schema"] is False
|
||||
|
||||
|
||||
def _np_ctx(*, server: Any = None, counter: int = 0) -> SimpleNamespace:
|
||||
ctx = _ctx(server=server)
|
||||
ctx.consecutive_no_progress_interaction_count = counter
|
||||
ctx.last_scout_act_observe_outcome = None
|
||||
ctx.blocker_signal = None
|
||||
return ctx
|
||||
|
||||
|
||||
class TestNoProgressInteractionAccounting:
|
||||
@pytest.mark.asyncio
|
||||
async def test_hollow_click_increments_counter(self) -> None:
|
||||
ctx = _np_ctx(server=_server_returning({"page_title": "Loading", "forms": []}))
|
||||
|
||||
await _run_click(ctx)
|
||||
|
||||
assert ctx.last_scout_act_observe_outcome == "hollow"
|
||||
assert ctx.consecutive_no_progress_interaction_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_attached_click_resets_counter(self) -> None:
|
||||
ctx = _np_ctx(server=_server_returning(_bounded_extractor_payload()), counter=2)
|
||||
|
||||
await _run_click(ctx)
|
||||
|
||||
assert ctx.last_scout_act_observe_outcome == "attached"
|
||||
assert ctx.consecutive_no_progress_interaction_count == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_click_increments_counter(self) -> None:
|
||||
ctx = _np_ctx()
|
||||
|
||||
await _click_post_hook(
|
||||
{"ok": False, "error": "Timeout 5000ms exceeded"},
|
||||
{"browser_context": {"url": _LANDING_URL}},
|
||||
ctx,
|
||||
)
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_timeout_observe_is_neutral(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS", 0.05)
|
||||
|
||||
async def slow_extract(*_args: object, **_kwargs: object) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.25)
|
||||
return {"ok": True, "data": {"result": _bounded_extractor_payload()}}
|
||||
|
||||
server = SimpleNamespace()
|
||||
server.call_internal_tool = AsyncMock(side_effect=slow_extract)
|
||||
ctx = _np_ctx(server=server, counter=2)
|
||||
|
||||
await _run_click(ctx)
|
||||
|
||||
assert ctx.last_scout_act_observe_outcome == "timeout"
|
||||
assert ctx.consecutive_no_progress_interaction_count == 2
|
||||
|
||||
def test_neutral_outcome_does_not_touch_counter(self) -> None:
|
||||
ctx = _np_ctx(counter=2)
|
||||
ctx.last_scout_act_observe_outcome = None
|
||||
|
||||
account_no_progress_interaction_click(ctx, {"ok": True, "data": {"selector": "#x"}})
|
||||
|
||||
assert ctx.consecutive_no_progress_interaction_count == 2
|
||||
|
||||
def test_delayed_credit_via_consume_resets_counter(self) -> None:
|
||||
ctx = _np_ctx(counter=3)
|
||||
_mark_pending_browser_interaction_observation(ctx, tool_name="click", url=_LANDING_URL)
|
||||
|
||||
consumed = _consume_pending_browser_interaction_observation(
|
||||
ctx,
|
||||
current_url=_LANDING_URL,
|
||||
evidence={**_bounded_extractor_payload(), "current_url": _LANDING_URL},
|
||||
)
|
||||
|
||||
assert consumed is True
|
||||
assert ctx.consecutive_no_progress_interaction_count == 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue