fix(SKY-11533): honest turn-halt for no-forward-progress interaction-click churn (#6869)
Some checks are pending
Run tests and pre-commit / Run tests and pre-commit hooks (push) Waiting to run
Run tests and pre-commit / Frontend Lint and Build (push) Waiting to run
Run tests and pre-commit / pip Package Smoke Tests (3.11) (push) Waiting to run
Run tests and pre-commit / pip Package Smoke Tests (3.13) (push) Waiting to run
Publish Fern Docs / run (push) Waiting to run

This commit is contained in:
Andrew Neilson 2026-06-27 12:52:57 -07:00 committed by GitHub
parent 89ffbccbdb
commit d30575bb9a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 413 additions and 4 deletions

View file

@ -391,6 +391,10 @@ _LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {
"I kept rewriting the generated code, but the safety checks rejected each version.",
"Tell me what to change and I'll try a different approach.",
),
"loop_detected_no_forward_progress_interaction": (
"I couldn't get past this step.",
"Tell me what to change and I'll try a different approach.",
),
}
_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."
_LOOP_RESULT_COMPOSITION_BLOCKER_COPY = (

View file

@ -16,6 +16,7 @@ from agents.run import Runner
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
clear_tool_blocker_signals_for_reason_codes,
compose_loop_blocker_user_facing_reason,
loop_blocker_evidence_from_ctx,
stash_blocker_signal,
@ -83,6 +84,7 @@ from skyvern.forge.sdk.copilot.terminal_predicates import (
)
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import (
blocker_signal_is_genuinely_terminal,
raise_if_turn_halt,
stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
@ -142,6 +144,9 @@ MAX_CODE_AUTHORING_GUARDRAIL_REJECTS = 4
# higher bound, which must stay above MAX_CODE_AUTHORING_GUARDRAIL_REJECTS so the
# non-credential backstop and the low-count credential deferral are untouched.
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS = 8
# Far under the SDK max-turns cap so the halt arrives first.
MAX_NO_PROGRESS_INTERACTION_ATTEMPTS = 4
_NO_PROGRESS_INTERACTION_REASON_CODES = frozenset({"loop_detected_no_forward_progress_interaction"})
MIN_BLOCKS_FOR_AUTO_COMPLETE = 10
TOTAL_TIMEOUT_SECONDS = 900
# Belt-and-braces cap alongside the elapsed-time budget. Per-nudge caps
@ -375,6 +380,54 @@ def credential_priority_authoring_churn_stop_signal(ctx: Any) -> CopilotToolBloc
)
def no_forward_progress_interaction_stop_signal(ctx: Any) -> CopilotToolBlockerSignal:
count = _get_int(ctx, "consecutive_no_progress_interaction_count")
evidence = loop_blocker_evidence_from_ctx(ctx)
user_facing, _tiers = compose_loop_blocker_user_facing_reason(
"loop_detected_no_forward_progress_interaction", evidence, blocked_tool="click"
)
agent_steering = (
f"Clicking has not advanced the page across {count} attempts; "
"stop trying new selectors and report the recorded blocker from the preserved draft."
)
return CopilotToolBlockerSignal(
blocker_kind="loop_detected",
agent_steering_text=agent_steering,
user_facing_reason=user_facing,
recovery_hint="report_blocker_to_user",
cleared_by_tools=frozenset(),
preserves_workflow_draft=evidence.has_draft,
renders_final_reply=True,
internal_reason_code="loop_detected_no_forward_progress_interaction",
blocked_tool="click",
)
def _needs_no_progress_interaction_halt(ctx: Any) -> bool:
return _get_int(ctx, "consecutive_no_progress_interaction_count") >= MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
def reset_no_progress_interaction_count(ctx: Any) -> None:
if _get_int(ctx, "consecutive_no_progress_interaction_count") == 0:
return
ctx.consecutive_no_progress_interaction_count = 0
clear_tool_blocker_signals_for_reason_codes(ctx, _NO_PROGRESS_INTERACTION_REASON_CODES)
LOG.info("copilot_no_progress_interaction_reset")
def register_no_progress_interaction_click(ctx: Any, *, outcome: str) -> None:
count = _get_int(ctx, "consecutive_no_progress_interaction_count") + 1
ctx.consecutive_no_progress_interaction_count = count
LOG.info("copilot_no_progress_interaction_click", outcome=outcome, count=count)
if count < MAX_NO_PROGRESS_INTERACTION_ATTEMPTS:
return
if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
return
signal = no_forward_progress_interaction_stop_signal(ctx)
stash_blocker_signal(ctx, signal)
ctx.blocker_signal = signal
def _needs_code_authoring_churn_halt(ctx: Any) -> bool:
count = _get_int(ctx, "code_authoring_guardrail_reject_count")
if getattr(ctx, "last_code_authoring_reject_was_credential_priority", False) is True:
@ -1420,6 +1473,11 @@ def _check_enforcement(
stash_blocker_signal(ctx, churn_signal)
stash_turn_halt_from_blocker_signal(ctx, churn_signal, source="enforcement_backstop")
raise_if_turn_halt(ctx)
if _needs_no_progress_interaction_halt(ctx):
no_progress_signal = no_forward_progress_interaction_stop_signal(ctx)
stash_blocker_signal(ctx, no_progress_signal)
stash_turn_halt_from_blocker_signal(ctx, no_progress_signal, source="enforcement_backstop")
raise_if_turn_halt(ctx)
# Response-time gate: peek at the model's final output to tell ASK_QUESTION
# (always allowed) from a REPLY with a coverage gap or progress-narration.

View file

@ -28,7 +28,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
from skyvern.forge.sdk.copilot.enforcement import terminal_challenge_blocker_signal_from_current_page_evidence
from skyvern.forge.sdk.copilot.enforcement import (
register_no_progress_interaction_click,
terminal_challenge_blocker_signal_from_current_page_evidence,
)
from skyvern.forge.sdk.copilot.loop_detection import (
detect_failed_tool_step_loop_for_ctx,
detect_tool_loop,
@ -72,6 +75,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"reached_download_target",
"synthesized_block_offered",
"synthesized_block_offered_trajectory_len",
"consecutive_no_progress_interaction_count",
)
@ -359,6 +363,8 @@ class SkyvernOverlayMCPServer(MCPServer):
exc_info=True,
)
err = scrub_secrets_from_structure(copilot_ctx, {"ok": False, "error": f"{tool_name} failed: {e}"})
if tool_name == "click":
register_no_progress_interaction_click(copilot_ctx, outcome="click_failed")
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, err)
return _copilot_to_call_tool_result(err)

View file

@ -204,6 +204,10 @@ class AgentContext:
repeated_failure_nudge_emitted_at_streak: int = 0
code_authoring_guardrail_reject_count: int = 0
last_code_authoring_reject_was_credential_priority: bool = False
# Climbs on each click that made no verified forward progress (failed/timed-out
# click or a hollow post-click observe); resets on verified progress.
consecutive_no_progress_interaction_count: int = 0
last_scout_act_observe_outcome: str | None = None
pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = None
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
challenge_gated_proxy_retry_count: int = 0

View file

@ -54,6 +54,7 @@ from .scouting import (
_reset_evaluate_tracker,
_resolve_scout_role_name,
_steer_evaluate_result,
account_no_progress_interaction_click,
)
LOG = structlog.get_logger()
@ -407,6 +408,7 @@ async def _click_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
ctx.last_scout_act_observe_outcome = None
_clear_pending_browser_interaction_observation(ctx)
source_url = _consume_scout_source_url(ctx)
pending_role_name = ctx.pending_scout_role_name
@ -444,6 +446,7 @@ async def _click_post_hook(
_attach_scout_page_summary(result, page_evidence)
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
await _maybe_attach_reached_download_target(ctx, result, url=url, page_evidence=page_evidence)
account_no_progress_interaction_click(ctx, result)
return result

View file

@ -45,7 +45,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
RepairNextAction,
build_diagnosis_repair_contract,
)
from skyvern.forge.sdk.copilot.enforcement import repair_ceiling_stop_signal
from skyvern.forge.sdk.copilot.enforcement import repair_ceiling_stop_signal, reset_no_progress_interaction_count
from skyvern.forge.sdk.copilot.failure_tracking import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
PER_TOOL_BUDGET_FAILURE_CATEGORY,
@ -2396,6 +2396,8 @@ def _update_repair_loop_state(copilot_ctx: Any, contract: DiagnosisRepairContrac
copilot_ctx.verified_criteria_high_water = high_water | current
copilot_ctx.verified_prefix_high_water_len = max(prefix_high, prefix_len)
copilot_ctx.verified_full_pass_consumed = full_pass
if progressed:
reset_no_progress_interaction_count(copilot_ctx)
signature = _repair_non_convergence_signature(contract)
if signature is None or progressed:

View file

@ -20,7 +20,11 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
has_bounded_page_schema,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
from skyvern.forge.sdk.copilot.enforcement import (
_RECENT_TOOL_OUTPUT_CHAR_CAP,
register_no_progress_interaction_click,
reset_no_progress_interaction_count,
)
from skyvern.forge.sdk.copilot.reached_download_target import (
ReachedDownloadTarget,
)
@ -90,6 +94,7 @@ def _consume_pending_browser_interaction_observation(
current_url=current_url,
)
return False
reset_no_progress_interaction_count(ctx)
return True
@ -349,6 +354,7 @@ async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> di
except Exception:
parsed = None
outcome = "error"
ctx.last_scout_act_observe_outcome = outcome
LOG.info(
"copilot_scout_act_observe",
outcome=outcome,
@ -391,6 +397,21 @@ async def _register_scout_interaction_observation(
return step, page_evidence
def account_no_progress_interaction_click(ctx: AgentContext, result: dict[str, Any]) -> None:
"""Climb or reset the no-forward-progress counter from a click's outcome: a failed click or hollow
observe is no progress, an attached observe is progress, a capture timeout/error is neutral."""
if not result.get("ok"):
register_no_progress_interaction_click(ctx, outcome="click_failed")
return
outcome = ctx.last_scout_act_observe_outcome
if outcome == "attached":
reset_no_progress_interaction_count(ctx)
elif outcome == "hollow":
register_no_progress_interaction_click(ctx, outcome="hollow")
else:
LOG.info("copilot_no_progress_interaction_neutral", outcome=outcome)
_PAGE_SUMMARY_TEXT_CAP = 80
_PAGE_SUMMARY_MAX_FIELDS = 8
_PAGE_SUMMARY_MAX_SUBMITS = 4

View file

@ -42,6 +42,7 @@ _LOOP_TERMINAL_REASON_CODES = frozenset(
"loop_detected_generic",
"code_authoring_guardrail_churn",
"credential_priority_authoring_churn",
"loop_detected_no_forward_progress_interaction",
}
)
_ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset(

View file

@ -533,6 +533,23 @@ def test_turn_halt_exit_renders_code_authoring_churn_reason() -> None:
assert "update_workflow" not in result.user_response
def test_turn_halt_exit_renders_no_forward_progress_interaction_reason() -> None:
ctx = _ctx()
signal = _signal(
kind="loop_detected",
user_facing="I couldn't get past this step. Tell me what to change and I'll try a different approach.",
internal_reason_code="loop_detected_no_forward_progress_interaction",
blocked_tool="click",
)
ctx.blocker_signal = signal
halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)
result = _build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
assert "couldn't get past this step" in result.user_response
assert "click" not in result.user_response
def test_turn_halt_exit_keeps_credential_scout_reply_through_refresh_with_draft() -> None:
ctx = _ctx()
ctx.last_workflow_yaml = "title: Draft\nworkflow_definition:\n blocks: []\n"

View file

@ -22,7 +22,10 @@ from typing import Any
import pytest
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
maybe_clear_blocker_signal_on_tool_success,
)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisInput,
@ -35,6 +38,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
from skyvern.forge.sdk.copilot.enforcement import (
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
MAX_NO_PROGRESS_INTERACTION_ATTEMPTS,
MAX_PROBABLE_SITE_BLOCK_STOP_NUDGES,
POST_FAILED_TEST_INSPECT_FIRST_NUDGE,
POST_FAILED_TEST_NUDGE,
@ -52,6 +56,8 @@ from skyvern.forge.sdk.copilot.enforcement import (
_prune_input_list,
_recover_from_context_overflow,
_strip_input_images,
register_no_progress_interaction_click,
reset_no_progress_interaction_count,
)
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
from skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_tool
@ -511,3 +517,146 @@ def test_credential_priority_churn_defers_to_repair_ceiling() -> None:
signal = ctx.blocker_signal
assert isinstance(signal, CopilotToolBlockerSignal)
assert signal.internal_reason_code == "repair_ceiling_reached"
def test_no_progress_interaction_floor_raises_at_ceiling() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
with pytest.raises(CopilotTurnHalt) as excinfo:
_check_enforcement(ctx)
assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
signal = ctx.blocker_signal
assert isinstance(signal, CopilotToolBlockerSignal)
assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"
assert signal.renders_final_reply is True
assert signal.recovery_hint == "report_blocker_to_user"
def test_no_progress_interaction_floor_does_not_raise_below_ceiling() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1
assert _check_enforcement(ctx) is None
assert ctx.blocker_signal is None
def test_no_progress_interaction_floor_defers_to_repair_ceiling() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
ctx.latest_diagnosis_repair_contract = _ceiling_reached_contract()
with pytest.raises(CopilotTurnHalt) as excinfo:
_check_enforcement(ctx)
assert excinfo.value.halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
signal = ctx.blocker_signal
assert isinstance(signal, CopilotToolBlockerSignal)
assert signal.internal_reason_code == "repair_ceiling_reached"
def test_no_progress_interaction_floor_yields_to_non_retriable_nav_error() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
ctx.last_test_ok = False
ctx.last_test_non_retriable_nav_error = (
"Failed to navigate to url https://does-not-resolve.example. Error message: net::ERR_NAME_NOT_RESOLVED"
)
nudge = _check_enforcement(ctx)
assert nudge is not None
assert ctx.blocker_signal is None
with pytest.raises(CopilotNonRetriableNavError):
_maybe_raise_non_retriable_nav(ctx)
def test_register_no_progress_interaction_click_stashes_blocker_at_cap() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1
register_no_progress_interaction_click(ctx, outcome="click_failed")
assert ctx.consecutive_no_progress_interaction_count == MAX_NO_PROGRESS_INTERACTION_ATTEMPTS
signal = ctx.blocker_signal
assert isinstance(signal, CopilotToolBlockerSignal)
assert signal.internal_reason_code == "loop_detected_no_forward_progress_interaction"
def test_register_no_progress_interaction_click_below_cap_does_not_stash() -> None:
ctx = _fresh_context()
register_no_progress_interaction_click(ctx, outcome="hollow")
assert ctx.consecutive_no_progress_interaction_count == 1
assert ctx.blocker_signal is None
def test_register_no_progress_interaction_click_defers_to_terminal_held_blocker() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = MAX_NO_PROGRESS_INTERACTION_ATTEMPTS - 1
terminal = CopilotToolBlockerSignal(
blocker_kind="loop_detected",
agent_steering_text="The repair made no progress.",
user_facing_reason="I couldn't get past the same problem after several attempts.",
recovery_hint="report_blocker_to_user",
internal_reason_code="repair_ceiling_reached",
blocked_tool="update_and_run_blocks",
)
ctx.blocker_signal = terminal
register_no_progress_interaction_click(ctx, outcome="click_failed")
assert ctx.blocker_signal is terminal
def test_reset_no_progress_interaction_count_clears_counter() -> None:
ctx = _fresh_context()
ctx.consecutive_no_progress_interaction_count = 3
reset_no_progress_interaction_count(ctx)
assert ctx.consecutive_no_progress_interaction_count == 0
def _hit_no_progress_cap(ctx: CopilotContext) -> CopilotToolBlockerSignal:
for _ in range(MAX_NO_PROGRESS_INTERACTION_ATTEMPTS):
register_no_progress_interaction_click(ctx, outcome="hollow")
held = ctx.blocker_signal
assert isinstance(held, CopilotToolBlockerSignal)
assert held.internal_reason_code == "loop_detected_no_forward_progress_interaction"
assert held.renders_final_reply is True
return held
def test_no_progress_reset_clears_held_blocker_after_cap() -> None:
ctx = _fresh_context()
_hit_no_progress_cap(ctx)
reset_no_progress_interaction_count(ctx)
assert ctx.consecutive_no_progress_interaction_count == 0
assert ctx.blocker_signal is None
assert ctx.latest_tool_blocker_signal is None
def test_no_progress_reset_at_progress_seam_stops_re_halt() -> None:
ctx = _fresh_context()
_hit_no_progress_cap(ctx)
reset_no_progress_interaction_count(ctx)
assert ctx.consecutive_no_progress_interaction_count == 0
assert ctx.blocker_signal is None
assert _check_enforcement(ctx) is None
@pytest.mark.parametrize("recovery_tool", ["evaluate", "inspect_page_for_composition"])
def test_no_progress_held_blocker_survives_progress_tool_success(recovery_tool: str) -> None:
ctx = _fresh_context()
held = _hit_no_progress_cap(ctx)
maybe_clear_blocker_signal_on_tool_success(ctx, recovery_tool)
assert ctx.blocker_signal is held

View file

@ -0,0 +1,64 @@
"""A raised ``client.call_tool`` for the click tool returns before the post-hook, so the shared
no-progress helper must fire from the adapter exception handler exactly once, with no double-count
against the post-hook path. OSS-synced fixture references use example.* only.
"""
from __future__ import annotations
from typing import Any, NoReturn
from unittest.mock import MagicMock
import pytest
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer
from skyvern.forge.sdk.copilot.runtime import AgentContext
class _RaisingClient:
async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> NoReturn:
raise RuntimeError("Timeout 5000ms exceeded")
def _agent_ctx() -> AgentContext:
return AgentContext(
organization_id="o_1",
workflow_id="w_1",
workflow_permanent_id="wpid_1",
workflow_yaml="",
browser_session_id="pbs_1",
stream=MagicMock(),
)
def _make_server(ctx: AgentContext, tool_name: str) -> SkyvernOverlayMCPServer:
server = SkyvernOverlayMCPServer(
transport=MagicMock(),
overlays={tool_name: SchemaOverlay()},
alias_map={},
allowlist=frozenset(),
context_provider=lambda: ctx,
)
server._client = _RaisingClient()
return server
@pytest.mark.asyncio
async def test_raised_click_increments_no_progress_counter_exactly_once() -> None:
ctx = _agent_ctx()
server = _make_server(ctx, "click")
result = await server.call_tool("click", {"selector": "#submit"})
assert result.isError is True
assert ctx.consecutive_no_progress_interaction_count == 1
@pytest.mark.asyncio
async def test_raised_non_click_tool_leaves_no_progress_counter_untouched() -> None:
ctx = _agent_ctx()
server = _make_server(ctx, "evaluate")
result = await server.call_tool("evaluate", {"expression": "scan()"})
assert result.isError is True
assert ctx.consecutive_no_progress_interaction_count == 0

View file

@ -23,6 +23,8 @@ from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
from skyvern.forge.sdk.copilot.tools import _click_post_hook
from skyvern.forge.sdk.copilot.tools.scouting import (
_consume_pending_browser_interaction_observation,
_mark_pending_browser_interaction_observation,
account_no_progress_interaction_click,
)
_SOURCE_URL = "https://example.com/product"
@ -348,3 +350,81 @@ class TestActObserveToolGate:
assert result["ok"] is True
assert "page" not in result["data"]
assert ctx.flow_evidence[0]["had_bounded_schema"] is False
def _np_ctx(*, server: Any = None, counter: int = 0) -> SimpleNamespace:
ctx = _ctx(server=server)
ctx.consecutive_no_progress_interaction_count = counter
ctx.last_scout_act_observe_outcome = None
ctx.blocker_signal = None
return ctx
class TestNoProgressInteractionAccounting:
@pytest.mark.asyncio
async def test_hollow_click_increments_counter(self) -> None:
ctx = _np_ctx(server=_server_returning({"page_title": "Loading", "forms": []}))
await _run_click(ctx)
assert ctx.last_scout_act_observe_outcome == "hollow"
assert ctx.consecutive_no_progress_interaction_count == 1
@pytest.mark.asyncio
async def test_attached_click_resets_counter(self) -> None:
ctx = _np_ctx(server=_server_returning(_bounded_extractor_payload()), counter=2)
await _run_click(ctx)
assert ctx.last_scout_act_observe_outcome == "attached"
assert ctx.consecutive_no_progress_interaction_count == 0
@pytest.mark.asyncio
async def test_failed_click_increments_counter(self) -> None:
ctx = _np_ctx()
await _click_post_hook(
{"ok": False, "error": "Timeout 5000ms exceeded"},
{"browser_context": {"url": _LANDING_URL}},
ctx,
)
assert ctx.consecutive_no_progress_interaction_count == 1
@pytest.mark.asyncio
async def test_capture_timeout_observe_is_neutral(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS", 0.05)
async def slow_extract(*_args: object, **_kwargs: object) -> dict[str, Any]:
await asyncio.sleep(0.25)
return {"ok": True, "data": {"result": _bounded_extractor_payload()}}
server = SimpleNamespace()
server.call_internal_tool = AsyncMock(side_effect=slow_extract)
ctx = _np_ctx(server=server, counter=2)
await _run_click(ctx)
assert ctx.last_scout_act_observe_outcome == "timeout"
assert ctx.consecutive_no_progress_interaction_count == 2
def test_neutral_outcome_does_not_touch_counter(self) -> None:
ctx = _np_ctx(counter=2)
ctx.last_scout_act_observe_outcome = None
account_no_progress_interaction_click(ctx, {"ok": True, "data": {"selector": "#x"}})
assert ctx.consecutive_no_progress_interaction_count == 2
def test_delayed_credit_via_consume_resets_counter(self) -> None:
ctx = _np_ctx(counter=3)
_mark_pending_browser_interaction_observation(ctx, tool_name="click", url=_LANDING_URL)
consumed = _consume_pending_browser_interaction_observation(
ctx,
current_url=_LANDING_URL,
evidence={**_bounded_extractor_payload(), "current_url": _LANDING_URL},
)
assert consumed is True
assert ctx.consecutive_no_progress_interaction_count == 0