vikarti.anatra/Skyvern
1
0
Fork 0
mirror of https://github.com/Skyvern-AI/skyvern.git synced 2026-07-09 16:09:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

SKY-11911 scout fill carry rebind (#7215)

Browse source
This commit is contained in:
Andrew Neilson 2026-07-08 14:40:40 -07:00 • committed by GitHub
parent 400fd43d7b
commit d72b6a1ba0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 730 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
skyvern/forge/sdk/copilot/agent.py
View file

@ -1012,6 +1012,16 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
if is_optional_dismissal_only_trajectory(ctx.scout_trajectory):
LOG.debug("copilot_synthesized_block_offer_skipped", reason="optional_dismissal_only")
return ""
fill_step_count = sum(
1
for interaction in ctx.scout_trajectory
if str(interaction.get("tool_name") or "") in {"type_text", "select_option", "fill_credential_field"}
)
LOG.info(
"copilot_synthesis_input_fill_steps",
trajectory_len=len(ctx.scout_trajectory),
fill_step_count=fill_step_count,
)
synthesized = synthesize_code_block(ctx.scout_trajectory, reached_download_target=ctx.reached_download_target)
if synthesized is None:
LOG.debug(
@ -4259,6 +4269,7 @@ async def _run_copilot_turn_impl(
ctx.prior_discovery_calls_made = prior_structured_context.discovery_calls_made
ctx.prior_page_inspection_calls_made = prior_structured_context.page_inspection_calls_made
ctx.prior_observed_acted_pages = [page.model_dump() for page in prior_structured_context.observed_acted_pages]
ctx.prior_fill_carry = [carry.model_dump() for carry in prior_structured_context.fill_carry]
ctx.build_phase = initial_build_phase(
ctx.turn_intent,
chat_request.message or "",

111
skyvern/forge/sdk/copilot/context.py
View file

@ -5,10 +5,11 @@ from __future__ import annotations
import json
import re
import uuid
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal, get_args
import structlog
from pydantic import BaseModel, Field
from typing_extensions import NotRequired, TypedDict
@ -22,6 +23,8 @@ from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.workflow.models.workflow import Workflow
LOG = structlog.get_logger()
ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]
COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)
ProposalDisposition = Literal["no_proposal", "auto_applicable", "review_untested", "review_tested"]
@ -147,6 +150,22 @@ class ObservedPage(BaseModel):
reached_via: str = ""
FillCarryToolName = Literal["type_text", "select_option", "fill_credential_field"]
class FillCarry(BaseModel):
source_url: str = ""
selector: str = ""
tool_name: FillCarryToolName
role: str = ""
accessible_name: str = ""
typed_length: int = 0
typed_value: str = ""
value: str = ""
credential_id: str = ""
credential_field: str = ""
class LoadedResultTargetContext(BaseModel):
selector: str = ""
is_table: bool = False
@ -214,6 +233,7 @@ class StructuredContext(BaseModel):
page_inspection_calls_made: int = 0
observed_acted_pages: list[ObservedPage] = Field(default_factory=list)
loaded_result_targets: list[LoadedResultTargetContext] = Field(default_factory=list)
fill_carry: list[FillCarry] = Field(default_factory=list)
def to_json_str(self) -> str:
payload = self.model_dump(mode="json")
@ -365,6 +385,82 @@ def render_loaded_result_context_for_prompt(global_llm_context: str) -> str:
_MAX_OBSERVED_ACTED_PAGES = 20
_MAX_FILL_CARRY = 20
_FILL_CARRY_TEXT_CAP = 240
_FILL_CARRY_TOOLS = frozenset({"type_text", "select_option", "fill_credential_field"})
_FILL_CARRY_CREDENTIAL_FIELDS = frozenset({"username", "password", "totp"})
FillCarryPrimitive = str | int | bool | None
def _carry_text(value: FillCarryPrimitive, *, max_chars: int = _FILL_CARRY_TEXT_CAP) -> str:
if not isinstance(value, str):
return ""
return value.strip()[:max_chars]
def _carry_int(value: FillCarryPrimitive) -> int:
if value is None or isinstance(value, bool):
return 0
try:
parsed = int(value)
except (TypeError, ValueError):
return 0
return parsed if parsed > 0 else 0
def _fill_carry_from_scout_trajectory(trajectory: Sequence[Mapping[str, FillCarryPrimitive]]) -> list[FillCarry]:
carry: list[FillCarry] = []
for interaction in trajectory:
tool_name = _carry_text(interaction.get("tool_name"), max_chars=40)
selector = _carry_text(interaction.get("selector"))
source_url = _carry_text(interaction.get("source_url"), max_chars=2048)
if tool_name not in _FILL_CARRY_TOOLS or not selector or not source_url:
continue
role = _carry_text(interaction.get("role"), max_chars=80)
accessible_name = _carry_text(interaction.get("accessible_name"), max_chars=160)
typed_length = _carry_int(interaction.get("typed_length"))
if tool_name == "type_text":
carry.append(
FillCarry(
source_url=source_url,
selector=selector,
tool_name="type_text",
role=role,
accessible_name=accessible_name,
typed_length=typed_length,
typed_value=_carry_text(interaction.get("typed_value")),
)
)
elif tool_name == "select_option":
value = _carry_text(interaction.get("value"))
if value:
carry.append(
FillCarry(
source_url=source_url,
selector=selector,
tool_name="select_option",
role=role,
accessible_name=accessible_name,
value=value,
)
)
elif tool_name == "fill_credential_field":
credential_id = _carry_text(interaction.get("credential_id"))
credential_field = _carry_text(interaction.get("credential_field"), max_chars=20)
if credential_id and credential_field in _FILL_CARRY_CREDENTIAL_FIELDS:
carry.append(
FillCarry(
source_url=source_url,
selector=selector,
tool_name="fill_credential_field",
role=role,
accessible_name=accessible_name,
typed_length=typed_length,
credential_id=credential_id,
credential_field=credential_field,
)
)
return carry[-_MAX_FILL_CARRY:]
def _merge_observed_acted_pages(prior: list[ObservedPage], flow_evidence: list[dict[str, Any]]) -> list[ObservedPage]:
@ -426,12 +522,18 @@ def finalize_discovery_counter_in_global_llm_context(ctx: Any, raw_context: str
loaded_result_targets = _loaded_result_targets_from_steer(
getattr(ctx, "latest_evaluate_result_composition_steer", None)
)
raw_scout_trajectory = getattr(ctx, "scout_trajectory", None)
scout_trajectory = raw_scout_trajectory if isinstance(raw_scout_trajectory, Sequence) else ()
fill_carry = _fill_carry_from_scout_trajectory(
[interaction for interaction in scout_trajectory if isinstance(interaction, Mapping)]
)
if (
not raw_context
and this_turn == 0
and inspections_this_turn == 0
and not flow_evidence
and not loaded_result_targets
and not fill_carry
):
return None
sc = StructuredContext.from_json_str(raw_context)
@ -440,6 +542,13 @@ def finalize_discovery_counter_in_global_llm_context(ctx: Any, raw_context: str
sc.observed_acted_pages = _merge_observed_acted_pages(sc.observed_acted_pages, flow_evidence)
# Replace with this turn's targets so stale extraction hints do not persist.
sc.loaded_result_targets = loaded_result_targets
sc.fill_carry = fill_carry
if fill_carry:
LOG.info(
"copilot_fill_carry_persisted",
source_url=fill_carry[0].source_url,
field_count=len(fill_carry),
)
return sc.to_json_str()

3
skyvern/forge/sdk/copilot/runtime.py
View file

@ -179,6 +179,7 @@ class ScoutedInteraction(TypedDict):
role: NotRequired[str]
accessible_name: NotRequired[str]
trajectory_index: NotRequired[int]
carried: NotRequired[bool]
# Credential fills carry references and metadata only — never secret values.
credential_id: NotRequired[str]
credential_field: NotRequired[str]
@ -329,6 +330,8 @@ class AgentContext:
# flow_evidence does not cover it (closes the spent-inspection-budget
# deadlock). Each item: {url, had_bounded_schema, reached_via}.
prior_observed_acted_pages: list[dict[str, Any]] = field(default_factory=list)
prior_fill_carry: list[dict[str, str | int]] = field(default_factory=list)
fill_carry_rebound_done: bool = False
post_budget_page_inspection_required: bool = False
post_budget_page_inspection_url: str | None = None
post_budget_page_inspection_run_id: str | None = None

167
skyvern/forge/sdk/copilot/tools/scouting.py
View file

@ -29,6 +29,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
has_bounded_page_schema,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import FillCarry
from skyvern.forge.sdk.copilot.enforcement import (
_RECENT_TOOL_OUTPUT_CHAR_CAP,
record_scouted_output_coverage,
@ -63,6 +64,8 @@ from .banned_blocks import _copilot_block_authoring_policy
LOG = structlog.get_logger()
_FILL_CARRY_RETRYABLE_VALIDATION_FAILURES = frozenset({"page_mismatch", "selector_absent_from_page_evidence"})
def _mark_page_inspected(ctx: AgentContext) -> None:
ctx.post_budget_page_inspection_required = False
@ -109,6 +112,7 @@ def _consume_pending_browser_interaction_observation(
_MAX_SCOUTED_INTERACTIONS = 60
_FILL_CARRY_SELECTOR_COUNT_TIMEOUT_SECONDS = 2.0
async def _live_working_page_url(ctx: AgentContext) -> str | None:
@ -129,7 +133,14 @@ async def _live_working_page_url(ctx: AgentContext) -> str | None:
async def _capture_scout_source_url(ctx: AgentContext) -> None:
# Pre-action: a navigating click/Enter would leave only the destination URL, not the page the selector acted on.
ctx.pending_scout_source_url = await _live_working_page_url(ctx)
source_url = await _live_working_page_url(ctx)
ctx.pending_scout_source_url = source_url
if not source_url or ctx.fill_carry_rebound_done or not ctx.prior_fill_carry:
return
page_evidence = await _scout_act_observe_page_evidence(ctx, url=source_url)
if page_evidence is None or not has_bounded_page_schema(page_evidence):
return
await rebind_prior_fill_carry_from_page_evidence(ctx, page_evidence=page_evidence, url=source_url)
def _consume_scout_source_url(ctx: AgentContext) -> str | None:
@ -372,6 +383,158 @@ def _record_scouted_interaction(
)
def _page_evidence_has_selector(value: Any, selector: str) -> bool:
if isinstance(value, dict):
if value.get("selector") == selector:
return True
return any(_page_evidence_has_selector(child, selector) for child in value.values())
if isinstance(value, list):
return any(_page_evidence_has_selector(child, selector) for child in value)
return False
def _page_evidence_with_inputs_as_fields(page_evidence: dict[str, Any]) -> dict[str, Any]:
inputs = page_evidence.get("inputs")
if not isinstance(inputs, list) or not inputs:
return page_evidence
fields: list[dict[str, Any]] = []
for item in inputs:
if not isinstance(item, dict):
continue
field = dict(item)
selector = field.get("selector")
if isinstance(selector, str) and selector.startswith("input#"):
field["selector"] = f"#{selector.split('#', 1)[1]}"
fields.append(field)
if not fields:
return page_evidence
forms = page_evidence.get("forms")
normalized = dict(page_evidence)
normalized["forms"] = [*(forms if isinstance(forms, list) else []), {"fields": fields}]
return normalized
async def _fill_carry_validation_failure(
ctx: AgentContext,
carry: FillCarry,
*,
page_evidence: dict[str, Any],
url: str,
) -> str | None:
evidence_url = str(page_evidence.get("current_url") or page_evidence.get("inspected_url") or url).strip()
if not evidence_url or not _same_page_ignoring_fragment(carry.source_url, evidence_url):
return "page_mismatch"
count = await _selector_live_match_count(
ctx, carry.selector, timeout_seconds=_FILL_CARRY_SELECTOR_COUNT_TIMEOUT_SECONDS
)
if count != 1:
return "selector_count_mismatch"
selector_in_page_evidence = _page_evidence_has_selector(
_page_evidence_with_inputs_as_fields(page_evidence), carry.selector
)
if carry.role and carry.accessible_name:
role, accessible_name = await _resolve_scout_role_name(ctx, carry.selector)
if role != carry.role or accessible_name != carry.accessible_name:
return "role_name_mismatch"
elif not selector_in_page_evidence:
return "selector_absent_from_page_evidence"
return None
def _fill_carry_to_interaction(carry: FillCarry, trajectory_index: int) -> ScoutedInteraction:
interaction: ScoutedInteraction = {
"tool_name": carry.tool_name,
"selector": carry.selector,
"source_url": carry.source_url,
"trajectory_index": trajectory_index,
"carried": True,
}
if carry.role:
interaction["role"] = carry.role
if carry.accessible_name:
interaction["accessible_name"] = carry.accessible_name
if carry.typed_length:
interaction["typed_length"] = carry.typed_length
if carry.tool_name == "type_text" and carry.typed_value:
interaction["typed_value"] = carry.typed_value
elif carry.tool_name == "select_option" and carry.value:
interaction["value"] = carry.value
elif carry.tool_name == "fill_credential_field":
interaction["credential_id"] = carry.credential_id
interaction["credential_field"] = carry.credential_field
return interaction
async def _maybe_rebind_prior_fill_carry(
ctx: AgentContext,
*,
page_evidence: dict[str, Any],
url: str,
) -> None:
if ctx.fill_carry_rebound_done:
return
prior = []
for raw in ctx.prior_fill_carry:
try:
prior.append(FillCarry.model_validate(raw))
except Exception:
continue
if not prior:
ctx.fill_carry_rebound_done = True
return
rebound: list[FillCarry] = []
for carry in prior:
failure = await _fill_carry_validation_failure(ctx, carry, page_evidence=page_evidence, url=url)
if failure is not None:
LOG.info(
"copilot_fill_carry_rebind_degraded",
reason=failure,
url=url,
source_url=carry.source_url,
)
if failure not in _FILL_CARRY_RETRYABLE_VALIDATION_FAILURES:
ctx.fill_carry_rebound_done = True
return
rebound.append(carry)
ctx.fill_carry_rebound_done = True
trajectory = list(ctx.scout_trajectory)
for carry in rebound:
trajectory.append(_fill_carry_to_interaction(carry, len(trajectory)))
ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]
LOG.info(
"copilot_fill_carry_rebound",
url=url,
field_count=len(rebound),
)
async def rebind_prior_fill_carry_from_current_page(ctx: AgentContext) -> bool:
if ctx.fill_carry_rebound_done or not ctx.prior_fill_carry:
return False
url = await _live_working_page_url(ctx)
if not url:
return False
page_evidence = await _scout_act_observe_page_evidence(ctx, url=url)
if page_evidence is None or not has_bounded_page_schema(page_evidence):
return False
return await rebind_prior_fill_carry_from_page_evidence(ctx, page_evidence=page_evidence, url=url)
async def rebind_prior_fill_carry_from_page_evidence(
ctx: AgentContext,
*,
page_evidence: dict[str, Any],
url: str,
) -> bool:
if ctx.fill_carry_rebound_done or not ctx.prior_fill_carry:
return False
if not has_bounded_page_schema(page_evidence):
return False
trajectory_len = len(ctx.scout_trajectory)
await _maybe_rebind_prior_fill_carry(ctx, page_evidence=page_evidence, url=url)
return len(ctx.scout_trajectory) > trajectory_len
_ACT_OBSERVE_TOOLS = frozenset({"click"})
@ -1150,6 +1313,8 @@ async def _steer_evaluate_result(ctx: AgentContext, result: dict[str, Any], *, u
if not isinstance(result.get("data"), dict):
return
page_evidence = await _scout_act_observe_page_evidence(ctx, url=url)
if page_evidence is not None and has_bounded_page_schema(page_evidence):
await _maybe_rebind_prior_fill_carry(ctx, page_evidence=page_evidence, url=url)
acted = await _maybe_steer_evaluate_to_action(ctx, result, url=url, page_evidence=page_evidence)
await _maybe_attach_reached_download_target(
ctx, result, url=url, page_evidence=_EVIDENCE_UNSET if acted else page_evidence