mirror of
https://github.com/Skyvern-AI/skyvern.git
synced 2026-07-09 16:09:13 +00:00
feat(SKY-8207): consistent egress IP for Save & Reuse Session runs (#7169)
Co-authored-by: Suchintan Singh <suchintan@skyvern.com> Co-authored-by: AronPerez <aperez0295@gmail.com>
This commit is contained in:
parent
147eea686f
commit
d0094d3568
55 changed files with 2745 additions and 146 deletions
|
|
@ -1357,7 +1357,7 @@ async def skyvern_workflow_create(
|
|||
Pass run_with="code" to opt into cached script execution. Blocks share a browser session automatically.
|
||||
|
||||
Leave optional toggles and overrides unset unless the user explicitly asks for them. This
|
||||
applies to workflow-level fields (persist_browser_session, extra_http_headers,
|
||||
applies to workflow-level fields (persist_browser_session, pin_saved_session_ip, extra_http_headers,
|
||||
totp_verification_url, totp_identifier, etc.) AND block-level overrides (max_retries,
|
||||
max_steps_per_run, totp_identifier, complete_criterion, error_code_mapping,
|
||||
continue_on_failure, engine, model, etc.). The schema defaults are intentional; silently
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ class Workflow(UniversalBaseModel):
|
|||
totp_verification_url: typing.Optional[str] = None
|
||||
totp_identifier: typing.Optional[str] = None
|
||||
persist_browser_session: typing.Optional[bool] = None
|
||||
pin_saved_session_ip: typing.Optional[bool] = None
|
||||
browser_profile_id: typing.Optional[str] = None
|
||||
browser_profile_key: typing.Optional[str] = None
|
||||
model: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ class WorkflowCreateYamlRequest(UniversalBaseModel):
|
|||
totp_verification_url: typing.Optional[str] = None
|
||||
totp_identifier: typing.Optional[str] = None
|
||||
persist_browser_session: typing.Optional[bool] = None
|
||||
pin_saved_session_ip: typing.Optional[bool] = None
|
||||
browser_profile_id: typing.Optional[str] = None
|
||||
browser_profile_key: typing.Optional[str] = None
|
||||
model: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = None
|
||||
|
|
|
|||
|
|
@ -616,6 +616,7 @@ _CANONICAL_WORKFLOW_SETTING_FIELDS: tuple[str, ...] = (
|
|||
"totp_verification_url",
|
||||
"totp_identifier",
|
||||
"persist_browser_session",
|
||||
"pin_saved_session_ip",
|
||||
"browser_profile_id",
|
||||
"browser_profile_key",
|
||||
"model",
|
||||
|
|
|
|||
|
|
@ -8647,6 +8647,7 @@ async def _update_workflow(
|
|||
totp_verification_url=workflow.totp_verification_url,
|
||||
totp_identifier=workflow.totp_identifier,
|
||||
persist_browser_session=workflow.persist_browser_session,
|
||||
pin_saved_session_ip=workflow.pin_saved_session_ip,
|
||||
browser_profile_id=workflow.browser_profile_id,
|
||||
browser_profile_key=workflow.browser_profile_key,
|
||||
model=workflow.model,
|
||||
|
|
|
|||
|
|
@ -514,6 +514,7 @@ class WorkflowModel(SoftDeleteMixin, Base):
|
|||
totp_verification_url = Column(String)
|
||||
totp_identifier = Column(String)
|
||||
persist_browser_session = Column(Boolean, default=False, nullable=False)
|
||||
pin_saved_session_ip = Column(Boolean, default=False, nullable=False, server_default=sqlalchemy.false())
|
||||
browser_profile_id = Column(String, nullable=True)
|
||||
browser_profile_key = Column(String, nullable=True)
|
||||
model = Column(JSON, nullable=True)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from typing import cast
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import case, desc, or_, select
|
||||
|
|
@ -27,7 +28,7 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
|
|||
PersistentBrowserSession,
|
||||
PersistentBrowserType,
|
||||
)
|
||||
from skyvern.schemas.proxy_pinning import generate_proxy_session_id
|
||||
from skyvern.schemas.proxy_pinning import generate_proxy_session_id, parse_proxy_location_input
|
||||
from skyvern.schemas.runs import ProxyLocation, ProxyLocationInput
|
||||
|
||||
LOG = structlog.get_logger()
|
||||
|
|
@ -387,17 +388,34 @@ class BrowserSessionsRepository(BaseRepository):
|
|||
browser_type: PersistentBrowserType | None = None,
|
||||
browser_profile_id: str | None = None,
|
||||
generate_browser_profile: bool = False,
|
||||
inherit_profile_proxy: bool = False,
|
||||
) -> PersistentBrowserSession:
|
||||
"""Create a new persistent browser session."""
|
||||
proxy_location, proxy_session_id = normalize_proxy_pin_for_create(
|
||||
proxy_location=proxy_location,
|
||||
proxy_session_id=proxy_session_id,
|
||||
)
|
||||
extensions_str: list[str] | None = (
|
||||
[extension.value for extension in extensions] if extensions is not None else None
|
||||
)
|
||||
serialized_proxy_location = serialize_proxy_location(proxy_location)
|
||||
async with self.Session() as session:
|
||||
if inherit_profile_proxy and browser_profile_id and proxy_session_id is None:
|
||||
query = (
|
||||
select(BrowserProfileModel)
|
||||
.filter_by(browser_profile_id=browser_profile_id)
|
||||
.filter_by(organization_id=organization_id)
|
||||
.filter(BrowserProfileModel.deleted_at.is_(None))
|
||||
)
|
||||
browser_profile = (await session.scalars(query)).first()
|
||||
if browser_profile and browser_profile.proxy_session_id:
|
||||
proxy_session_id = browser_profile.proxy_session_id
|
||||
# The ORM column stores the serialized string; deserialize before it flows
|
||||
# into serialize_proxy_location, which rejects a bare str.
|
||||
proxy_location = cast(
|
||||
ProxyLocationInput, parse_proxy_location_input(browser_profile.proxy_location)
|
||||
)
|
||||
|
||||
proxy_location, proxy_session_id = normalize_proxy_pin_for_create(
|
||||
proxy_location=proxy_location,
|
||||
proxy_session_id=proxy_session_id,
|
||||
)
|
||||
serialized_proxy_location = serialize_proxy_location(proxy_location)
|
||||
browser_session = PersistentBrowserSessionModel(
|
||||
organization_id=organization_id,
|
||||
runnable_type=runnable_type,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,19 @@ class WorkflowRunCredentialSelectionsRepository(BaseRepository):
|
|||
).first()
|
||||
return selection.credential_id if selection else None
|
||||
|
||||
@db_operation("get_workflow_run_credential_selections_for_run")
|
||||
async def get_selections_for_run(self, workflow_run_id: str) -> dict[str, str]:
|
||||
async with self.Session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
WorkflowRunCredentialSelectionModel.parameter_key,
|
||||
WorkflowRunCredentialSelectionModel.credential_id,
|
||||
).where(WorkflowRunCredentialSelectionModel.workflow_run_id == workflow_run_id)
|
||||
)
|
||||
).all()
|
||||
return {parameter_key: credential_id for parameter_key, credential_id in rows}
|
||||
|
||||
async def _get_selection(self, session: AsyncSession, workflow_run_id: str, parameter_key: str) -> str | None:
|
||||
selection = (
|
||||
await session.scalars(
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ if TYPE_CHECKING:
|
|||
|
||||
from skyvern.forge.sdk.db._sentinels import _UNSET
|
||||
from skyvern.forge.sdk.db.models import (
|
||||
PersistentBrowserSessionModel,
|
||||
TaskModel,
|
||||
TaskRunModel,
|
||||
WorkflowModel,
|
||||
|
|
@ -54,6 +55,7 @@ from skyvern.forge.sdk.db.utils import (
|
|||
truncate_oversized_jsonb_value,
|
||||
)
|
||||
from skyvern.forge.sdk.log_artifacts import save_workflow_run_logs
|
||||
from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
|
||||
from skyvern.forge.sdk.schemas.tasks import Task
|
||||
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter
|
||||
from skyvern.forge.sdk.workflow.models.workflow import (
|
||||
|
|
@ -726,10 +728,12 @@ class WorkflowRunsRepository(BaseRepository):
|
|||
workflow_permanent_id: str,
|
||||
organization_id: str | None = None,
|
||||
sequential_key: str | None = None,
|
||||
include_browser_session_rows: bool = False,
|
||||
) -> WorkflowRun | None:
|
||||
async with self.Session() as session:
|
||||
query = select(WorkflowRunModel).filter_by(workflow_permanent_id=workflow_permanent_id)
|
||||
query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
if not include_browser_session_rows:
|
||||
query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
if organization_id:
|
||||
query = query.filter_by(organization_id=organization_id)
|
||||
query = query.filter_by(status=WorkflowRunStatus.queued)
|
||||
|
|
@ -761,10 +765,12 @@ class WorkflowRunsRepository(BaseRepository):
|
|||
workflow_permanent_id: str,
|
||||
organization_id: str | None = None,
|
||||
sequential_key: str | None = None,
|
||||
include_browser_session_rows: bool = False,
|
||||
) -> WorkflowRun | None:
|
||||
async with self.Session() as session:
|
||||
query = select(WorkflowRunModel).filter_by(workflow_permanent_id=workflow_permanent_id)
|
||||
query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
if not include_browser_session_rows:
|
||||
query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
if organization_id:
|
||||
query = query.filter_by(organization_id=organization_id)
|
||||
query = query.filter_by(status=WorkflowRunStatus.running)
|
||||
|
|
@ -792,11 +798,35 @@ class WorkflowRunsRepository(BaseRepository):
|
|||
if run is None:
|
||||
return None
|
||||
|
||||
# Lane resolution mirrors enqueue priority: browser_session_id > browser_address
|
||||
# > sequential_key > whole workflow. A debug-session run carries a browser_session_id
|
||||
# but is excluded from the session lane (as at enqueue), so it serializes on its key.
|
||||
query = select(WorkflowRunModel).filter_by(organization_id=run.organization_id)
|
||||
self_forced = False
|
||||
if run.browser_session_id and not run.debug_session_id:
|
||||
try:
|
||||
persistent_browser_session = (
|
||||
await session.scalars(
|
||||
select(PersistentBrowserSessionModel).filter_by(
|
||||
persistent_browser_session_id=run.browser_session_id,
|
||||
organization_id=run.organization_id,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
self_forced = (
|
||||
persistent_browser_session is not None
|
||||
and persistent_browser_session.runnable_type == FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
|
||||
)
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
"Failed to fetch persistent browser session for runtime lane selection",
|
||||
workflow_run_id=workflow_run_id,
|
||||
browser_session_id=run.browser_session_id,
|
||||
organization_id=run.organization_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Lane resolution mirrors enqueue priority: browser_session_id > browser_address
|
||||
# > sequential_key > whole workflow. Debug and forced-session runs carry a
|
||||
# browser_session_id but are excluded from the session lane as they are at enqueue.
|
||||
query = select(WorkflowRunModel).filter_by(organization_id=run.organization_id)
|
||||
if run.browser_session_id and not run.debug_session_id and not self_forced:
|
||||
query = query.filter_by(browser_session_id=run.browser_session_id)
|
||||
elif run.browser_address:
|
||||
query = query.filter_by(browser_address=run.browser_address)
|
||||
|
|
@ -804,11 +834,13 @@ class WorkflowRunsRepository(BaseRepository):
|
|||
query = query.filter_by(
|
||||
workflow_permanent_id=run.workflow_permanent_id,
|
||||
sequential_key=run.sequential_key,
|
||||
).filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
else:
|
||||
query = query.filter_by(workflow_permanent_id=run.workflow_permanent_id).filter(
|
||||
WorkflowRunModel.browser_session_id.is_(None)
|
||||
)
|
||||
if not self_forced:
|
||||
query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
else:
|
||||
query = query.filter_by(workflow_permanent_id=run.workflow_permanent_id)
|
||||
if not self_forced:
|
||||
query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
|
||||
|
||||
# Sequential runs are stamped queued_at before Temporal submission; the fallback
|
||||
# only guards hand-created rows (e.g. tests) from comparing against None.
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ class WorkflowsRepository(BaseRepository):
|
|||
totp_verification_url: str | None = None,
|
||||
totp_identifier: str | None = None,
|
||||
persist_browser_session: bool = False,
|
||||
pin_saved_session_ip: bool = False,
|
||||
browser_profile_id: str | None = None,
|
||||
browser_profile_key: str | None = None,
|
||||
model: dict[str, Any] | None = None,
|
||||
|
|
@ -137,6 +138,7 @@ class WorkflowsRepository(BaseRepository):
|
|||
extra_http_headers=extra_http_headers,
|
||||
cdp_connect_headers=cdp_connect_headers,
|
||||
persist_browser_session=persist_browser_session,
|
||||
pin_saved_session_ip=pin_saved_session_ip,
|
||||
browser_profile_id=browser_profile_id,
|
||||
browser_profile_key=browser_profile_key,
|
||||
model=model,
|
||||
|
|
@ -670,6 +672,7 @@ class WorkflowsRepository(BaseRepository):
|
|||
totp_verification_url: str | None | object = _UNSET,
|
||||
totp_identifier: str | None | object = _UNSET,
|
||||
persist_browser_session: bool | None = None,
|
||||
pin_saved_session_ip: bool | None = None,
|
||||
browser_profile_id: str | None | object = _UNSET,
|
||||
browser_profile_key: str | None | object = _UNSET,
|
||||
model: dict[str, Any] | None | object = _UNSET,
|
||||
|
|
@ -720,6 +723,8 @@ class WorkflowsRepository(BaseRepository):
|
|||
workflow.totp_identifier = cast(str | None, totp_identifier)
|
||||
if persist_browser_session is not None:
|
||||
workflow.persist_browser_session = persist_browser_session
|
||||
if pin_saved_session_ip is not None:
|
||||
workflow.pin_saved_session_ip = pin_saved_session_ip
|
||||
if browser_profile_id is not _UNSET:
|
||||
workflow.browser_profile_id = cast(str | None, browser_profile_id)
|
||||
if browser_profile_key is not _UNSET:
|
||||
|
|
@ -986,6 +991,7 @@ class WorkflowsRepository(BaseRepository):
|
|||
totp_verification_url: str | None | object = _UNSET,
|
||||
totp_identifier: str | None | object = _UNSET,
|
||||
persist_browser_session: bool | None = None,
|
||||
pin_saved_session_ip: bool | None = None,
|
||||
browser_profile_id: str | None | object = _UNSET,
|
||||
browser_profile_key: str | None | object = _UNSET,
|
||||
model: dict[str, Any] | None | object = _UNSET,
|
||||
|
|
@ -1072,6 +1078,8 @@ class WorkflowsRepository(BaseRepository):
|
|||
workflow.totp_identifier = cast(str | None, totp_identifier)
|
||||
if persist_browser_session is not None:
|
||||
workflow.persist_browser_session = persist_browser_session
|
||||
if pin_saved_session_ip is not None:
|
||||
workflow.pin_saved_session_ip = pin_saved_session_ip
|
||||
if browser_profile_id is not _UNSET:
|
||||
workflow.browser_profile_id = cast(str | None, browser_profile_id)
|
||||
if browser_profile_key is not _UNSET:
|
||||
|
|
|
|||
|
|
@ -559,6 +559,7 @@ def convert_to_workflow(
|
|||
totp_verification_url=workflow_model.totp_verification_url,
|
||||
totp_identifier=workflow_model.totp_identifier,
|
||||
persist_browser_session=workflow_model.persist_browser_session,
|
||||
pin_saved_session_ip=workflow_model.pin_saved_session_ip,
|
||||
browser_profile_id=workflow_model.browser_profile_id,
|
||||
browser_profile_key=workflow_model.browser_profile_key,
|
||||
model=workflow_model.model,
|
||||
|
|
|
|||
|
|
@ -932,6 +932,7 @@ async def _commit_staged_workflow(
|
|||
totp_verification_url=staged_workflow.totp_verification_url,
|
||||
totp_identifier=staged_workflow.totp_identifier,
|
||||
persist_browser_session=staged_workflow.persist_browser_session,
|
||||
pin_saved_session_ip=staged_workflow.pin_saved_session_ip,
|
||||
browser_profile_id=staged_workflow.browser_profile_id,
|
||||
browser_profile_key=staged_workflow.browser_profile_key,
|
||||
model=staged_workflow.model,
|
||||
|
|
@ -970,6 +971,7 @@ async def _restore_workflow_definition(original_workflow: Workflow | None, organ
|
|||
totp_verification_url=original_workflow.totp_verification_url,
|
||||
totp_identifier=original_workflow.totp_identifier,
|
||||
persist_browser_session=original_workflow.persist_browser_session,
|
||||
pin_saved_session_ip=original_workflow.pin_saved_session_ip,
|
||||
browser_profile_id=original_workflow.browser_profile_id,
|
||||
browser_profile_key=original_workflow.browser_profile_key,
|
||||
model=original_workflow.model,
|
||||
|
|
@ -1567,7 +1569,7 @@ def _normalize_copilot_yaml(workflow_yaml: str) -> WorkflowCreateYAMLRequest:
|
|||
return workflow_yaml_request
|
||||
|
||||
|
||||
def _yaml_enable_self_healing(workflow_yaml: str | None) -> bool | None:
|
||||
def _yaml_bool_setting(workflow_yaml: str | None, setting_name: str) -> bool | None:
|
||||
if not workflow_yaml:
|
||||
return None
|
||||
try:
|
||||
|
|
@ -1576,10 +1578,18 @@ def _yaml_enable_self_healing(workflow_yaml: str | None) -> bool | None:
|
|||
return None
|
||||
if not isinstance(parsed, dict):
|
||||
return None
|
||||
value = parsed.get("enable_self_healing")
|
||||
value = parsed.get(setting_name)
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _yaml_enable_self_healing(workflow_yaml: str | None) -> bool | None:
|
||||
return _yaml_bool_setting(workflow_yaml, "enable_self_healing")
|
||||
|
||||
|
||||
def _yaml_pin_saved_session_ip(workflow_yaml: str | None) -> bool | None:
|
||||
return _yaml_bool_setting(workflow_yaml, "pin_saved_session_ip")
|
||||
|
||||
|
||||
async def _process_workflow_yaml(
|
||||
workflow_id: str,
|
||||
workflow_permanent_id: str,
|
||||
|
|
@ -1607,7 +1617,13 @@ async def _process_workflow_yaml(
|
|||
# unsaved editor toggle survives an unrelated copilot edit. A persisted-lookup
|
||||
# failure propagates: failing the save is safer than writing an implicit disable.
|
||||
enable_self_healing = _yaml_enable_self_healing(settings_fallback_yaml)
|
||||
if enable_self_healing is None:
|
||||
|
||||
pin_saved_session_ip = _yaml_pin_saved_session_ip(workflow_yaml)
|
||||
if pin_saved_session_ip is None:
|
||||
pin_saved_session_ip = _yaml_pin_saved_session_ip(settings_fallback_yaml)
|
||||
|
||||
current_workflow: Workflow | None = None
|
||||
if enable_self_healing is None or pin_saved_session_ip is None:
|
||||
try:
|
||||
current_workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
|
|
@ -1615,7 +1631,10 @@ async def _process_workflow_yaml(
|
|||
)
|
||||
except WorkflowNotFound:
|
||||
current_workflow = None
|
||||
enable_self_healing = bool(current_workflow and current_workflow.enable_self_healing)
|
||||
if enable_self_healing is None:
|
||||
enable_self_healing = bool(current_workflow and getattr(current_workflow, "enable_self_healing", False))
|
||||
if pin_saved_session_ip is None:
|
||||
pin_saved_session_ip = bool(current_workflow and getattr(current_workflow, "pin_saved_session_ip", False))
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
return Workflow(
|
||||
|
|
@ -1632,6 +1651,7 @@ async def _process_workflow_yaml(
|
|||
totp_verification_url=workflow_yaml_request.totp_verification_url,
|
||||
totp_identifier=workflow_yaml_request.totp_identifier,
|
||||
persist_browser_session=workflow_yaml_request.persist_browser_session or False,
|
||||
pin_saved_session_ip=pin_saved_session_ip,
|
||||
browser_profile_id=workflow_yaml_request.browser_profile_id,
|
||||
browser_profile_key=workflow_yaml_request.browser_profile_key,
|
||||
model=workflow_yaml_request.model,
|
||||
|
|
|
|||
|
|
@ -60,6 +60,9 @@ class Extensions(StrEnum):
|
|||
CaptchaSolver = "captcha-solver"
|
||||
|
||||
|
||||
FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE = "forced_workflow_run"
|
||||
|
||||
|
||||
class PersistentBrowserSession(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ class Workflow(BaseModel):
|
|||
totp_verification_url: str | None = None
|
||||
totp_identifier: str | None = None
|
||||
persist_browser_session: bool = False
|
||||
pin_saved_session_ip: bool = False
|
||||
browser_profile_id: str | None = None
|
||||
browser_profile_key: str | None = None
|
||||
model: dict[str, Any] | None = None
|
||||
|
|
|
|||
|
|
@ -66,9 +66,13 @@ from skyvern.forge.sdk.db.id import generate_output_parameter_id, generate_workf
|
|||
from skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_features
|
||||
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
|
||||
from skyvern.forge.sdk.models import Step, StepStatus
|
||||
from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile
|
||||
from skyvern.forge.sdk.schemas.files import FileInfo
|
||||
from skyvern.forge.sdk.schemas.organizations import Organization
|
||||
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
|
||||
from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
|
||||
FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE,
|
||||
PersistentBrowserSession,
|
||||
)
|
||||
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
|
||||
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType
|
||||
from skyvern.forge.sdk.trace import traced
|
||||
|
|
@ -138,6 +142,11 @@ from skyvern.forge.sdk.workflow.status_mapping import (
|
|||
TASK_STATUS_MAP,
|
||||
)
|
||||
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
|
||||
from skyvern.schemas.proxy_pinning import (
|
||||
derive_proxy_session_id,
|
||||
redact_proxy_session_id,
|
||||
should_generate_proxy_session_id,
|
||||
)
|
||||
from skyvern.schemas.run_enums import RunEngine
|
||||
from skyvern.schemas.runs import (
|
||||
ProxyLocation,
|
||||
|
|
@ -188,6 +197,9 @@ RECORDING_WINDOW_END_BUFFER = timedelta(minutes=15)
|
|||
# Skip post-run work when only a sub-millisecond budget remains; asyncio.timeout would fire on the first await.
|
||||
POST_RUN_TIMEOUT_EXHAUSTED_THRESHOLD_SECONDS = 0.001
|
||||
|
||||
# Bound pre-finalization write-back so a stuck profile upload cannot leave the run in a non-terminal state.
|
||||
BROWSER_SESSION_WRITE_BACK_TIMEOUT = SAVE_DOWNLOADED_FILES_TIMEOUT
|
||||
|
||||
# Short lifespan for auto-provisioned CodeBlock sessions; the renewal loop extends it while the
|
||||
# run is active, so this only bounds how long a leaked session lingers if cleanup never runs.
|
||||
CODE_BLOCK_SESSION_TIMEOUT_MINUTES = 20
|
||||
|
|
@ -431,6 +443,16 @@ class CachedScriptBlocks:
|
|||
blocks_to_clear: list[ScriptBlock]
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowBrowserCleanupResult:
|
||||
browser_state: BrowserState | None
|
||||
tasks: list[Task]
|
||||
all_workflow_task_ids: list[str]
|
||||
child_workflow_run_ids: list[str]
|
||||
close_browser_on_completion: bool
|
||||
browser_session_write_back_attempted: bool = False
|
||||
|
||||
|
||||
def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition) -> dict[str, Any]:
|
||||
"""
|
||||
This function dumps the workflow definition and removes the irrelevant data to the definition, like created_at and modified_at fields inside:
|
||||
|
|
@ -1020,6 +1042,50 @@ class WorkflowService:
|
|||
|
||||
await self._validate_credential_ids(credential_ids_to_validate, organization)
|
||||
|
||||
@staticmethod
|
||||
def _get_rotating_credential_parameters(workflow: Workflow) -> list[CredentialParameter]:
|
||||
workflow_definition = getattr(workflow, "workflow_definition", None)
|
||||
if workflow_definition is None:
|
||||
return []
|
||||
return [
|
||||
parameter
|
||||
for parameter in workflow_definition.parameters
|
||||
if isinstance(parameter, CredentialParameter) and bool(parameter.credential_ids)
|
||||
]
|
||||
|
||||
async def _select_rotating_credential_parameters_for_render(
|
||||
self,
|
||||
*,
|
||||
workflow: Workflow,
|
||||
workflow_run: WorkflowRun,
|
||||
organization_id: str,
|
||||
) -> dict[str, str]:
|
||||
try:
|
||||
selections: dict[str, str] = {}
|
||||
for parameter in self._get_rotating_credential_parameters(workflow):
|
||||
credential_ids = parameter.credential_ids
|
||||
if not credential_ids:
|
||||
continue
|
||||
selections[parameter.key] = await select_credential_for_run(
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
organization_id=organization_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
parameter_key=parameter.key,
|
||||
credential_ids=credential_ids,
|
||||
selection_strategy=parameter.selection_strategy,
|
||||
)
|
||||
return selections
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
"Failed to select rotating credentials for workflow render parameters",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
exc_info=True,
|
||||
)
|
||||
if workflow.browser_profile_key:
|
||||
raise
|
||||
return {}
|
||||
|
||||
async def validate_schedule_parameters(
|
||||
self,
|
||||
workflow: Workflow,
|
||||
|
|
@ -1336,10 +1402,21 @@ class WorkflowService:
|
|||
organization,
|
||||
)
|
||||
|
||||
parameter_values = {param.key: value for param, value in workflow_parameter_values}
|
||||
rotating_credential_selections = await self._select_rotating_credential_parameters_for_render(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
organization_id=organization.organization_id,
|
||||
)
|
||||
parameter_values.update(rotating_credential_selections)
|
||||
workflow_run = await self._prepare_persisted_workflow_browser_profile(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
parameter_values={param.key: value for param, value in workflow_parameter_values},
|
||||
parameter_values=parameter_values,
|
||||
# Keyless workflows keep best-effort rotation selection; keyed workflows re-raise above.
|
||||
allow_missing_browser_profile_key=(
|
||||
bool(self._get_rotating_credential_parameters(workflow)) and not rotating_credential_selections
|
||||
),
|
||||
)
|
||||
|
||||
if workflow_parameter_values:
|
||||
|
|
@ -1407,6 +1484,7 @@ class WorkflowService:
|
|||
workflow: Workflow,
|
||||
workflow_run: WorkflowRun,
|
||||
parameter_values: dict[str, Any],
|
||||
allow_missing_browser_profile_key: bool = False,
|
||||
) -> WorkflowRun:
|
||||
if not workflow.persist_browser_session:
|
||||
return workflow_run
|
||||
|
|
@ -1414,11 +1492,22 @@ class WorkflowService:
|
|||
if workflow_run.browser_profile_id:
|
||||
return workflow_run
|
||||
|
||||
rendered_key = await self._render_workflow_browser_profile_key(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
parameter_values=parameter_values,
|
||||
)
|
||||
try:
|
||||
rendered_key = await self._render_workflow_browser_profile_key(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
parameter_values=parameter_values,
|
||||
)
|
||||
except MissingValueForParameter:
|
||||
if not allow_missing_browser_profile_key:
|
||||
raise
|
||||
LOG.warning(
|
||||
"Falling back to keyless managed browser profile after missing browser profile key",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
browser_profile_key=workflow.browser_profile_key,
|
||||
)
|
||||
rendered_key = None
|
||||
digest = build_browser_profile_key_digest(rendered_key)
|
||||
profile, created = await app.DATABASE.browser_sessions.get_or_create_managed_browser_profile(
|
||||
organization_id=workflow_run.organization_id,
|
||||
|
|
@ -1430,7 +1519,8 @@ class WorkflowService:
|
|||
try:
|
||||
await self._seed_managed_browser_profile_from_legacy_session(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
organization_id=workflow_run.organization_id,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
rendered_key=rendered_key,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
)
|
||||
|
|
@ -1448,6 +1538,15 @@ class WorkflowService:
|
|||
organization_id=workflow_run.organization_id,
|
||||
)
|
||||
return workflow_run
|
||||
|
||||
await self._reconcile_managed_browser_profile_proxy_pin(
|
||||
workflow=workflow,
|
||||
profile=profile,
|
||||
browser_profile_key_digest=digest,
|
||||
organization_id=workflow_run.organization_id,
|
||||
proxy_location=workflow_run.proxy_location,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
)
|
||||
updated_workflow_run = await app.DATABASE.workflow_runs.update_workflow_run(
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
|
|
@ -1461,29 +1560,175 @@ class WorkflowService:
|
|||
)
|
||||
return updated_workflow_run
|
||||
|
||||
async def _reconcile_managed_browser_profile_proxy_pin(
|
||||
self,
|
||||
*,
|
||||
workflow: Workflow,
|
||||
profile: BrowserProfile,
|
||||
browser_profile_key_digest: str,
|
||||
organization_id: str,
|
||||
proxy_location: ProxyLocationInput,
|
||||
workflow_run_id: str | None = None,
|
||||
) -> None:
|
||||
should_pin = workflow.pin_saved_session_ip and should_generate_proxy_session_id(proxy_location)
|
||||
try:
|
||||
existing_pin = profile.proxy_session_id
|
||||
if should_pin:
|
||||
if browser_profile_key_digest:
|
||||
proxy_session_id = derive_proxy_session_id(
|
||||
organization_id,
|
||||
workflow.workflow_permanent_id,
|
||||
browser_profile_key_digest,
|
||||
)
|
||||
else:
|
||||
proxy_session_id = derive_proxy_session_id(
|
||||
organization_id,
|
||||
workflow.workflow_permanent_id,
|
||||
)
|
||||
# Re-derive rather than trust existing values so a drifted or corrupted
|
||||
# pin or location self-heals to the deterministic per-segment state.
|
||||
if existing_pin == proxy_session_id and should_generate_proxy_session_id(profile.proxy_location):
|
||||
return
|
||||
await app.DATABASE.browser_sessions.update_browser_profile(
|
||||
profile_id=profile.browser_profile_id,
|
||||
organization_id=organization_id,
|
||||
proxy_location=ProxyLocation.RESIDENTIAL_ISP,
|
||||
proxy_session_id=proxy_session_id,
|
||||
)
|
||||
LOG.info(
|
||||
"Pinned managed browser profile proxy session",
|
||||
workflow_run_id=workflow_run_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
proxy_session_id=redact_proxy_session_id(proxy_session_id),
|
||||
)
|
||||
return
|
||||
|
||||
if profile.is_managed and existing_pin:
|
||||
await app.DATABASE.browser_sessions.update_browser_profile(
|
||||
profile_id=profile.browser_profile_id,
|
||||
organization_id=organization_id,
|
||||
proxy_location=None,
|
||||
proxy_session_id=None,
|
||||
)
|
||||
LOG.info(
|
||||
"Cleared managed browser profile proxy session pin",
|
||||
workflow_run_id=workflow_run_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
proxy_session_id=redact_proxy_session_id(existing_pin),
|
||||
)
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
"Failed to reconcile managed browser profile proxy pin",
|
||||
workflow_run_id=workflow_run_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
exc_info=True,
|
||||
)
|
||||
if should_pin:
|
||||
raise
|
||||
|
||||
async def _resolve_managed_browser_profile_for_run_request(
|
||||
self,
|
||||
*,
|
||||
workflow: Workflow,
|
||||
organization_id: str,
|
||||
workflow_request: WorkflowRequestBody,
|
||||
effective_proxy_location: ProxyLocationInput,
|
||||
extra_parameter_values: dict[str, str] | None = None,
|
||||
) -> str | None:
|
||||
try:
|
||||
if not workflow.persist_browser_session or workflow_request.browser_profile_id:
|
||||
return None
|
||||
|
||||
rendered_key = None
|
||||
if workflow.browser_profile_key:
|
||||
parameter_values: dict[str, Any] = {}
|
||||
for parameter in workflow.workflow_definition.parameters:
|
||||
if isinstance(parameter, WorkflowParameter) and parameter.default_value is not None:
|
||||
parameter_values[parameter.key] = parameter.default_value
|
||||
if workflow_request.data:
|
||||
parameter_values.update(
|
||||
{key: value for key, value in workflow_request.data.items() if value is not None}
|
||||
)
|
||||
if extra_parameter_values:
|
||||
parameter_values.update(extra_parameter_values)
|
||||
rendered_key = render_browser_profile_key(workflow.browser_profile_key, parameter_values)
|
||||
if not rendered_key:
|
||||
return None
|
||||
|
||||
digest = build_browser_profile_key_digest(rendered_key)
|
||||
profile, created = await app.DATABASE.browser_sessions.get_or_create_managed_browser_profile(
|
||||
organization_id=organization_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
browser_profile_key_digest=digest,
|
||||
name=_build_managed_browser_profile_name(workflow.title, rendered_key),
|
||||
)
|
||||
if created:
|
||||
try:
|
||||
await self._seed_managed_browser_profile_from_legacy_session(
|
||||
workflow=workflow,
|
||||
organization_id=organization_id,
|
||||
rendered_key=rendered_key,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
)
|
||||
except Exception:
|
||||
# Roll back the empty row so the next run re-attempts creation + seed;
|
||||
# this run degrades to the legacy session path instead of failing setup.
|
||||
LOG.warning(
|
||||
"Failed to seed managed browser profile from legacy session; rolling back and using legacy path",
|
||||
workflow_run_id=None,
|
||||
browser_profile_id=profile.browser_profile_id,
|
||||
exc_info=True,
|
||||
)
|
||||
await app.DATABASE.browser_sessions.hard_delete_browser_profile(
|
||||
profile_id=profile.browser_profile_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
return None
|
||||
await self._reconcile_managed_browser_profile_proxy_pin(
|
||||
workflow=workflow,
|
||||
profile=profile,
|
||||
browser_profile_key_digest=digest,
|
||||
organization_id=organization_id,
|
||||
proxy_location=effective_proxy_location,
|
||||
workflow_run_id=None,
|
||||
)
|
||||
return profile.browser_profile_id
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
"Failed to resolve managed browser profile for workflow run request",
|
||||
organization_id=organization_id,
|
||||
workflow_permanent_id=workflow.workflow_permanent_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
async def _seed_managed_browser_profile_from_legacy_session(
|
||||
self,
|
||||
*,
|
||||
workflow: Workflow,
|
||||
workflow_run: WorkflowRun,
|
||||
organization_id: str,
|
||||
rendered_key: str | None,
|
||||
browser_profile_id: str,
|
||||
workflow_run_id: str | None = None,
|
||||
) -> None:
|
||||
# Lazy migration: carry a workflow's accumulated Save & Reuse login state onto its new
|
||||
# managed profile the first time we create it, so the next run starts logged in. The OSS
|
||||
# launch path has no legacy-archive fallback, so this copy must happen here, not at launch.
|
||||
storage_key = build_workflow_browser_session_storage_key(workflow.workflow_permanent_id, rendered_key)
|
||||
legacy_session_dir = await app.STORAGE.retrieve_browser_session(workflow_run.organization_id, storage_key)
|
||||
legacy_session_dir = await app.STORAGE.retrieve_browser_session(organization_id, storage_key)
|
||||
if not legacy_session_dir:
|
||||
return
|
||||
await app.STORAGE.store_browser_profile(
|
||||
workflow_run.organization_id,
|
||||
organization_id,
|
||||
profile_id=browser_profile_id,
|
||||
directory=legacy_session_dir,
|
||||
)
|
||||
LOG.info(
|
||||
"Seeded managed browser profile from legacy session archive",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
workflow_run_id=workflow_run_id,
|
||||
browser_profile_id=browser_profile_id,
|
||||
browser_session_storage_key=storage_key,
|
||||
)
|
||||
|
|
@ -1635,6 +1880,7 @@ class WorkflowService:
|
|||
timeout_minutes=timeout_seconds // 60,
|
||||
browser_profile_id=browser_profile_id,
|
||||
proxy_location=proxy_location,
|
||||
inherit_profile_proxy=True,
|
||||
)
|
||||
|
||||
return browser_session
|
||||
|
|
@ -1692,6 +1938,7 @@ class WorkflowService:
|
|||
timeout_minutes=CODE_BLOCK_SESSION_TIMEOUT_MINUTES,
|
||||
browser_profile_id=browser_profile_id,
|
||||
proxy_location=proxy_location,
|
||||
inherit_profile_proxy=True,
|
||||
)
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
|
|
@ -2332,7 +2579,46 @@ class WorkflowService:
|
|||
# in place of the real terminal reason. When pre_finally_status is
|
||||
# still ``None`` (cancellation landed before block execution
|
||||
# completed), there's no captured intent to restore and we skip.
|
||||
browser_cleanup_result: WorkflowBrowserCleanupResult | None = None
|
||||
if pre_finally_status is not None:
|
||||
if workflow.persist_browser_session and pre_finally_status not in (
|
||||
WorkflowRunStatus.canceled,
|
||||
WorkflowRunStatus.failed,
|
||||
WorkflowRunStatus.terminated,
|
||||
WorkflowRunStatus.timed_out,
|
||||
):
|
||||
try:
|
||||
# Sequential dependency gates clear on terminal status, so persisted profile state must land first.
|
||||
browser_cleanup_result = await self._clean_up_workflow_browser(
|
||||
workflow_run=workflow_run,
|
||||
close_browser_on_completion=close_browser_on_completion,
|
||||
browser_session_id=browser_session_id,
|
||||
)
|
||||
except BaseException:
|
||||
LOG.warning(
|
||||
"Pre-finalization browser cleanup failed during execute_workflow cleanup",
|
||||
workflow_run_id=workflow_run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
if browser_cleanup_result is not None:
|
||||
try:
|
||||
async with asyncio.timeout(BROWSER_SESSION_WRITE_BACK_TIMEOUT):
|
||||
await self._persist_workflow_browser_session_if_needed(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
browser_state=browser_cleanup_result.browser_state,
|
||||
close_browser_on_completion=browser_cleanup_result.close_browser_on_completion,
|
||||
workflow_run_status=WorkflowRunStatus.completed,
|
||||
)
|
||||
# Only suppress clean_up_workflow's write-back once this one has
|
||||
# actually persisted; a failed/timed-out attempt must still be retried.
|
||||
browser_cleanup_result.browser_session_write_back_attempted = True
|
||||
except BaseException:
|
||||
LOG.warning(
|
||||
"Pre-finalization browser session write-back failed during execute_workflow cleanup",
|
||||
workflow_run_id=workflow_run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
workflow_run = await asyncio.shield(
|
||||
self._finalize_workflow_run_status(
|
||||
|
|
@ -2368,6 +2654,7 @@ class WorkflowService:
|
|||
browser_session_id=browser_session_id,
|
||||
close_browser_on_completion=close_browser_on_completion,
|
||||
need_call_webhook=need_call_webhook,
|
||||
browser_cleanup_result=browser_cleanup_result,
|
||||
)
|
||||
|
||||
return workflow_run
|
||||
|
|
@ -3971,25 +4258,16 @@ class WorkflowService:
|
|||
workflow_run_id=workflow_run_id,
|
||||
)
|
||||
if proxy_session_id:
|
||||
updated_headers = app.AGENT_FUNCTION.merge_proxy_session_extra_http_headers(
|
||||
headers, proxy_session_id
|
||||
)
|
||||
# The OSS AgentFunction stub returns the original headers; only persist when cloud injected a pin.
|
||||
if updated_headers == headers:
|
||||
return
|
||||
proxy_location_update: ProxyLocationInput | None = None
|
||||
if workflow_run.proxy_location is None:
|
||||
proxy_location_update = proxy_location or ProxyLocation.RESIDENTIAL_ISP
|
||||
update_kwargs: dict[str, Any] = {
|
||||
"workflow_run_id": workflow_run_id,
|
||||
"extra_http_headers": updated_headers,
|
||||
}
|
||||
if proxy_location_update is not None:
|
||||
update_kwargs["proxy_location"] = proxy_location_update
|
||||
await app.DATABASE.workflow_runs.update_workflow_run(**update_kwargs)
|
||||
workflow_run.extra_http_headers = updated_headers
|
||||
if proxy_location_update is not None:
|
||||
workflow_run.proxy_location = proxy_location_update
|
||||
if not await self._persist_proxy_pin_headers(
|
||||
workflow_run=workflow_run,
|
||||
headers=headers,
|
||||
proxy_session_id=proxy_session_id,
|
||||
proxy_location_update=proxy_location_update,
|
||||
):
|
||||
return
|
||||
LOG.info(
|
||||
"Applied LoginBlock credential proxy pin to workflow run",
|
||||
credential_id=credential_id,
|
||||
|
|
@ -4005,6 +4283,30 @@ class WorkflowService:
|
|||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _persist_proxy_pin_headers(
|
||||
self,
|
||||
*,
|
||||
workflow_run: WorkflowRun,
|
||||
headers: dict[str, str],
|
||||
proxy_session_id: str,
|
||||
proxy_location_update: ProxyLocationInput | None = None,
|
||||
) -> bool:
|
||||
updated_headers = app.AGENT_FUNCTION.merge_proxy_session_extra_http_headers(headers, proxy_session_id)
|
||||
# The OSS AgentFunction stub returns the original headers; only persist when cloud injected a pin.
|
||||
if updated_headers == headers:
|
||||
return False
|
||||
update_kwargs: dict[str, Any] = {
|
||||
"workflow_run_id": workflow_run.workflow_run_id,
|
||||
"extra_http_headers": updated_headers,
|
||||
}
|
||||
if proxy_location_update is not None:
|
||||
update_kwargs["proxy_location"] = proxy_location_update
|
||||
await app.DATABASE.workflow_runs.update_workflow_run(**update_kwargs)
|
||||
workflow_run.extra_http_headers = updated_headers
|
||||
if proxy_location_update is not None:
|
||||
workflow_run.proxy_location = proxy_location_update
|
||||
return True
|
||||
|
||||
async def _evaluate_debug_session_profile_decision(
|
||||
self,
|
||||
*,
|
||||
|
|
@ -4414,6 +4716,7 @@ class WorkflowService:
|
|||
totp_verification_url: str | None = None,
|
||||
totp_identifier: str | None = None,
|
||||
persist_browser_session: bool = False,
|
||||
pin_saved_session_ip: bool = False,
|
||||
browser_profile_id: str | None = None,
|
||||
browser_profile_key: str | None = None,
|
||||
model: dict[str, Any] | None = None,
|
||||
|
|
@ -4449,6 +4752,7 @@ class WorkflowService:
|
|||
totp_verification_url=totp_verification_url,
|
||||
totp_identifier=totp_identifier,
|
||||
persist_browser_session=persist_browser_session,
|
||||
pin_saved_session_ip=pin_saved_session_ip,
|
||||
browser_profile_id=browser_profile_id,
|
||||
browser_profile_key=browser_profile_key,
|
||||
model=model,
|
||||
|
|
@ -4842,6 +5146,7 @@ class WorkflowService:
|
|||
totp_verification_url: str | None | object = _UNSET,
|
||||
totp_identifier: str | None | object = _UNSET,
|
||||
persist_browser_session: bool | None = None,
|
||||
pin_saved_session_ip: bool | None = None,
|
||||
browser_profile_id: str | None | object = _UNSET,
|
||||
browser_profile_key: str | None | object = _UNSET,
|
||||
model: dict[str, Any] | None | object = _UNSET,
|
||||
|
|
@ -4879,6 +5184,7 @@ class WorkflowService:
|
|||
totp_verification_url=totp_verification_url,
|
||||
totp_identifier=totp_identifier,
|
||||
persist_browser_session=persist_browser_session,
|
||||
pin_saved_session_ip=pin_saved_session_ip,
|
||||
browser_profile_id=browser_profile_id,
|
||||
browser_profile_key=browser_profile_key,
|
||||
model=model,
|
||||
|
|
@ -4909,6 +5215,7 @@ class WorkflowService:
|
|||
totp_verification_url=totp_verification_url,
|
||||
totp_identifier=totp_identifier,
|
||||
persist_browser_session=persist_browser_session,
|
||||
pin_saved_session_ip=pin_saved_session_ip,
|
||||
browser_profile_id=browser_profile_id,
|
||||
browser_profile_key=browser_profile_key,
|
||||
model=model,
|
||||
|
|
@ -5255,22 +5562,119 @@ class WorkflowService:
|
|||
},
|
||||
)
|
||||
if force_browser_session:
|
||||
workflow_run = await app.DATABASE.workflow_runs.create_workflow_run(
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
workflow_id=workflow_id,
|
||||
organization_id=organization_id,
|
||||
browser_session_id=workflow_request.browser_session_id,
|
||||
browser_profile_id=browser_profile_id,
|
||||
proxy_location=workflow_request.proxy_location,
|
||||
webhook_callback_url=workflow_request.webhook_callback_url,
|
||||
totp_verification_url=workflow_request.totp_verification_url,
|
||||
totp_identifier=workflow_request.totp_identifier,
|
||||
parent_workflow_run_id=parent_workflow_run_id,
|
||||
max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolls,
|
||||
max_elapsed_time_minutes=max_elapsed_time_minutes,
|
||||
extra_http_headers=workflow_request.extra_http_headers,
|
||||
cdp_connect_headers=workflow_request.cdp_connect_headers,
|
||||
browser_address=workflow_request.browser_address,
|
||||
sequential_key=sequential_key,
|
||||
run_with=workflow_request.run_with,
|
||||
debug_session_id=debug_session_id,
|
||||
ai_fallback=workflow_request.ai_fallback,
|
||||
code_gen=code_gen,
|
||||
workflow_run_id=workflow_run_id,
|
||||
trigger_type=trigger_type,
|
||||
workflow_schedule_id=workflow_schedule_id,
|
||||
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
|
||||
copilot_session_id=copilot_session_id,
|
||||
)
|
||||
LOG.info(
|
||||
"Force-creating browser session for workflow run",
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
browser_session = await app.PERSISTENT_SESSIONS_MANAGER.create_session(
|
||||
organization_id=organization_id,
|
||||
proxy_location=workflow_request.proxy_location,
|
||||
timeout_minutes=60, # 60 minutes default timeout for forced browser sessions
|
||||
)
|
||||
browser_session_id = browser_session.persistent_browser_session_id
|
||||
LOG.info(
|
||||
"Browser session created for workflow run",
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
browser_session_id=browser_session_id,
|
||||
)
|
||||
forced_browser_profile_id = None
|
||||
effective_proxy_location = workflow_request.proxy_location
|
||||
pin_required = False
|
||||
try:
|
||||
workflow = await self.get_workflow(workflow_id=workflow_id)
|
||||
# pin_required must be known before any awaited call that can raise, so the
|
||||
# except path never falls through to creating an unprofiled pinned session.
|
||||
effective_proxy_location = (
|
||||
workflow_request.proxy_location
|
||||
if workflow_request.proxy_location is not None
|
||||
else workflow.proxy_location
|
||||
)
|
||||
pin_required = (
|
||||
workflow.persist_browser_session
|
||||
and workflow.pin_saved_session_ip
|
||||
and should_generate_proxy_session_id(effective_proxy_location)
|
||||
)
|
||||
# Rotating credential profile keys need the persisted run id before rendering.
|
||||
rotating_credential_selections = await self._select_rotating_credential_parameters_for_render(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
forced_browser_profile_id = await self._resolve_managed_browser_profile_for_run_request(
|
||||
workflow=workflow,
|
||||
organization_id=organization_id,
|
||||
workflow_request=workflow_request,
|
||||
effective_proxy_location=effective_proxy_location,
|
||||
extra_parameter_values=rotating_credential_selections,
|
||||
)
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
"Failed to resolve managed browser profile for forced browser session",
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
workflow_id=workflow_id,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
organization_id=organization_id,
|
||||
exc_info=True,
|
||||
)
|
||||
if pin_required and forced_browser_profile_id is None:
|
||||
LOG.info(
|
||||
"Skipping forced browser session without managed browser profile for pinned workflow",
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
workflow_id=workflow_id,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
browser_session = None
|
||||
else:
|
||||
try:
|
||||
browser_session = await app.PERSISTENT_SESSIONS_MANAGER.create_session(
|
||||
organization_id=organization_id,
|
||||
proxy_location=workflow_request.proxy_location,
|
||||
timeout_minutes=60, # 60 minutes default timeout for forced browser sessions
|
||||
runnable_type=FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE,
|
||||
browser_profile_id=forced_browser_profile_id,
|
||||
inherit_profile_proxy=True,
|
||||
)
|
||||
browser_session_id = browser_session.persistent_browser_session_id
|
||||
LOG.info(
|
||||
"Browser session created for workflow run",
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_session_id=browser_session_id,
|
||||
)
|
||||
workflow_run = await app.DATABASE.workflow_runs.update_workflow_run(
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_session_id=browser_session_id,
|
||||
)
|
||||
except Exception:
|
||||
LOG.warning(
|
||||
"Failed to force-create browser session for workflow run",
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
workflow_id=workflow_id,
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
organization_id=organization_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return workflow_run
|
||||
|
||||
return await app.DATABASE.workflow_runs.create_workflow_run(
|
||||
workflow_permanent_id=workflow_permanent_id,
|
||||
|
|
@ -6325,16 +6729,12 @@ class WorkflowService:
|
|||
errors=errors,
|
||||
)
|
||||
|
||||
async def clean_up_workflow(
|
||||
async def _clean_up_workflow_browser(
|
||||
self,
|
||||
workflow: Workflow,
|
||||
workflow_run: WorkflowRun,
|
||||
api_key: str | None = None,
|
||||
close_browser_on_completion: bool = True,
|
||||
need_call_webhook: bool = True,
|
||||
browser_session_id: str | None = None,
|
||||
) -> None:
|
||||
analytics.capture("skyvern-oss-agent-workflow-status", {"status": workflow_run.status})
|
||||
) -> WorkflowBrowserCleanupResult:
|
||||
tasks = await self.get_tasks_by_workflow_run_id(workflow_run.workflow_run_id)
|
||||
|
||||
# Look up child workflow runs (e.g. from task_v2 blocks) to flatten their
|
||||
|
|
@ -6368,80 +6768,133 @@ class WorkflowService:
|
|||
organization_id=workflow_run.organization_id,
|
||||
child_workflow_run_ids=child_workflow_run_ids,
|
||||
)
|
||||
return WorkflowBrowserCleanupResult(
|
||||
browser_state=browser_state,
|
||||
tasks=tasks,
|
||||
all_workflow_task_ids=all_workflow_task_ids,
|
||||
child_workflow_run_ids=child_workflow_run_ids,
|
||||
close_browser_on_completion=close_browser_on_completion,
|
||||
)
|
||||
|
||||
async def _persist_workflow_browser_session_if_needed(
|
||||
self,
|
||||
workflow: Workflow,
|
||||
workflow_run: WorkflowRun,
|
||||
browser_state: BrowserState | None,
|
||||
close_browser_on_completion: bool,
|
||||
workflow_run_status: WorkflowRunStatus | None = None,
|
||||
) -> None:
|
||||
if not (
|
||||
browser_state and workflow.persist_browser_session and browser_state.browser_artifacts.browser_session_dir
|
||||
):
|
||||
return
|
||||
|
||||
effective_workflow_run_status = workflow_run_status or workflow_run.status
|
||||
if effective_workflow_run_status != WorkflowRunStatus.completed:
|
||||
LOG.info(
|
||||
"Skipped persisting browser session for non-completed workflow run",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
workflow_run_status=effective_workflow_run_status,
|
||||
)
|
||||
return
|
||||
|
||||
browser_profile = None
|
||||
if workflow_run.browser_profile_id:
|
||||
browser_profile = await app.DATABASE.browser_sessions.get_browser_profile(
|
||||
workflow_run.browser_profile_id,
|
||||
organization_id=workflow_run.organization_id,
|
||||
)
|
||||
|
||||
if (
|
||||
browser_profile
|
||||
and browser_profile.is_managed
|
||||
# Only write back to a managed profile this workflow owns; a foreign
|
||||
# managed id supplied explicitly is treated like a user profile.
|
||||
and browser_profile.workflow_permanent_id == workflow_run.workflow_permanent_id
|
||||
):
|
||||
if not close_browser_on_completion:
|
||||
await persist_session_cookies(
|
||||
browser_state.browser_context,
|
||||
browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
await app.STORAGE.store_browser_profile(
|
||||
workflow_run.organization_id,
|
||||
profile_id=browser_profile.browser_profile_id,
|
||||
directory=browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
LOG.info(
|
||||
"Persisted managed browser profile for workflow run",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_profile_id=browser_profile.browser_profile_id,
|
||||
)
|
||||
elif browser_profile is None:
|
||||
# No managed profile to write back to: either none was stamped, or a
|
||||
# stamped managed profile was deleted between setup and finalization.
|
||||
# Persist to the legacy session archive so the run's state isn't lost —
|
||||
# the next run reseeds a managed profile from it. Only the deleted-mid-run
|
||||
# case is unexpected, so warn there.
|
||||
if workflow_run.browser_profile_id:
|
||||
LOG.warning(
|
||||
"Managed browser profile missing at finalization; persisting to legacy session archive instead",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_profile_id=workflow_run.browser_profile_id,
|
||||
)
|
||||
browser_session_storage_key = await self.get_workflow_browser_session_storage_key(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
)
|
||||
if not close_browser_on_completion:
|
||||
await persist_session_cookies(
|
||||
browser_state.browser_context,
|
||||
browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
await app.STORAGE.store_browser_session(
|
||||
workflow_run.organization_id,
|
||||
browser_session_storage_key,
|
||||
browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
LOG.info(
|
||||
"Persisted browser session for workflow run",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_session_storage_key=browser_session_storage_key,
|
||||
)
|
||||
|
||||
async def clean_up_workflow(
|
||||
self,
|
||||
workflow: Workflow,
|
||||
workflow_run: WorkflowRun,
|
||||
api_key: str | None = None,
|
||||
close_browser_on_completion: bool = True,
|
||||
need_call_webhook: bool = True,
|
||||
browser_session_id: str | None = None,
|
||||
browser_cleanup_result: WorkflowBrowserCleanupResult | None = None,
|
||||
) -> None:
|
||||
analytics.capture("skyvern-oss-agent-workflow-status", {"status": workflow_run.status})
|
||||
if browser_cleanup_result is None:
|
||||
browser_cleanup_result = await self._clean_up_workflow_browser(
|
||||
workflow_run=workflow_run,
|
||||
close_browser_on_completion=close_browser_on_completion,
|
||||
browser_session_id=browser_session_id,
|
||||
)
|
||||
|
||||
browser_state = browser_cleanup_result.browser_state
|
||||
tasks = browser_cleanup_result.tasks
|
||||
all_workflow_task_ids = browser_cleanup_result.all_workflow_task_ids
|
||||
child_workflow_run_ids = browser_cleanup_result.child_workflow_run_ids
|
||||
close_browser_on_completion = browser_cleanup_result.close_browser_on_completion
|
||||
if browser_state:
|
||||
await self.persist_video_data(
|
||||
browser_state, workflow, workflow_run, close_browser_on_completion=close_browser_on_completion
|
||||
)
|
||||
if tasks:
|
||||
await self.persist_debug_artifacts(browser_state, tasks[-1], workflow, workflow_run)
|
||||
if workflow.persist_browser_session and browser_state.browser_artifacts.browser_session_dir:
|
||||
if workflow_run.status != WorkflowRunStatus.completed:
|
||||
LOG.info(
|
||||
"Skipped persisting browser session for non-completed workflow run",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
workflow_run_status=workflow_run.status,
|
||||
)
|
||||
else:
|
||||
browser_profile = None
|
||||
if workflow_run.browser_profile_id:
|
||||
browser_profile = await app.DATABASE.browser_sessions.get_browser_profile(
|
||||
workflow_run.browser_profile_id,
|
||||
organization_id=workflow_run.organization_id,
|
||||
)
|
||||
|
||||
if (
|
||||
browser_profile
|
||||
and browser_profile.is_managed
|
||||
# Only write back to a managed profile this workflow owns; a foreign
|
||||
# managed id supplied explicitly is treated like a user profile.
|
||||
and browser_profile.workflow_permanent_id == workflow_run.workflow_permanent_id
|
||||
):
|
||||
if not close_browser_on_completion:
|
||||
await persist_session_cookies(
|
||||
browser_state.browser_context,
|
||||
browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
await app.STORAGE.store_browser_profile(
|
||||
workflow_run.organization_id,
|
||||
profile_id=browser_profile.browser_profile_id,
|
||||
directory=browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
LOG.info(
|
||||
"Persisted managed browser profile for workflow run",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_profile_id=browser_profile.browser_profile_id,
|
||||
)
|
||||
elif browser_profile is None:
|
||||
# No managed profile to write back to: either none was stamped, or a
|
||||
# stamped managed profile was deleted between setup and finalization.
|
||||
# Persist to the legacy session archive so the run's state isn't lost —
|
||||
# the next run reseeds a managed profile from it. Only the deleted-mid-run
|
||||
# case is unexpected, so warn there.
|
||||
if workflow_run.browser_profile_id:
|
||||
LOG.warning(
|
||||
"Managed browser profile missing at finalization; persisting to legacy session archive instead",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_profile_id=workflow_run.browser_profile_id,
|
||||
)
|
||||
browser_session_storage_key = await self.get_workflow_browser_session_storage_key(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
)
|
||||
if not close_browser_on_completion:
|
||||
await persist_session_cookies(
|
||||
browser_state.browser_context,
|
||||
browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
await app.STORAGE.store_browser_session(
|
||||
workflow_run.organization_id,
|
||||
browser_session_storage_key,
|
||||
browser_state.browser_artifacts.browser_session_dir,
|
||||
)
|
||||
LOG.info(
|
||||
"Persisted browser session for workflow run",
|
||||
workflow_run_id=workflow_run.workflow_run_id,
|
||||
browser_session_storage_key=browser_session_storage_key,
|
||||
)
|
||||
if not browser_cleanup_result.browser_session_write_back_attempted:
|
||||
await self._persist_workflow_browser_session_if_needed(
|
||||
workflow=workflow,
|
||||
workflow_run=workflow_run,
|
||||
browser_state=browser_state,
|
||||
close_browser_on_completion=close_browser_on_completion,
|
||||
)
|
||||
|
||||
await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks(all_workflow_task_ids)
|
||||
|
||||
|
|
@ -6946,6 +7399,7 @@ class WorkflowService:
|
|||
totp_verification_url=runtime_workflow.totp_verification_url,
|
||||
totp_identifier=runtime_workflow.totp_identifier,
|
||||
persist_browser_session=runtime_workflow.persist_browser_session,
|
||||
pin_saved_session_ip=runtime_workflow.pin_saved_session_ip,
|
||||
browser_profile_id=runtime_workflow.browser_profile_id,
|
||||
browser_profile_key=runtime_workflow.browser_profile_key,
|
||||
model=runtime_workflow.model,
|
||||
|
|
@ -7085,6 +7539,9 @@ class WorkflowService:
|
|||
totp_verification_url=request.totp_verification_url,
|
||||
totp_identifier=request.totp_identifier,
|
||||
persist_browser_session=request.persist_browser_session,
|
||||
pin_saved_session_ip=request.pin_saved_session_ip
|
||||
if "pin_saved_session_ip" in request.model_fields_set
|
||||
else existing_latest_workflow.pin_saved_session_ip,
|
||||
browser_profile_id=request.browser_profile_id,
|
||||
browser_profile_key=request.browser_profile_key,
|
||||
model=request.model,
|
||||
|
|
@ -7129,6 +7586,7 @@ class WorkflowService:
|
|||
totp_verification_url=request.totp_verification_url,
|
||||
totp_identifier=request.totp_identifier,
|
||||
persist_browser_session=request.persist_browser_session,
|
||||
pin_saved_session_ip=request.pin_saved_session_ip,
|
||||
browser_profile_id=request.browser_profile_id,
|
||||
browser_profile_key=request.browser_profile_key,
|
||||
model=request.model,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
import secrets
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from skyvern.schemas.proxy_location import GeoTarget, ProxyLocation, ProxyLocationInput
|
||||
|
|
@ -31,6 +32,12 @@ def generate_proxy_session_id(source_id: str) -> str:
|
|||
return proxy_session_id
|
||||
|
||||
|
||||
def derive_proxy_session_id(*parts: str) -> str:
|
||||
if not parts or any(not part or not part.strip() for part in parts):
|
||||
raise ValueError("Cannot derive proxy session id from empty parts")
|
||||
return sha256(":".join(parts).encode("utf-8")).hexdigest()[:10]
|
||||
|
||||
|
||||
def redact_proxy_session_id(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1315,6 +1315,7 @@ class WorkflowCreateYAMLRequest(BaseModel):
|
|||
totp_verification_url: str | None = None
|
||||
totp_identifier: str | None = None
|
||||
persist_browser_session: bool = False
|
||||
pin_saved_session_ip: bool = False
|
||||
browser_profile_id: str | None = None
|
||||
browser_profile_key: str | None = None
|
||||
model: dict[str, Any] | None = None
|
||||
|
|
|
|||
|
|
@ -349,6 +349,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
|
|||
is_high_priority: bool = False,
|
||||
browser_profile_id: str | None = None,
|
||||
generate_browser_profile: bool = False,
|
||||
inherit_profile_proxy: bool = False,
|
||||
wait_for_startup: bool = True,
|
||||
) -> PersistentBrowserSession:
|
||||
"""Create a new browser session for an organization and return its ID with the browser state."""
|
||||
|
|
@ -367,6 +368,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
|
|||
browser_type=browser_type,
|
||||
browser_profile_id=browser_profile_id,
|
||||
generate_browser_profile=generate_browser_profile,
|
||||
inherit_profile_proxy=inherit_profile_proxy,
|
||||
)
|
||||
|
||||
# Launch the browser immediately for standalone sessions so the
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ class PersistentSessionsManager(Protocol):
|
|||
is_high_priority: bool = False,
|
||||
browser_profile_id: str | None = None,
|
||||
generate_browser_profile: bool = False,
|
||||
inherit_profile_proxy: bool = False,
|
||||
wait_for_startup: bool = True,
|
||||
) -> PersistentBrowserSession:
|
||||
"""Create a new browser session."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue