diff --git a/alembic/versions/2026_04_25_0046-d1474f2d1581_add_script_run_to_workflow_run_blocks.py b/alembic/versions/2026_04_25_0046-d1474f2d1581_add_script_run_to_workflow_run_blocks.py new file mode 100644 index 000000000..8ccdc3804 --- /dev/null +++ b/alembic/versions/2026_04_25_0046-d1474f2d1581_add_script_run_to_workflow_run_blocks.py @@ -0,0 +1,30 @@ +"""add script_run to workflow_run_blocks + +Revision ID: d1474f2d1581 +Revises: c19d7d385560 +Create Date: 2026-04-25T00:46:05.444225+00:00 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "d1474f2d1581" +down_revision: Union[str, None] = "c19d7d385560" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "workflow_run_blocks", + sa.Column("script_run", sa.JSON(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("workflow_run_blocks", "script_run") diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index 0a6f07b5b..057be95a3 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -13343,6 +13343,15 @@ "type": "string", "nullable": true, "title": "Executed Branch Next Block" + }, + "script_run": { + "allOf": [ + { + "$ref": "#/components/schemas/ScriptRunResponse" + } + ], + "nullable": true, + "title": "Script Run" } }, "type": "object", diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.json index 29a7dc6b2..f1f9f3de3 100644 --- a/fern/openapi/skyvern_openapi.json +++ b/fern/openapi/skyvern_openapi.json @@ -19487,7 +19487,8 @@ { "type": "null" } - ] + ], + "title": "Script Run" }, "job_id": { "anyOf": [ @@ -20054,6 +20055,17 @@ } ], "title": "Executed Branch Next Block" + }, + "script_run": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScriptRunResponse" + }, + { + "type": "null" + } + ], + "title": "Script Run" } }, "type": "object", diff --git a/skyvern/client/types/workflow_run_block.py b/skyvern/client/types/workflow_run_block.py index 7ce873c33..b47642342 100644 --- a/skyvern/client/types/workflow_run_block.py +++ b/skyvern/client/types/workflow_run_block.py @@ -8,6 +8,7 @@ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from .action import Action from .block_type import BlockType from .run_engine import RunEngine +from .script_run_response import ScriptRunResponse from .workflow_run_block_data_schema import WorkflowRunBlockDataSchema from .workflow_run_block_navigation_payload import WorkflowRunBlockNavigationPayload from .workflow_run_block_output import WorkflowRunBlockOutput @@ -55,6 +56,7 @@ class WorkflowRunBlock(UniversalBaseModel): executed_branch_expression: typing.Optional[str] = None executed_branch_result: typing.Optional[bool] = None executed_branch_next_block: typing.Optional[str] = None + script_run: typing.Optional[ScriptRunResponse] = None if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py index 707f5a8e4..5b4dbd1c8 100644 --- a/skyvern/forge/sdk/db/models.py +++ b/skyvern/forge/sdk/db/models.py @@ -827,6 +827,13 @@ class WorkflowRunBlockModel(Base): # Accumulates LLM cost for block-scoped calls (no step/thought attribution). llm_cost = Column(Numeric, default=0, nullable=False) + # Per-block cached-script execution state. Written (via the writer bridge + # in `services/script_service.py::_update_workflow_block`) when a script + # block falls back to AI mid-execution. Always null for blocks that ran + # cleanly from cache or were always-agent. Mirrors the `script_run` + # column on `WorkflowRunModel` but at block granularity. + script_run = Column(JSON, nullable=True) + created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.py index fdb1a3ca9..d0cbc86e5 100644 --- a/skyvern/forge/sdk/db/repositories/observer.py +++ b/skyvern/forge/sdk/db/repositories/observer.py @@ -28,7 +28,7 @@ from skyvern.forge.sdk.db.utils import ( ) from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status, Thought, ThoughtType from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock -from skyvern.schemas.runs import ProxyLocationInput, RunEngine +from skyvern.schemas.runs import ProxyLocationInput, RunEngine, ScriptRunResponse from skyvern.schemas.workflows import BlockStatus, BlockType LOG = structlog.get_logger() @@ -550,7 +550,9 @@ class ObserverRepository(BaseRepository): if http_request_follow_redirects is not None: workflow_run_block.http_request_follow_redirects = http_request_follow_redirects if ai_fallback_triggered is not None: - workflow_run_block.script_run = {"ai_fallback_triggered": ai_fallback_triggered} + workflow_run_block.script_run = ScriptRunResponse( + ai_fallback_triggered=ai_fallback_triggered + ).model_dump(mode="json") if error_codes is not None: workflow_run_block.error_codes = error_codes # human interaction block fields diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py index a05f36d71..eac4ed7ed 100644 --- a/skyvern/forge/sdk/db/utils.py +++ b/skyvern/forge/sdk/db/utils.py @@ -686,6 +686,9 @@ def convert_to_workflow_run_block( executed_branch_expression=workflow_run_block_model.executed_branch_expression, executed_branch_result=workflow_run_block_model.executed_branch_result, executed_branch_next_block=workflow_run_block_model.executed_branch_next_block, + script_run=ScriptRunResponse.model_validate(workflow_run_block_model.script_run) + if workflow_run_block_model.script_run + else None, ) if task: if task.finished_at and task.started_at: diff --git a/skyvern/forge/sdk/schemas/workflow_runs.py b/skyvern/forge/sdk/schemas/workflow_runs.py index a5cd9fdaa..451216aaa 100644 --- a/skyvern/forge/sdk/schemas/workflow_runs.py +++ b/skyvern/forge/sdk/schemas/workflow_runs.py @@ -7,7 +7,7 @@ from typing import Any from pydantic import BaseModel from skyvern.forge.sdk.schemas.task_v2 import Thought -from skyvern.schemas.runs import RunEngine +from skyvern.schemas.runs import RunEngine, ScriptRunResponse from skyvern.schemas.workflows import BlockType from skyvern.webeye.actions.actions import Action @@ -66,6 +66,10 @@ class WorkflowRunBlock(BaseModel): executed_branch_result: bool | None = None executed_branch_next_block: str | None = None + # Set when a script→AI fallback fires on this block. `None` covers + # clean cached execution, always-agent blocks, and pre-column rows. + script_run: ScriptRunResponse | None = None + class WorkflowRunTimelineType(StrEnum): thought = "thought" diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py index 63f682214..b8d36dc31 100644 --- a/skyvern/schemas/runs.py +++ b/skyvern/schemas/runs.py @@ -590,6 +590,10 @@ class BlockRunRequest(WorkflowRunRequest): class ScriptRunResponse(BaseModel): + # `extra="ignore"` is the Pydantic v2 default; making it explicit + # pins the forward-compat guarantee (unknown keys silently dropped). + model_config = ConfigDict(extra="ignore") + # True iff a fallback fired during this run, flipping at least one # block's execution from cached script to the agent. Writers: the two # `services/script_service.py` fallback paths (script-block failure + diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py index 18fe2883a..90e3d08df 100644 --- a/skyvern/services/script_service.py +++ b/skyvern/services/script_service.py @@ -630,9 +630,14 @@ async def _update_workflow_block( label: str | None = None, failure_reason: str | None = None, output: dict[str, Any] | list | str | None = None, - ai_fallback_triggered: bool = False, + ai_fallback_triggered: bool | None = None, ) -> None: - """Update the status of a workflow run block.""" + """Update workflow_run_block status, optionally setting `script_run`. + + `ai_fallback_triggered` is three-valued: `None` = no assertion (no + write); `True`/`False` = explicit fallback signal, written to + `workflow_run_blocks.script_run` as `{"ai_fallback_triggered": }`. + """ try: context = skyvern_context.current() if not context or not context.organization_id or not context.workflow_run_id or not context.workflow_id: @@ -721,7 +726,9 @@ async def _update_workflow_block( ) if step_for_billing: try: - if not ai_fallback_triggered: + # Explicit `is not True` — `None` means "caller made no + # assertion" and falls through to billing like False. + if ai_fallback_triggered is not True: await app.AGENT_FUNCTION.post_cache_step_execution( updated_task, step_for_billing, @@ -748,6 +755,7 @@ async def _update_workflow_block( status=status, failure_reason=failure_reason, output=final_output, + ai_fallback_triggered=ai_fallback_triggered, ) await _record_output_parameter_value( @@ -1173,6 +1181,8 @@ async def _fallback_to_ai_run( task_failure_reason = f"{task_failure_reason}. Detected errors: {', '.join(error_codes)}" if workflow_run_block_id: + # No `ai_fallback_triggered` here — the script step failed + # before the AI agent ran, so no fallback actually fired. await _update_workflow_block( workflow_run_block_id, BlockStatus.failed, @@ -1418,6 +1428,7 @@ async def _fallback_to_ai_run( task_status=TaskStatus.failed, label=cache_key, failure_reason=str(e), + ai_fallback_triggered=True, ) raise e diff --git a/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py b/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py new file mode 100644 index 000000000..3c981213e --- /dev/null +++ b/tests/unit/workflow/test_workflow_run_block_script_run_exposure.py @@ -0,0 +1,124 @@ +"""`workflow_run_blocks.script_run` surfaces on the `WorkflowRunBlock` +schema, so timeline consumers can distinguish cached-execution from +script-to-AI fallback without hitting Datadog. +""" + +from __future__ import annotations + +from datetime import datetime +from unittest.mock import MagicMock + +from skyvern.forge.sdk.db.models import WorkflowRunBlockModel +from skyvern.forge.sdk.db.utils import convert_to_workflow_run_block +from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock +from skyvern.schemas.runs import ScriptRunResponse + + +def _fake_block_model(script_run_value: dict | None) -> MagicMock: + """Build a `WorkflowRunBlockModel` stub that only carries the attrs + `convert_to_workflow_run_block` touches. Avoids standing up a DB or + constructing the SQLAlchemy model (which requires a session).""" + now = datetime.utcnow() + model = MagicMock(spec=WorkflowRunBlockModel) + model.workflow_run_block_id = "wrb_test" + model.workflow_run_id = "wr_test" + model.block_workflow_run_id = None + model.organization_id = "o_test" + model.parent_workflow_run_block_id = None + model.description = None + model.block_type = "navigation" + model.label = "nav_block" + model.status = "completed" + model.output = None + model.continue_on_failure = False + model.failure_reason = None + model.error_codes = None + model.engine = None + model.task_id = None + model.loop_values = None + model.current_value = None + model.current_index = None + model.recipients = None + model.attachments = None + model.subject = None + model.body = None + model.created_at = now + model.modified_at = now + model.instructions = None + model.positive_descriptor = None + model.negative_descriptor = None + model.executed_branch_id = None + model.executed_branch_expression = None + model.executed_branch_result = None + model.executed_branch_next_block = None + model.script_run = script_run_value + return model + + +def test_schema_accepts_script_run_field() -> None: + """WorkflowRunBlock pydantic model accepts a ScriptRunResponse for + its new `script_run` field.""" + now = datetime.utcnow() + block = WorkflowRunBlock( + workflow_run_block_id="wrb_test", + workflow_run_id="wr_test", + organization_id="o_test", + block_type="navigation", + created_at=now, + modified_at=now, + script_run=ScriptRunResponse(ai_fallback_triggered=True), + ) + assert block.script_run is not None + assert block.script_run.ai_fallback_triggered is True + + +def test_schema_defaults_script_run_to_none() -> None: + """Omitting the new field yields None — backward-compat for existing + callers / serialized rows that predate this change.""" + now = datetime.utcnow() + block = WorkflowRunBlock( + workflow_run_block_id="wrb_test", + workflow_run_id="wr_test", + organization_id="o_test", + block_type="navigation", + created_at=now, + modified_at=now, + ) + assert block.script_run is None + + +def test_converter_passes_script_run_when_fallback_recorded() -> None: + """The DB writer at `observer.py:~492` stores + `{"ai_fallback_triggered": True}` in the column when a script→AI + fallback fires for this block. Consumers hitting the timeline API + must now see that as a populated `ScriptRunResponse`.""" + model = _fake_block_model(script_run_value={"ai_fallback_triggered": True}) + block = convert_to_workflow_run_block(model) + assert block.script_run is not None + assert block.script_run.ai_fallback_triggered is True + + +def test_converter_propagates_null_script_run() -> None: + """Null DB column → None in the pydantic model. A block that ran + cleanly from cache (or always-agent) never gets the column written.""" + model = _fake_block_model(script_run_value=None) + block = convert_to_workflow_run_block(model) + assert block.script_run is None + + +def test_converter_preserves_unknown_future_script_run_keys() -> None: + """Pins our forward-compat contract: `ScriptRunResponse` ignores extra + keys (set explicitly via `model_config = ConfigDict(extra="ignore")`). + Block rows persisted with future-added keys still round-trip cleanly + instead of raising — guards against the contract being flipped + accidentally to `extra="forbid"` or `"allow"`. + """ + model = _fake_block_model( + script_run_value={ + "ai_fallback_triggered": False, + "future_field": "ignored", + }, + ) + block = convert_to_workflow_run_block(model) + assert block.script_run is not None + assert block.script_run.ai_fallback_triggered is False