diff --git a/fern/docs/changelog.mdx b/fern/docs/changelog.mdx index efa473ca4..c0297da06 100644 --- a/fern/docs/changelog.mdx +++ b/fern/docs/changelog.mdx @@ -56,6 +56,7 @@ mode: custom - Fixed code block runs not generating a recording entry, causing the Recording tab to appear empty. ([#6607](https://github.com/Skyvern-AI/skyvern/pull/6607)) - Fixed code block action screenshots not appearing on the run Overview page. ([#6605](https://github.com/Skyvern-AI/skyvern/pull/6605)) - Fixed long usernames or email addresses hiding the sidebar collapse button on desktop. +- Fixed the File Parser block crashing when a downloaded file isn't a valid PDF (e.g. a site returns a short plain-text error instead of the expected document); the block now surfaces a clear parse error instead of failing the run unexpectedly. diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py index 72bafabb2..0ffcd664a 100644 --- a/skyvern/forge/sdk/copilot/agent.py +++ b/skyvern/forge/sdk/copilot/agent.py @@ -129,6 +129,7 @@ from skyvern.forge.sdk.copilot.turn_intent import ( from skyvern.forge.sdk.copilot.turn_outcome import ( apply_repeated_reply_guard, derive_response_kind, + with_copilot_code_mode_diagnostics, ) from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status @@ -949,6 +950,8 @@ def _make_agent_result( if payload_updates: kwargs["narrative_payload"] = {**narrative_payload, **payload_updates} result = AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs) + if ctx is not None and result.turn_outcome is not None: + result.turn_outcome = with_copilot_code_mode_diagnostics(result.turn_outcome, ctx) if ctx is not None and result.completion_criteria_turn_state is None: result.completion_criteria_turn_state = getattr(ctx, "completion_criteria_turn_state", None) return result diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py index 04c8dff28..bff83b40f 100644 --- a/skyvern/forge/sdk/copilot/context.py +++ b/skyvern/forge/sdk/copilot/context.py @@ -408,6 +408,7 @@ class CopilotContext(AgentContext): code_only_code_schema_seen: bool = False code_only_target_page_evidence_seen: bool = False last_failed_workflow_yaml: str | None = None + code_native_pending_capability: str | None = None # Set when a block-running tool timed out and the run's true outcome # could not be reconciled (post-drain row was ``canceled``, non-final, or # unreadable). Blocks further block-running tool calls until the LLM diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py index 38405b00b..525684c29 100644 --- a/skyvern/forge/sdk/copilot/runtime.py +++ b/skyvern/forge/sdk/copilot/runtime.py @@ -197,6 +197,7 @@ class AgentContext: last_failed_workflow_yaml: str | None = None code_only_code_schema_seen: bool = False code_only_target_page_evidence_seen: bool = False + code_native_pending_capability: str | None = None repeated_failure_streak_count: int = 0 repeated_failure_nudge_emitted_at_streak: int = 0 challenge_gated_proxy_retry_count: int = 0 diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py index 467760a8f..b036b3b95 100644 --- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py +++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py @@ -176,6 +176,15 @@ def _render_block_policy_detail(block_type: str, policy: CopilotBlockPolicy) -> return f"`{block_type}` is {policy.status.value} and requires {policy.required_capability}. {policy.guidance}" +def _record_code_native_pending_capability(ctx: AgentContext | None, policy: CopilotBlockPolicy) -> None: + if ( + ctx is not None + and policy.status == CopilotBlockPolicyStatus.CODE_NATIVE_PENDING + and ctx.code_native_pending_capability is None + ): + ctx.code_native_pending_capability = policy.required_capability + + def _code_only_browser_unavailable_types() -> list[str]: return sorted( block_type @@ -273,6 +282,7 @@ def _banned_block_reject_message(items: list[tuple[str, str]], ctx: AgentContext if policy_entry is None: continue _normalized, policy = policy_entry + _record_code_native_pending_capability(ctx, policy) type_labels = ", ".join(sorted(grouped[block_type])) details.append(f"{block_type} [{type_labels}]: {_render_block_policy_detail(block_type, policy)}") details_part = " ".join(details) diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py index be89e9349..fa10b8f29 100644 --- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py +++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py @@ -31,6 +31,7 @@ from .banned_blocks import ( _copilot_banned_block_types, _copilot_block_authoring_policy, _copilot_block_policy, + _record_code_native_pending_capability, _render_block_policy_detail, ) from .completion import _maybe_run_completion_verification_from_page_observation @@ -90,6 +91,7 @@ async def _get_block_schema_pre_hook( if policy_entry is None: return None normalized, policy = policy_entry + _record_code_native_pending_capability(ctx, policy) return { "ok": False, "error": ( @@ -131,6 +133,7 @@ async def _validate_block_pre_hook( if policy_entry is None: return None normalized, policy = policy_entry + _record_code_native_pending_capability(ctx, policy) return { "ok": False, "error": ( diff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.py index 0ff906c16..42ad6555d 100644 --- a/skyvern/forge/sdk/copilot/turn_outcome.py +++ b/skyvern/forge/sdk/copilot/turn_outcome.py @@ -9,7 +9,7 @@ direction allowed. from __future__ import annotations from collections.abc import Iterable -from typing import Any +from typing import Any, Literal import structlog @@ -20,6 +20,7 @@ from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOut LOG = structlog.get_logger() IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON = "identical_reply_blocked" +CopilotComposerMode = Literal["ask", "build", "code"] REPEATED_REPLY_ESCALATION_TEMPLATES: dict[ResponseKind, str] = { ResponseKind.BUILD: ( @@ -162,6 +163,43 @@ def _dedup_signatures(signatures: Iterable[str]) -> list[str]: return sorted({sig for sig in signatures if isinstance(sig, str) and sig}) +def _string_or_none(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, bool | str | None]: + turn_halt_kind = getattr(getattr(ctx, "turn_halt", None), "kind", None) + turn_halt_kind_value = getattr(turn_halt_kind, "value", turn_halt_kind) + pending_capability = _string_or_none(getattr(ctx, "code_native_pending_capability", None)) + return { + "copilot_last_code_build_failed": bool( + getattr(ctx, "last_test_ok", None) is False or getattr(ctx, "last_failed_workflow_yaml", None) + ), + "copilot_repair_ceiling_hit": turn_halt_kind_value == "repair_ceiling_reached", + "copilot_pending_capability": pending_capability, + } + + +def with_copilot_code_mode_diagnostics(outcome: TurnOutcome, ctx: Any) -> TurnOutcome: + return outcome.model_copy(update=derive_copilot_code_mode_diagnostics(ctx)) + + +def with_copilot_code_mode_metadata( + outcome: TurnOutcome, + *, + effective_mode: CopilotComposerMode, + code_available: bool, + turn_id: str | None, +) -> TurnOutcome: + return outcome.model_copy( + update={ + "copilot_effective_mode": effective_mode, + "copilot_code_available": code_available, + "copilot_turn_id": turn_id, + } + ) + + def build_minimal_turn_outcome( final_text: str, response_kind: ResponseKind, diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py index 4189f6a61..719eeca6c 100644 --- a/skyvern/forge/sdk/routes/workflow_copilot.py +++ b/skyvern/forge/sdk/routes/workflow_copilot.py @@ -16,6 +16,7 @@ from opentelemetry import trace as otel_trace from pydantic import ValidationError from sse_starlette import EventSourceResponse +from skyvern import analytics from skyvern.config import settings from skyvern.constants import DEFAULT_LOGIN_PROMPT from skyvern.forge import app @@ -35,7 +36,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import ( criteria_to_json, plan_persistence, ) -from skyvern.forge.sdk.copilot.config import CopilotConfig +from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig, normalize_block_authoring_policy from skyvern.forge.sdk.copilot.context import AgentResult, ProposalDisposition, TurnNarrativePayload from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler from skyvern.forge.sdk.copilot.output_utils import truncate_output @@ -45,10 +46,15 @@ from skyvern.forge.sdk.copilot.recoverable_failure import ( format_recoverable_failure_reply, merge_failure_into_context, ) +from skyvern.forge.sdk.copilot.turn_outcome import ( + CopilotComposerMode, + build_minimal_turn_outcome, + with_copilot_code_mode_metadata, +) from skyvern.forge.sdk.core import skyvern_context from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStream from skyvern.forge.sdk.routes.routers import base_router -from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome +from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome from skyvern.forge.sdk.schemas.organizations import Organization from skyvern.forge.sdk.schemas.workflow_copilot import ( WorkflowCopilotApplyProposedWorkflowRequest, @@ -182,6 +188,155 @@ def bind_copilot_session_id(chat_id: str | None) -> Iterator[None]: ctx.copilot_session_id = prev +COPILOT_CODE_MODE_OPT_OUT_EVENT = "copilot_code_mode_opt_out" +COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON = "copilot_recoverable_failure" + + +def _effective_copilot_composer_mode( + chat_request: WorkflowCopilotChatRequest, + *, + uses_v2: bool, + code_mode_fallback: bool = False, +) -> CopilotComposerMode: + if chat_request.mode == "ask": + return "ask" + if chat_request.mode == "build": + return "code" if chat_request.code_block is True else "build" + if uses_v2: + if chat_request.code_block is not None: + return "code" if chat_request.code_block is True else "build" + return "code" if code_mode_fallback else "build" + return "ask" + + +def _latest_assistant_turn_outcome(chat_messages: list[WorkflowCopilotChatMessage]) -> TurnOutcome | None: + for message in reversed(chat_messages): + if message.sender == WorkflowCopilotChatSender.AI and message.turn_outcome is not None: + return message.turn_outcome + return None + + +def _should_emit_copilot_code_mode_opt_out( + *, + prior_turn_outcome: TurnOutcome | None, + to_mode: CopilotComposerMode, + current_code_available: bool, +) -> bool: + if prior_turn_outcome is None: + return False + from_mode = prior_turn_outcome.copilot_effective_mode + if from_mode is None or from_mode == to_mode: + return False + if from_mode == "code" and to_mode != "code": + return True + return ( + from_mode == "build" + and to_mode == "ask" + and (prior_turn_outcome.copilot_code_available or current_code_available) + ) + + +def _reason_category_for_copilot_code_mode_opt_out( + prior_turn_outcome: TurnOutcome, +) -> str: + if ( + prior_turn_outcome.copilot_last_code_build_failed + or prior_turn_outcome.copilot_repair_ceiling_hit + or prior_turn_outcome.terminal_reason == COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON + ): + return "failure" + if prior_turn_outcome.copilot_pending_capability: + return "missing_capability" + return "confusion" + + +def _capture_copilot_code_mode_opt_out( + *, + prior_turn_outcome: TurnOutcome | None, + to_mode: CopilotComposerMode, + current_code_available: bool, + workflow_copilot_chat_id: str, + workflow_permanent_id: str, + organization_id: str, + turn_id: str | None, +) -> None: + if prior_turn_outcome is None or not _should_emit_copilot_code_mode_opt_out( + prior_turn_outcome=prior_turn_outcome, + to_mode=to_mode, + current_code_available=current_code_available, + ): + return + try: + analytics.capture( + COPILOT_CODE_MODE_OPT_OUT_EVENT, + data={ + "from_mode": prior_turn_outcome.copilot_effective_mode, + "to_mode": to_mode, + "reason_category": _reason_category_for_copilot_code_mode_opt_out(prior_turn_outcome), + "last_code_build_failed": prior_turn_outcome.copilot_last_code_build_failed, + "repair_ceiling_hit": prior_turn_outcome.copilot_repair_ceiling_hit, + "pending_capability": prior_turn_outcome.copilot_pending_capability, + "org_id": organization_id, + "workflow_permanent_id": workflow_permanent_id, + "workflow_copilot_chat_id": workflow_copilot_chat_id, + "turn_id": turn_id, + "prior_turn_id": prior_turn_outcome.copilot_turn_id, + }, + distinct_id=workflow_copilot_chat_id, + ) + except Exception: + LOG.warning( + "Failed to capture copilot code mode opt-out event", + workflow_copilot_chat_id=workflow_copilot_chat_id, + organization_id=organization_id, + exc_info=True, + ) + + +async def _resolve_copilot_code_available( + organization_id: str, + chat_request: WorkflowCopilotChatRequest, +) -> bool: + try: + has_code_block_access = await app.AGENT_FUNCTION.has_code_block_access(organization_id) + except Exception: + LOG.warning("Failed to resolve copilot code block access", organization_id=organization_id, exc_info=True) + return False + if not has_code_block_access: + return False + if chat_request.code_block is not None: + return True + try: + copilot_config = await app.AGENT_FUNCTION.get_copilot_config_for_request( + organization_id, + code_block_mode=None, + ) + except Exception: + LOG.warning("Failed to resolve copilot code mode availability", organization_id=organization_id, exc_info=True) + return False + return ( + normalize_block_authoring_policy(getattr(copilot_config, "block_authoring_policy", None)) + == BlockAuthoringPolicy.CODE_ONLY_BROWSER + ) + + +def _with_current_copilot_code_mode_metadata( + turn_outcome: TurnOutcome | None, + *, + effective_mode: CopilotComposerMode, + code_available: bool, + turn_id: str | None, +) -> TurnOutcome | None: + if turn_outcome is None: + return None + return with_copilot_code_mode_metadata( + turn_outcome, + effective_mode=effective_mode, + code_available=code_available, + turn_id=turn_id, + ) + + @dataclass(frozen=True) class RunInfo: block_label: str | None @@ -381,6 +536,12 @@ def _build_recoverable_route_agent_result( clear_proposed_workflow=clear_proposed_workflow, turn_id=turn_id, narrative_payload=_make_error_narrative_payload(turn_id, turn_index, user_response), + turn_outcome=build_minimal_turn_outcome( + user_response, + response_kind=ResponseKind.RECOVER, + reason_code=failure.failure_kind, + terminal_reason=COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON, + ), ) _record_recoverable_failure_span_attrs(failure, proposal_disposition="no_proposal") return agent_result, failure @@ -1606,6 +1767,23 @@ async def _new_copilot_chat_post( global_llm_context: str | None = None terminal_frame_emitted = False cancel_watcher: asyncio.Task[None] | None = None + current_code_available = False + effective_mode = _effective_copilot_composer_mode(chat_request, uses_v2=True) + prior_turn_outcome: TurnOutcome | None = None + + def capture_code_mode_opt_out_after_persist() -> None: + if chat is None: + return + _capture_copilot_code_mode_opt_out( + prior_turn_outcome=prior_turn_outcome, + to_mode=effective_mode, + current_code_available=current_code_available, + workflow_copilot_chat_id=chat.workflow_copilot_chat_id, + workflow_permanent_id=chat.workflow_permanent_id, + organization_id=organization.organization_id, + turn_id=turn_id, + ) + # Single-element list used as a closure flag (mutable bool by reference). # The watcher sets [0] = True before issuing handler_task.cancel() so the # except CancelledError block can distinguish a user-driven cancel from @@ -1637,6 +1815,16 @@ async def _new_copilot_chat_post( chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages( workflow_copilot_chat_id=chat.workflow_copilot_chat_id, ) + prior_turn_outcome = _latest_assistant_turn_outcome(chat_messages) + current_code_available = await _resolve_copilot_code_available( + organization.organization_id, + chat_request, + ) + effective_mode = _effective_copilot_composer_mode( + chat_request, + uses_v2=True, + code_mode_fallback=current_code_available, + ) for message in reversed(chat_messages): if message.global_llm_context is not None: global_llm_context = message.global_llm_context @@ -1790,6 +1978,13 @@ async def _new_copilot_chat_post( stored_completion_criteria=stored_completion_criteria, ) + agent_result.turn_outcome = _with_current_copilot_code_mode_metadata( + agent_result.turn_outcome, + effective_mode=effective_mode, + code_available=current_code_available, + turn_id=turn_id, + ) + if getattr(agent_result, "cancelled", False): # The agent absorbed the CancelledError and returned a result # carrying ``workflow_was_persisted`` so rollback proceeds normally. @@ -1804,6 +1999,7 @@ async def _new_copilot_chat_post( turn_id=turn_id, ) terminal_frame_emitted = True + capture_code_mode_opt_out_after_persist() LOG.info( "Workflow copilot v2 cancelled by user", workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id, @@ -1824,6 +2020,7 @@ async def _new_copilot_chat_post( ) ) terminal_frame_emitted = True + capture_code_mode_opt_out_after_persist() except HTTPException as exc: if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result): try: @@ -1864,6 +2061,12 @@ async def _new_copilot_chat_post( turn_id=turn_id, turn_index=turn_index, ) + recovered_result.turn_outcome = _with_current_copilot_code_mode_metadata( + recovered_result.turn_outcome, + effective_mode=effective_mode, + code_available=current_code_available, + turn_id=turn_id, + ) LOG.error( "LLM provider error translated to recoverable workflow copilot v2 reply", organization_id=organization.organization_id, @@ -1884,6 +2087,7 @@ async def _new_copilot_chat_post( ) ) terminal_frame_emitted = True + capture_code_mode_opt_out_after_persist() else: LOG.error( "LLM provider error (copilot v2)", @@ -1965,6 +2169,12 @@ async def _new_copilot_chat_post( turn_id=turn_id, turn_index=turn_index, ) + recovered_result.turn_outcome = _with_current_copilot_code_mode_metadata( + recovered_result.turn_outcome, + effective_mode=effective_mode, + code_available=current_code_available, + turn_id=turn_id, + ) LOG.error( "Unexpected workflow copilot v2 error translated to recoverable reply", organization_id=organization.organization_id, @@ -1985,6 +2195,7 @@ async def _new_copilot_chat_post( ) ) terminal_frame_emitted = True + capture_code_mode_opt_out_after_persist() else: LOG.error( "Unexpected error in workflow copilot v2", @@ -2164,6 +2375,7 @@ async def workflow_copilot_chat_post( return await _new_copilot_chat_post(request, chat_request, organization) async def stream_handler(stream: EventSourceStream) -> None: + turn_id = uuid.uuid4().hex LOG.info( "Workflow copilot chat request", workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id, @@ -2198,6 +2410,16 @@ async def workflow_copilot_chat_post( chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages( workflow_copilot_chat_id=chat.workflow_copilot_chat_id, ) + prior_turn_outcome = _latest_assistant_turn_outcome(chat_messages) + current_code_available = await _resolve_copilot_code_available( + organization.organization_id, + chat_request, + ) + effective_mode = _effective_copilot_composer_mode( + chat_request, + uses_v2=False, + code_mode_fallback=current_code_available, + ) global_llm_context = None for message in reversed(chat_messages): if message.global_llm_context is not None: @@ -2243,6 +2465,16 @@ async def workflow_copilot_chat_post( debug_run_info_text, ) + legacy_turn_outcome = _with_current_copilot_code_mode_metadata( + build_minimal_turn_outcome( + user_response, + response_kind=ResponseKind.CLARIFY if effective_mode == "ask" else ResponseKind.BUILD, + ), + effective_mode=effective_mode, + code_available=current_code_available, + turn_id=turn_id, + ) + if updated_workflow and chat.auto_accept is not True: proposed_data = updated_workflow.model_dump(mode="json") # _copilot_yaml is what /apply-proposed-workflow re-parses into @@ -2269,8 +2501,18 @@ async def workflow_copilot_chat_post( sender=WorkflowCopilotChatSender.AI, content=user_response, global_llm_context=updated_global_llm_context, + turn_outcome=legacy_turn_outcome, ) + _capture_copilot_code_mode_opt_out( + prior_turn_outcome=prior_turn_outcome, + to_mode=effective_mode, + current_code_available=current_code_available, + workflow_copilot_chat_id=chat.workflow_copilot_chat_id, + workflow_permanent_id=chat.workflow_permanent_id, + organization_id=organization.organization_id, + turn_id=turn_id, + ) terminal_frame_emitted = True await stream.send( WorkflowCopilotStreamResponseUpdate( @@ -2318,7 +2560,7 @@ async def workflow_copilot_chat_post( ) ) finally: - await _ensure_terminal_frame(stream, terminal_frame_emitted) + await _ensure_terminal_frame(stream, terminal_frame_emitted, turn_id=turn_id) return FastAPIEventSourceStream.create(request, stream_handler) diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py index 8a6bc4871..b52dc73ac 100644 --- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py +++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py @@ -8,7 +8,7 @@ in any ``copilot/`` business logic — derivation lives in from __future__ import annotations from enum import StrEnum -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -33,3 +33,9 @@ class TurnOutcome(BaseModel): tool_calls: list[str] = Field(default_factory=list) terminal_reason: str | None = None blocked_signatures: list[str] = Field(default_factory=list) + copilot_effective_mode: Literal["ask", "build", "code"] | None = None + copilot_code_available: bool = False + copilot_last_code_build_failed: bool = False + copilot_repair_ceiling_hit: bool = False + copilot_pending_capability: str | None = None + copilot_turn_id: str | None = None diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py index a6ff3f9f7..7bacdcf1d 100644 --- a/skyvern/forge/sdk/workflow/models/block.py +++ b/skyvern/forge/sdk/workflow/models/block.py @@ -5904,31 +5904,40 @@ class FileParserBlock(Block): # Parse the file based on type parsed_data: str | list[dict[str, Any]] - if self.file_type == FileType.CSV: - parsed_data = await self._parse_csv_file(file_path) - elif self.file_type == FileType.EXCEL: - parsed_data = await self._parse_excel_file(file_path) - elif self.file_type == FileType.PDF: - parsed_data = await self._parse_pdf_file( - file_path, - workflow_run_block_id=workflow_run_block_id, - organization_id=organization_id, - ) - elif self.file_type == FileType.IMAGE: - parsed_data = await self._parse_image_file( - file_path, - workflow_run_block_id=workflow_run_block_id, - organization_id=organization_id, - ) - elif self.file_type == FileType.DOCX: - parsed_data = await self._parse_docx_file(file_path) - else: + try: + if self.file_type == FileType.CSV: + parsed_data = await self._parse_csv_file(file_path) + elif self.file_type == FileType.EXCEL: + parsed_data = await self._parse_excel_file(file_path) + elif self.file_type == FileType.PDF: + parsed_data = await self._parse_pdf_file( + file_path, + workflow_run_block_id=workflow_run_block_id, + organization_id=organization_id, + ) + elif self.file_type == FileType.IMAGE: + parsed_data = await self._parse_image_file( + file_path, + workflow_run_block_id=workflow_run_block_id, + organization_id=organization_id, + ) + elif self.file_type == FileType.DOCX: + parsed_data = await self._parse_docx_file(file_path) + else: + return await self._record_failure( + workflow_run_context, + workflow_run_id, + workflow_run_block_id, + organization_id, + f"Unsupported file type: {self.file_type}", + ) + except Exception as e: return await self._record_failure( workflow_run_context, workflow_run_id, workflow_run_block_id, organization_id, - f"Unsupported file type: {self.file_type}", + f"Failed to parse {self.file_type} file: {str(e)}", ) # If json_schema is provided, use AI to extract structured data diff --git a/tests/unit/test_copilot_code_mode_opt_out.py b/tests/unit/test_copilot_code_mode_opt_out.py new file mode 100644 index 000000000..8fbac65e5 --- /dev/null +++ b/tests/unit/test_copilot_code_mode_opt_out.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from skyvern.forge import app +from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig +from skyvern.forge.sdk.copilot.turn_outcome import ( + derive_copilot_code_mode_diagnostics, + with_copilot_code_mode_metadata, +) +from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route +from skyvern.forge.sdk.routes.workflow_copilot import ( + COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON, + _build_recoverable_route_agent_result, + _capture_copilot_code_mode_opt_out, + _effective_copilot_composer_mode, + _reason_category_for_copilot_code_mode_opt_out, + _resolve_copilot_code_available, + _should_emit_copilot_code_mode_opt_out, +) +from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome +from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest + + +def _request(mode: str | None, code_block: bool | None) -> WorkflowCopilotChatRequest: + return WorkflowCopilotChatRequest( + workflow_permanent_id="wpid-1", + workflow_id="wf-1", + workflow_copilot_chat_id="chat-1", + message="message", + workflow_yaml="title: Example", + mode=mode, + code_block=code_block, + ) + + +def _outcome( + *, + mode: str | None, + code_available: bool = True, + last_code_build_failed: bool = False, + repair_ceiling_hit: bool = False, + pending_capability: str | None = None, + turn_id: str | None = "prior-turn", +) -> TurnOutcome: + return TurnOutcome( + response_kind=ResponseKind.BUILD, + copilot_effective_mode=mode, + copilot_code_available=code_available, + copilot_last_code_build_failed=last_code_build_failed, + copilot_repair_ceiling_hit=repair_ceiling_hit, + copilot_pending_capability=pending_capability, + copilot_turn_id=turn_id, + ) + + +@pytest.mark.parametrize( + ("mode", "code_block", "uses_v2", "code_mode_fallback", "expected"), + [ + ("ask", None, False, False, "ask"), + ("build", None, True, True, "build"), + ("build", False, True, True, "build"), + ("build", True, True, False, "code"), + (None, True, True, False, "code"), + (None, False, True, True, "build"), + (None, None, True, False, "build"), + (None, None, True, True, "code"), + (None, None, False, True, "ask"), + ], +) +def test_effective_copilot_composer_mode( + mode: str | None, code_block: bool | None, uses_v2: bool, code_mode_fallback: bool, expected: str +) -> None: + assert ( + _effective_copilot_composer_mode( + _request(mode, code_block), + uses_v2=uses_v2, + code_mode_fallback=code_mode_fallback, + ) + == expected + ) + + +@pytest.mark.parametrize( + ("prior", "to_mode", "current_code_available", "expected"), + [ + (_outcome(mode="code"), "build", True, True), + (_outcome(mode="code"), "ask", False, True), + (_outcome(mode="build", code_available=True), "ask", False, True), + (_outcome(mode="build", code_available=False), "ask", True, True), + (_outcome(mode="build", code_available=False), "ask", False, False), + (_outcome(mode="code"), "code", True, False), + (_outcome(mode="build", code_available=True), "build", True, False), + (_outcome(mode="ask", code_available=True), "build", True, False), + (None, "ask", True, False), + (_outcome(mode=None, code_available=True), "ask", True, False), + ], +) +def test_should_emit_copilot_code_mode_opt_out_transitions( + prior: TurnOutcome | None, + to_mode: str, + current_code_available: bool, + expected: bool, +) -> None: + assert ( + _should_emit_copilot_code_mode_opt_out( + prior_turn_outcome=prior, + to_mode=to_mode, + current_code_available=current_code_available, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("prior", "expected"), + [ + (_outcome(mode="code", last_code_build_failed=True, pending_capability="capability"), "failure"), + (_outcome(mode="code", repair_ceiling_hit=True, pending_capability="capability"), "failure"), + ( + TurnOutcome( + response_kind=ResponseKind.RECOVER, + copilot_effective_mode="code", + terminal_reason=COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON, + copilot_pending_capability="capability", + ), + "failure", + ), + (_outcome(mode="code", pending_capability="capability"), "missing_capability"), + (_outcome(mode="code"), "confusion"), + ], +) +def test_reason_category_for_copilot_code_mode_opt_out(prior: TurnOutcome, expected: str) -> None: + assert _reason_category_for_copilot_code_mode_opt_out(prior) == expected + + +def test_capture_copilot_code_mode_opt_out_uses_chat_id_as_distinct_id(monkeypatch: pytest.MonkeyPatch) -> None: + capture = MagicMock() + monkeypatch.setattr(workflow_copilot_route.analytics, "capture", capture) + + prior = _outcome( + mode="code", + last_code_build_failed=True, + repair_ceiling_hit=False, + pending_capability="credential-typed code synthesis", + turn_id="turn-prior", + ) + + _capture_copilot_code_mode_opt_out( + prior_turn_outcome=prior, + to_mode="ask", + current_code_available=True, + workflow_copilot_chat_id="chat-123", + workflow_permanent_id="wpid-123", + organization_id="org-123", + turn_id="turn-current", + ) + + capture.assert_called_once_with( + "copilot_code_mode_opt_out", + data={ + "from_mode": "code", + "to_mode": "ask", + "reason_category": "failure", + "last_code_build_failed": True, + "repair_ceiling_hit": False, + "pending_capability": "credential-typed code synthesis", + "org_id": "org-123", + "workflow_permanent_id": "wpid-123", + "workflow_copilot_chat_id": "chat-123", + "turn_id": "turn-current", + "prior_turn_id": "turn-prior", + }, + distinct_id="chat-123", + ) + + +def test_capture_copilot_code_mode_opt_out_skips_non_transition(monkeypatch: pytest.MonkeyPatch) -> None: + capture = MagicMock() + monkeypatch.setattr(workflow_copilot_route.analytics, "capture", capture) + + _capture_copilot_code_mode_opt_out( + prior_turn_outcome=_outcome(mode="build", code_available=False), + to_mode="ask", + current_code_available=False, + workflow_copilot_chat_id="chat-123", + workflow_permanent_id="wpid-123", + organization_id="org-123", + turn_id="turn-current", + ) + + capture.assert_not_called() + + +def test_build_recoverable_route_agent_result_sets_failure_turn_outcome() -> None: + agent_result, failure = _build_recoverable_route_agent_result( + RuntimeError("boom"), + workflow_modified=False, + clear_proposed_workflow=False, + global_llm_context=None, + turn_id="turn-error", + turn_index=2, + ) + + assert agent_result.turn_outcome is not None + assert agent_result.turn_outcome.response_kind is ResponseKind.RECOVER + assert agent_result.turn_outcome.reason_code == failure.failure_kind + assert agent_result.turn_outcome.terminal_reason == COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON + assert _reason_category_for_copilot_code_mode_opt_out(agent_result.turn_outcome) == "failure" + + +@pytest.mark.asyncio +async def test_resolve_copilot_code_available_uses_access_and_rollout(monkeypatch: pytest.MonkeyPatch) -> None: + config = CopilotConfig(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER) + agent_function = SimpleNamespace(has_code_block_access=AsyncMock(), get_copilot_config_for_request=AsyncMock()) + monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function) + + agent_function.has_code_block_access.return_value = False + assert await _resolve_copilot_code_available("org-1", _request("build", False)) is False + agent_function.get_copilot_config_for_request.assert_not_awaited() + + agent_function.has_code_block_access.return_value = True + agent_function.get_copilot_config_for_request.return_value = config + assert await _resolve_copilot_code_available("org-1", _request("ask", None)) is True + + +def test_with_copilot_code_mode_metadata_preserves_turn_outcome_fields() -> None: + outcome = TurnOutcome( + response_kind=ResponseKind.CLARIFY, + reason_code="request_policy_clarification", + terminal_reason="terminal", + ) + + updated = with_copilot_code_mode_metadata( + outcome, + effective_mode="build", + code_available=True, + turn_id="turn-123", + ) + + assert updated.response_kind == ResponseKind.CLARIFY + assert updated.reason_code == "request_policy_clarification" + assert updated.terminal_reason == "terminal" + assert updated.copilot_effective_mode == "build" + assert updated.copilot_code_available is True + assert updated.copilot_turn_id == "turn-123" + + +def test_derive_copilot_code_mode_diagnostics_uses_context_state() -> None: + ctx = type("Ctx", (), {})() + ctx.last_test_ok = False + ctx.last_failed_workflow_yaml = None + ctx.code_native_pending_capability = "credential-typed code synthesis" + ctx.turn_halt = type("Halt", (), {"kind": type("Kind", (), {"value": "repair_ceiling_reached"})()})() + + diagnostics = derive_copilot_code_mode_diagnostics(ctx) + + assert diagnostics == { + "copilot_last_code_build_failed": True, + "copilot_repair_ceiling_hit": True, + "copilot_pending_capability": "credential-typed code synthesis", + } diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py index ed23f2aff..ccab8b168 100644 --- a/tests/unit/test_workflow_copilot_route.py +++ b/tests/unit/test_workflow_copilot_route.py @@ -691,6 +691,19 @@ async def test_flag_on_route_error_after_chat_persists_recoverable_reply( assert "The workflow was not modified" in assistant_contents[0] assert "reference cpe_" in assistant_contents[0] + assistant_messages = [ + call.kwargs + for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list + if call.kwargs.get("sender") == WorkflowCopilotChatSender.AI + ] + assert len(assistant_messages) == 1 + turn_outcome = assistant_messages[0]["turn_outcome"] + assert turn_outcome is not None + assert turn_outcome.response_kind == "recover" + assert turn_outcome.terminal_reason == workflow_copilot_route.COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON + assert turn_outcome.copilot_effective_mode == "build" + assert turn_outcome.copilot_turn_id is not None + frames = [call.args[0] for call in stream.send.await_args_list if call.args] response_frames = [frame for frame in frames if isinstance(frame, WorkflowCopilotStreamResponseUpdate)] assert response_frames diff --git a/tests/unit/workflow/test_file_parser_error_code.py b/tests/unit/workflow/test_file_parser_error_code.py index 0640c807f..11b6a044f 100644 --- a/tests/unit/workflow/test_file_parser_error_code.py +++ b/tests/unit/workflow/test_file_parser_error_code.py @@ -16,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from skyvern.forge.sdk.workflow.exceptions import InvalidFileType from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType from skyvern.schemas.workflows import BlockResult, BlockStatus, FileType @@ -190,6 +191,68 @@ class TestFileParserBlockExecuteErrorCodes: assert result.error_codes == ["FILE_PARSER_ERROR"] assert "unsupported" in result.failure_reason.lower() + @pytest.mark.asyncio + async def test_parse_invalid_file_type_returns_error_codes(self) -> None: + """SKY-11131: a file that passes validation but fails parsing (e.g. a non-PDF + that pypdf opens but cannot extract) must surface a graceful failure, not raise.""" + block = _make_file_parser_block(file_url="https://example.com/statement.pdf", file_type=FileType.PDF) + mock_ctx = _mock_workflow_run_context() + + with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx): + with patch.object(FileParserBlock, "format_potential_template_parameters"): + with patch( + "skyvern.forge.sdk.workflow.models.block.download_file", + return_value="/tmp/statement.pdf", + ): + with patch.object(FileParserBlock, "validate_file_type"): + with patch.object( + FileParserBlock, + "_parse_pdf_file", + side_effect=InvalidFileType( + file_url="https://example.com/statement.pdf", + file_type=FileType.PDF, + error="both parsers failed", + ), + ): + result = await block.execute( + workflow_run_id="wr_test", + workflow_run_block_id="wrb_test", + organization_id="org_test", + ) + + assert result.success is False + assert result.error_codes == ["FILE_PARSER_ERROR"] + assert "parse" in result.failure_reason.lower() + + @pytest.mark.asyncio + async def test_parse_unexpected_exception_returns_error_codes(self) -> None: + """SKY-11131: a raw exception from the parse stage (e.g. the vision-LLM PDF + fallback) must be caught and surfaced as a graceful failure.""" + block = _make_file_parser_block(file_url="https://example.com/statement.pdf", file_type=FileType.PDF) + mock_ctx = _mock_workflow_run_context() + + with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx): + with patch.object(FileParserBlock, "format_potential_template_parameters"): + with patch( + "skyvern.forge.sdk.workflow.models.block.download_file", + return_value="/tmp/statement.pdf", + ): + with patch.object(FileParserBlock, "validate_file_type"): + with patch.object( + FileParserBlock, + "_parse_pdf_file", + side_effect=Exception("No /Root object! - Is this really a PDF?"), + ): + result = await block.execute( + workflow_run_id="wr_test", + workflow_run_block_id="wrb_test", + organization_id="org_test", + ) + + assert result.success is False + assert result.error_codes == ["FILE_PARSER_ERROR"] + assert "parse" in result.failure_reason.lower() + @pytest.mark.asyncio async def test_ai_extraction_failure_returns_error_codes(self) -> None: block = _make_file_parser_block(file_url="https://example.com/file.csv") @@ -393,6 +456,37 @@ class TestFileParserBlockRecordsOutputOnFailure: self._assert_failure_output_shape(recorded, "unsupported") assert result.output_parameter_value == recorded + @pytest.mark.asyncio + async def test_parse_failure_records_failed_output(self) -> None: + block = _make_file_parser_block(file_url="https://example.com/statement.pdf", file_type=FileType.PDF) + mock_ctx = _mock_workflow_run_context() + + with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx): + with patch.object(FileParserBlock, "format_potential_template_parameters"): + with patch( + "skyvern.forge.sdk.workflow.models.block.download_file", + return_value="/tmp/statement.pdf", + ): + with patch.object(FileParserBlock, "validate_file_type"): + with patch.object( + FileParserBlock, + "_parse_pdf_file", + side_effect=Exception("No /Root object! - Is this really a PDF?"), + ): + with patch.object( + FileParserBlock, "record_output_parameter_value", new_callable=AsyncMock + ) as mock_record: + result = await block.execute( + workflow_run_id="wr_test", + workflow_run_block_id="wrb_test", + organization_id="org_test", + ) + + assert result.success is False + recorded = self._recorded_value(mock_record) + self._assert_failure_output_shape(recorded, "parse") + assert result.output_parameter_value == recorded + @pytest.mark.asyncio async def test_ai_extraction_failure_records_failed_output(self) -> None: block = _make_file_parser_block(file_url="https://example.com/file.csv")