diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.py index 1cdc63af6..922be3e06 100644 --- a/skyvern/forge/failure_classifier.py +++ b/skyvern/forge/failure_classifier.py @@ -1,5 +1,12 @@ from __future__ import annotations +import re + +# Matches the activity/heartbeat timeout tokens the worker persists when a Temporal-activity +# timeout finalizes a run. Word-anchored so "inactivity timeout" (a distinct page-level reason) +# is NOT caught and correctly stays PAGE_LOAD_TIMEOUT. +_INFRA_TIMEOUT_RE = re.compile(r"\b(?:activity|heartbeat) timeout\b") + def classify_from_failure_reason( failure_reason: str | None, @@ -102,8 +109,21 @@ def classify_from_failure_reason( } ) + # Infrastructure timeout — a Temporal activity / heartbeat timeout finalizes the run from + # the worker layer (a stalled activity or worker interruption), not a site/page-load issue. + # Classify before PAGE_LOAD_TIMEOUT so these don't masquerade as site slowness. + _is_infra_timeout = bool(_INFRA_TIMEOUT_RE.search(reason)) + if _is_infra_timeout: + categories.append( + { + "category": "INFRASTRUCTURE_ERROR", + "confidence_float": 0.9, + "reasoning": "Activity/heartbeat timeout finalized the run", + } + ) + # Page load timeout - if "Timeout" in exc_name or "timeout" in reason: + if ("Timeout" in exc_name or "timeout" in reason) and not _is_infra_timeout: categories.append( { "category": "PAGE_LOAD_TIMEOUT", diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py index 2149d288e..a8ba80b86 100644 --- a/tests/unit/test_failure_classifier.py +++ b/tests/unit/test_failure_classifier.py @@ -214,3 +214,37 @@ def test_keyword_match_ignores_fallback() -> None: result_without = _classify("browser crash detected", fallback_to_unknown=False) assert result_with[0]["category"] == result_without[0]["category"] == "BROWSER_ERROR" + + +def test_activity_heartbeat_timeout_classifies_as_infrastructure() -> None: + reason = "Workflow run timed out: the workflow activity became unresponsive (activity heartbeat timeout)." + categories = _categories_for(reason) + + assert categories[0] == "INFRASTRUCTURE_ERROR" + # An infra-level activity timeout must not masquerade as site/page-load slowness. + assert "PAGE_LOAD_TIMEOUT" not in categories + + +def test_generic_activity_timeout_classifies_as_infrastructure() -> None: + reason = "Workflow run timed out: the workflow activity became unresponsive (activity timeout)." + categories = _categories_for(reason) + + assert categories[0] == "INFRASTRUCTURE_ERROR" + assert "PAGE_LOAD_TIMEOUT" not in categories + + +def test_page_load_timeout_still_classifies_without_activity_context() -> None: + categories = _categories_for("Navigation timeout while waiting for page load") + + assert "PAGE_LOAD_TIMEOUT" in categories + assert "INFRASTRUCTURE_ERROR" not in categories + + +def test_inactivity_timeout_is_not_infrastructure() -> None: + # "inactivity timeout" is a session/page-level reason; a naive substring match on + # "activity timeout" would wrongly reclassify it as INFRASTRUCTURE_ERROR and drop + # PAGE_LOAD_TIMEOUT. The word-anchored match must leave it alone. + categories = _categories_for("Session ended: inactivity timeout on the page") + + assert "INFRASTRUCTURE_ERROR" not in categories + assert "PAGE_LOAD_TIMEOUT" in categories