From 246689715870cf7d0bfa42588af467627238b9c9 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Thu, 16 May 2024 18:20:11 -0700 Subject: [PATCH] adopt ruff as the replacement for python black (#332) --- .pre-commit-config.yaml | 16 +- ...4_03_01_0537-99423c1dec60_create_tables.py | 100 ++++++-- ...10-ffe2f57bd288_create_output_parameter.py | 18 +- ..._create_bitwarden_credential_parameter_.py | 3 +- ...20-68d78072fdb5_add_org_task_step_index.py | 7 +- ...7_add_workflow_permanent_id_constraint_.py | 18 +- poetry.lock | 68 +----- pyproject.toml | 38 ++- scripts/create_organization.py | 5 +- skyvern/exceptions.py | 22 +- skyvern/forge/__main__.py | 8 +- skyvern/forge/agent.py | 99 ++++++-- skyvern/forge/api_app.py | 3 +- skyvern/forge/app.py | 3 +- skyvern/forge/async_operations.py | 9 +- .../forge/sdk/api/llm/api_handler_factory.py | 8 +- skyvern/forge/sdk/api/llm/config_registry.py | 42 +++- skyvern/forge/sdk/api/llm/utils.py | 5 +- skyvern/forge/sdk/artifact/manager.py | 16 +- skyvern/forge/sdk/artifact/storage/local.py | 18 +- skyvern/forge/sdk/db/client.py | 31 ++- skyvern/forge/sdk/db/models.py | 112 +++++++-- skyvern/forge/sdk/db/utils.py | 30 ++- skyvern/forge/sdk/executor/async_executor.py | 5 +- skyvern/forge/sdk/models.py | 7 +- skyvern/forge/sdk/prompting.py | 21 +- skyvern/forge/sdk/routes/agent_protocol.py | 60 ++++- skyvern/forge/sdk/schemas/tasks.py | 20 +- skyvern/forge/sdk/services/bitwarden.py | 22 +- .../forge/sdk/services/org_auth_service.py | 8 +- .../sdk/services/org_auth_token_service.py | 2 +- skyvern/forge/sdk/workflow/context_manager.py | 10 +- skyvern/forge/sdk/workflow/models/block.py | 63 +++-- .../forge/sdk/workflow/models/parameter.py | 6 +- skyvern/forge/sdk/workflow/service.py | 86 +++++-- skyvern/webeye/actions/actions.py | 29 ++- skyvern/webeye/actions/handler.py | 219 ++++++++++++++---- skyvern/webeye/actions/models.py | 2 +- skyvern/webeye/browser_factory.py | 23 +- skyvern/webeye/browser_manager.py | 37 ++- skyvern/webeye/scraper/scraper.py | 6 +- streamlit_app/visualizer/artifact_loader.py | 16 +- streamlit_app/visualizer/sample_data.py | 22 +- streamlit_app/visualizer/streamlit.py | 59 +++-- 44 files changed, 1081 insertions(+), 321 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6242c417..9daa711f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,19 +13,21 @@ repos: - id: check-symlinks - id: debug-statements - id: detect-private-key - + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.4.4 + hooks: + # Run the linter. + - id: ruff + args: [--fix] + # Run the formatter. + - id: ruff-format - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: - id: isort language_version: python3.11 - - repo: https://github.com/psf/black - rev: 24.4.2 - hooks: - - id: black - language_version: python3.11 - - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: diff --git a/alembic/versions/2024_03_01_0537-99423c1dec60_create_tables.py b/alembic/versions/2024_03_01_0537-99423c1dec60_create_tables.py index d12658cc..00139ac0 100644 --- a/alembic/versions/2024_03_01_0537-99423c1dec60_create_tables.py +++ b/alembic/versions/2024_03_01_0537-99423c1dec60_create_tables.py @@ -1,7 +1,7 @@ """Create tables Revision ID: 99423c1dec60 -Revises: +Revises: Create Date: 2024-03-01 05:37:31.862957+00:00 """ @@ -31,12 +31,21 @@ def upgrade() -> None: sa.Column("modified_at", sa.DateTime(), nullable=False), sa.PrimaryKeyConstraint("organization_id"), ) - op.create_index(op.f("ix_organizations_organization_id"), "organizations", ["organization_id"], unique=False) + op.create_index( + op.f("ix_organizations_organization_id"), + "organizations", + ["organization_id"], + unique=False, + ) op.create_table( "organization_auth_tokens", sa.Column("id", sa.String(), nullable=False), sa.Column("organization_id", sa.String(), nullable=False), - sa.Column("token_type", sa.Enum("api", name="organizationauthtokentype"), nullable=False), + sa.Column( + "token_type", + sa.Enum("api", name="organizationauthtokentype"), + nullable=False, + ), sa.Column("token", sa.String(), nullable=False), sa.Column("valid", sa.Boolean(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False), @@ -48,14 +57,24 @@ def upgrade() -> None: ), sa.PrimaryKeyConstraint("id"), ) - op.create_index(op.f("ix_organization_auth_tokens_id"), "organization_auth_tokens", ["id"], unique=False) + op.create_index( + op.f("ix_organization_auth_tokens_id"), + "organization_auth_tokens", + ["id"], + unique=False, + ) op.create_index( op.f("ix_organization_auth_tokens_organization_id"), "organization_auth_tokens", ["organization_id"], unique=False, ) - op.create_index(op.f("ix_organization_auth_tokens_token"), "organization_auth_tokens", ["token"], unique=False) + op.create_index( + op.f("ix_organization_auth_tokens_token"), + "organization_auth_tokens", + ["token"], + unique=False, + ) op.create_table( "workflows", sa.Column("workflow_id", sa.String(), nullable=False), @@ -96,7 +115,10 @@ def upgrade() -> None: unique=False, ) op.create_index( - op.f("ix_aws_secret_parameters_workflow_id"), "aws_secret_parameters", ["workflow_id"], unique=False + op.f("ix_aws_secret_parameters_workflow_id"), + "aws_secret_parameters", + ["workflow_id"], + unique=False, ) op.create_table( "workflow_parameters", @@ -115,7 +137,12 @@ def upgrade() -> None: ), sa.PrimaryKeyConstraint("workflow_parameter_id"), ) - op.create_index(op.f("ix_workflow_parameters_workflow_id"), "workflow_parameters", ["workflow_id"], unique=False) + op.create_index( + op.f("ix_workflow_parameters_workflow_id"), + "workflow_parameters", + ["workflow_id"], + unique=False, + ) op.create_index( op.f("ix_workflow_parameters_workflow_parameter_id"), "workflow_parameters", @@ -129,7 +156,16 @@ def upgrade() -> None: sa.Column("status", sa.String(), nullable=False), sa.Column( "proxy_location", - sa.Enum("US_CA", "US_NY", "US_TX", "US_FL", "US_WA", "RESIDENTIAL", "NONE", name="proxylocation"), + sa.Enum( + "US_CA", + "US_NY", + "US_TX", + "US_FL", + "US_WA", + "RESIDENTIAL", + "NONE", + name="proxylocation", + ), nullable=True, ), sa.Column("webhook_callback_url", sa.String(), nullable=True), @@ -141,7 +177,12 @@ def upgrade() -> None: ), sa.PrimaryKeyConstraint("workflow_run_id"), ) - op.create_index(op.f("ix_workflow_runs_workflow_run_id"), "workflow_runs", ["workflow_run_id"], unique=False) + op.create_index( + op.f("ix_workflow_runs_workflow_run_id"), + "workflow_runs", + ["workflow_run_id"], + unique=False, + ) op.create_table( "tasks", sa.Column("task_id", sa.String(), nullable=False), @@ -156,7 +197,16 @@ def upgrade() -> None: sa.Column("failure_reason", sa.String(), nullable=True), sa.Column( "proxy_location", - sa.Enum("US_CA", "US_NY", "US_TX", "US_FL", "US_WA", "RESIDENTIAL", "NONE", name="proxylocation"), + sa.Enum( + "US_CA", + "US_NY", + "US_TX", + "US_FL", + "US_WA", + "RESIDENTIAL", + "NONE", + name="proxylocation", + ), nullable=True, ), sa.Column("extracted_information_schema", sa.JSON(), nullable=True), @@ -199,7 +249,10 @@ def upgrade() -> None: unique=False, ) op.create_index( - op.f("ix_workflow_run_parameters_workflow_run_id"), "workflow_run_parameters", ["workflow_run_id"], unique=False + op.f("ix_workflow_run_parameters_workflow_run_id"), + "workflow_run_parameters", + ["workflow_run_id"], + unique=False, ) op.create_table( "steps", @@ -261,23 +314,38 @@ def downgrade() -> None: op.drop_table("artifacts") op.drop_index(op.f("ix_steps_step_id"), table_name="steps") op.drop_table("steps") - op.drop_index(op.f("ix_workflow_run_parameters_workflow_run_id"), table_name="workflow_run_parameters") - op.drop_index(op.f("ix_workflow_run_parameters_workflow_parameter_id"), table_name="workflow_run_parameters") + op.drop_index( + op.f("ix_workflow_run_parameters_workflow_run_id"), + table_name="workflow_run_parameters", + ) + op.drop_index( + op.f("ix_workflow_run_parameters_workflow_parameter_id"), + table_name="workflow_run_parameters", + ) op.drop_table("workflow_run_parameters") op.drop_index(op.f("ix_tasks_task_id"), table_name="tasks") op.drop_table("tasks") op.drop_index(op.f("ix_workflow_runs_workflow_run_id"), table_name="workflow_runs") op.drop_table("workflow_runs") - op.drop_index(op.f("ix_workflow_parameters_workflow_parameter_id"), table_name="workflow_parameters") + op.drop_index( + op.f("ix_workflow_parameters_workflow_parameter_id"), + table_name="workflow_parameters", + ) op.drop_index(op.f("ix_workflow_parameters_workflow_id"), table_name="workflow_parameters") op.drop_table("workflow_parameters") op.drop_index(op.f("ix_aws_secret_parameters_workflow_id"), table_name="aws_secret_parameters") - op.drop_index(op.f("ix_aws_secret_parameters_aws_secret_parameter_id"), table_name="aws_secret_parameters") + op.drop_index( + op.f("ix_aws_secret_parameters_aws_secret_parameter_id"), + table_name="aws_secret_parameters", + ) op.drop_table("aws_secret_parameters") op.drop_index(op.f("ix_workflows_workflow_id"), table_name="workflows") op.drop_table("workflows") op.drop_index(op.f("ix_organization_auth_tokens_token"), table_name="organization_auth_tokens") - op.drop_index(op.f("ix_organization_auth_tokens_organization_id"), table_name="organization_auth_tokens") + op.drop_index( + op.f("ix_organization_auth_tokens_organization_id"), + table_name="organization_auth_tokens", + ) op.drop_index(op.f("ix_organization_auth_tokens_id"), table_name="organization_auth_tokens") op.drop_table("organization_auth_tokens") op.drop_index(op.f("ix_organizations_organization_id"), table_name="organizations") diff --git a/alembic/versions/2024_03_22_0010-ffe2f57bd288_create_output_parameter.py b/alembic/versions/2024_03_22_0010-ffe2f57bd288_create_output_parameter.py index 054070c7..0cca5ef8 100644 --- a/alembic/versions/2024_03_22_0010-ffe2f57bd288_create_output_parameter.py +++ b/alembic/versions/2024_03_22_0010-ffe2f57bd288_create_output_parameter.py @@ -37,9 +37,17 @@ def upgrade() -> None: sa.PrimaryKeyConstraint("output_parameter_id"), ) op.create_index( - op.f("ix_output_parameters_output_parameter_id"), "output_parameters", ["output_parameter_id"], unique=False + op.f("ix_output_parameters_output_parameter_id"), + "output_parameters", + ["output_parameter_id"], + unique=False, + ) + op.create_index( + op.f("ix_output_parameters_workflow_id"), + "output_parameters", + ["workflow_id"], + unique=False, ) - op.create_index(op.f("ix_output_parameters_workflow_id"), "output_parameters", ["workflow_id"], unique=False) op.create_table( "workflow_run_output_parameters", sa.Column("workflow_run_id", sa.String(), nullable=False), @@ -74,10 +82,12 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index( - op.f("ix_workflow_run_output_parameters_workflow_run_id"), table_name="workflow_run_output_parameters" + op.f("ix_workflow_run_output_parameters_workflow_run_id"), + table_name="workflow_run_output_parameters", ) op.drop_index( - op.f("ix_workflow_run_output_parameters_output_parameter_id"), table_name="workflow_run_output_parameters" + op.f("ix_workflow_run_output_parameters_output_parameter_id"), + table_name="workflow_run_output_parameters", ) op.drop_table("workflow_run_output_parameters") op.drop_index(op.f("ix_output_parameters_workflow_id"), table_name="output_parameters") diff --git a/alembic/versions/2024_04_03_2257-4630ab8c198e_create_bitwarden_credential_parameter_.py b/alembic/versions/2024_04_03_2257-4630ab8c198e_create_bitwarden_credential_parameter_.py index 70cff2f2..1eec22e8 100644 --- a/alembic/versions/2024_04_03_2257-4630ab8c198e_create_bitwarden_credential_parameter_.py +++ b/alembic/versions/2024_04_03_2257-4630ab8c198e_create_bitwarden_credential_parameter_.py @@ -58,7 +58,8 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index( - op.f("ix_bitwarden_login_credential_parameters_workflow_id"), table_name="bitwarden_login_credential_parameters" + op.f("ix_bitwarden_login_credential_parameters_workflow_id"), + table_name="bitwarden_login_credential_parameters", ) op.drop_index( op.f("ix_bitwarden_login_credential_parameters_bitwarden_login_credential_parameter_id"), diff --git a/alembic/versions/2024_04_28_2320-68d78072fdb5_add_org_task_step_index.py b/alembic/versions/2024_04_28_2320-68d78072fdb5_add_org_task_step_index.py index b4f00cbd..04d6ebfd 100644 --- a/alembic/versions/2024_04_28_2320-68d78072fdb5_add_org_task_step_index.py +++ b/alembic/versions/2024_04_28_2320-68d78072fdb5_add_org_task_step_index.py @@ -19,7 +19,12 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_index("org_task_step_index", "artifacts", ["organization_id", "task_id", "step_id"], unique=False) + op.create_index( + "org_task_step_index", + "artifacts", + ["organization_id", "task_id", "step_id"], + unique=False, + ) # ### end Alembic commands ### diff --git a/alembic/versions/2024_05_14_0245-baec12642d77_add_workflow_permanent_id_constraint_.py b/alembic/versions/2024_05_14_0245-baec12642d77_add_workflow_permanent_id_constraint_.py index 77785ca6..46c5421a 100644 --- a/alembic/versions/2024_05_14_0245-baec12642d77_add_workflow_permanent_id_constraint_.py +++ b/alembic/versions/2024_05_14_0245-baec12642d77_add_workflow_permanent_id_constraint_.py @@ -23,10 +23,22 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column("workflows", "workflow_permanent_id", existing_type=sa.VARCHAR(), nullable=False) op.alter_column("workflows", "version", existing_type=sa.INTEGER(), nullable=False) - op.create_index(op.f("ix_workflows_workflow_permanent_id"), "workflows", ["workflow_permanent_id"], unique=False) - op.create_index("permanent_id_version_idx", "workflows", ["workflow_permanent_id", "version"], unique=False) + op.create_index( + op.f("ix_workflows_workflow_permanent_id"), + "workflows", + ["workflow_permanent_id"], + unique=False, + ) + op.create_index( + "permanent_id_version_idx", + "workflows", + ["workflow_permanent_id", "version"], + unique=False, + ) op.create_unique_constraint( - "uc_org_permanent_id_version", "workflows", ["organization_id", "workflow_permanent_id", "version"] + "uc_org_permanent_id_version", + "workflows", + ["organization_id", "workflow_permanent_id", "version"], ) # ### end Alembic commands ### diff --git a/poetry.lock b/poetry.lock index 442c947f..95bd6b4f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -583,50 +583,6 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] -[[package]] -name = "black" -version = "23.12.1" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.8" -files = [ - {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, - {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, - {file = "black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0"}, - {file = "black-23.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3"}, - {file = "black-23.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba"}, - {file = "black-23.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b"}, - {file = "black-23.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59"}, - {file = "black-23.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50"}, - {file = "black-23.12.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e"}, - {file = "black-23.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec"}, - {file = "black-23.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e"}, - {file = "black-23.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9"}, - {file = "black-23.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f"}, - {file = "black-23.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d"}, - {file = "black-23.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a"}, - {file = "black-23.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e"}, - {file = "black-23.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055"}, - {file = "black-23.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54"}, - {file = "black-23.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea"}, - {file = "black-23.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2"}, - {file = "black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e"}, - {file = "black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=0.9.0" -platformdirs = ">=2" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - [[package]] name = "bleach" version = "6.1.0" @@ -3075,6 +3031,7 @@ files = [ {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8"}, {file = "lxml-5.2.1-cp37-cp37m-win32.whl", hash = "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd"}, {file = "lxml-5.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c"}, + {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5"}, {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b"}, {file = "lxml-5.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a"}, {file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585"}, @@ -4215,17 +4172,6 @@ files = [ qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] testing = ["docopt", "pytest"] -[[package]] -name = "pathspec" -version = "0.12.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, - {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, -] - [[package]] name = "pexpect" version = "4.9.0" @@ -5252,6 +5198,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -5259,8 +5206,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -5277,6 +5231,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -5284,6 +5239,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -7234,4 +7190,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.11,<3.12" -content-hash = "d2db23b7c08b0c9225fe746d22a029a372d2848b4895106612888fc1a0041e79" +content-hash = "0dc27842c8de420ea8e4d4573e2e062f4a99d27b0e8e4c398abea4b7137b7e2d" diff --git a/pyproject.toml b/pyproject.toml index 93fa913d..ab733d0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,8 +57,7 @@ fpdf = "^1.7.2" pypdf = "^4.2.0" [tool.poetry.group.dev.dependencies] -isort = "^5.12.0" -black = "^23.3.0" +isort = "^5.13.2" pre-commit = "^3.3.3" mypy = "^1.4.1" flake8 = "^6.0.0" @@ -84,11 +83,38 @@ clipboard = "^0.0.4" requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" -[tool.black] +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", + "alembic/env.py", +] line-length = 120 -target-version = ['py311'] -include = '\.pyi?$' -extend-exclude = '(/dist|/.venv|/venv|/build)/' +target-version = "py311" [tool.isort] profile = "black" diff --git a/scripts/create_organization.py b/scripts/create_organization.py index dda29ebd..af1c6e46 100644 --- a/scripts/create_organization.py +++ b/scripts/create_organization.py @@ -13,7 +13,10 @@ async def create_org(org_name: str, webhook_callback_url: str | None = None) -> await create_org_api_token(organization.organization_id) -def main(org_name: str, webhook_callback_url: Annotated[Optional[str], typer.Argument()] = None) -> None: +def main( + org_name: str, + webhook_callback_url: Annotated[Optional[str], typer.Argument()] = None, +) -> None: asyncio.run(create_org(org_name, webhook_callback_url)) diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py index 0d08d6c4..1199503d 100644 --- a/skyvern/exceptions.py +++ b/skyvern/exceptions.py @@ -19,7 +19,12 @@ class InvalidOpenAIResponseFormat(SkyvernException): class FailedToSendWebhook(SkyvernException): - def __init__(self, task_id: str | None = None, workflow_run_id: str | None = None, workflow_id: str | None = None): + def __init__( + self, + task_id: str | None = None, + workflow_run_id: str | None = None, + workflow_id: str | None = None, + ): workflow_run_str = f"workflow_run_id={workflow_run_id}" if workflow_run_id else "" workflow_str = f"workflow_id={workflow_id}" if workflow_id else "" task_str = f"task_id={task_id}" if task_id else "" @@ -122,7 +127,10 @@ class WorkflowNotFound(SkyvernHTTPException): else: workflow_repr = f"workflow_permanent_id={workflow_permanent_id}" - super().__init__(f"Workflow not found. {workflow_repr}", status_code=status.HTTP_404_NOT_FOUND) + super().__init__( + f"Workflow not found. {workflow_repr}", + status_code=status.HTTP_404_NOT_FOUND, + ) class WorkflowRunNotFound(SkyvernException): @@ -144,7 +152,10 @@ class MissingValueForParameter(SkyvernException): class WorkflowParameterNotFound(SkyvernHTTPException): def __init__(self, workflow_parameter_id: str) -> None: - super().__init__(f"Workflow parameter {workflow_parameter_id} not found", status_code=status.HTTP_404_NOT_FOUND) + super().__init__( + f"Workflow parameter {workflow_parameter_id} not found", + status_code=status.HTTP_404_NOT_FOUND, + ) class FailedToNavigateToUrl(SkyvernException): @@ -188,7 +199,10 @@ class BrowserStateMissingPage(SkyvernException): class OrganizationNotFound(SkyvernHTTPException): def __init__(self, organization_id: str) -> None: - super().__init__(f"Organization {organization_id} not found", status_code=status.HTTP_404_NOT_FOUND) + super().__init__( + f"Organization {organization_id} not found", + status_code=status.HTTP_404_NOT_FOUND, + ) class StepNotFound(SkyvernHTTPException): diff --git a/skyvern/forge/__main__.py b/skyvern/forge/__main__.py index db4246d4..8bb4626b 100644 --- a/skyvern/forge/__main__.py +++ b/skyvern/forge/__main__.py @@ -17,4 +17,10 @@ if __name__ == "__main__": load_dotenv() reload = SettingsManager.get_settings().ENV == "local" - uvicorn.run("skyvern.forge.api_app:app", host="0.0.0.0", port=port, log_level="info", reload=reload) + uvicorn.run( + "skyvern.forge.api_app:app", + host="0.0.0.0", + port=port, + log_level="info", + reload=reload, + ) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index c5433eb8..f729468c 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -54,7 +54,10 @@ class ForgeAgent: for module in SettingsManager.get_settings().ADDITIONAL_MODULES: LOG.info("Loading additional module", module=module) __import__(module) - LOG.info("Additional modules loaded", modules=SettingsManager.get_settings().ADDITIONAL_MODULES) + LOG.info( + "Additional modules loaded", + modules=SettingsManager.get_settings().ADDITIONAL_MODULES, + ) LOG.info( "Initializing ForgeAgent", env=SettingsManager.get_settings().ENV, @@ -115,7 +118,10 @@ class ForgeAgent: if task_url is None: browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(workflow_run=workflow_run) if not browser_state.page: - LOG.error("BrowserState has no page", workflow_run_id=workflow_run.workflow_run_id) + LOG.error( + "BrowserState has no page", + workflow_run_id=workflow_run.workflow_run_id, + ) raise MissingBrowserStatePage(workflow_run_id=workflow_run.workflow_run_id) if browser_state.page.url == "about:blank": @@ -155,7 +161,9 @@ class ForgeAgent: ) # Update task status to running task = await app.DATABASE.update_task( - task_id=task.task_id, organization_id=task.organization_id, status=TaskStatus.running + task_id=task.task_id, + organization_id=task.organization_id, + status=TaskStatus.running, ) step = await app.DATABASE.create_step( task.task_id, @@ -215,7 +223,11 @@ class ForgeAgent: try: # Check some conditions before executing the step, throw an exception if the step can't be executed await self.validate_step_execution(task, step) - step, browser_state, detailed_output = await self._initialize_execution_state(task, step, workflow_run) + ( + step, + browser_state, + detailed_output, + ) = await self._initialize_execution_state(task, step, workflow_run) if browser_state.page: self.register_async_operations(organization, task, browser_state.page) @@ -242,9 +254,11 @@ class ForgeAgent: return step, detailed_output, None elif step.status == StepStatus.completed: # TODO (kerem): keep the task object uptodate at all times so that send_task_response can just use it - is_task_completed, maybe_last_step, maybe_next_step = await self.handle_completed_step( - organization, task, step - ) + ( + is_task_completed, + maybe_last_step, + maybe_next_step, + ) = await self.handle_completed_step(organization, task, step) if is_task_completed is not None and maybe_last_step: last_step = maybe_last_step await self.send_task_response( @@ -358,7 +372,10 @@ class ForgeAgent: step_retry=step.retry_index, ) step = await self.update_step(step=step, status=StepStatus.running) - scraped_page, extract_action_prompt = await self._build_and_record_step_prompt( + ( + scraped_page, + extract_action_prompt, + ) = await self._build_and_record_step_prompt( task, step, browser_state, @@ -380,7 +397,8 @@ class ForgeAgent: else: actions = [ CompleteAction( - reasoning="Task has no navigation goal.", data_extraction_goal=task.data_extraction_goal + reasoning="Task has no navigation goal.", + data_extraction_goal=task.data_extraction_goal, ) ] detailed_agent_step_output.actions = actions @@ -393,7 +411,9 @@ class ForgeAgent: step_retry=step.retry_index, ) step = await self.update_step( - step=step, status=StepStatus.failed, output=detailed_agent_step_output.to_agent_step_output() + step=step, + status=StepStatus.failed, + output=detailed_agent_step_output.to_agent_step_output(), ) detailed_agent_step_output = DetailedAgentStepOutput( scraped_page=scraped_page, @@ -426,7 +446,11 @@ class ForgeAgent: # if there are wait actions and there are other actions in the list, skip wait actions if wait_actions_len > 0 and wait_actions_len < len(actions): actions = [action for action in actions if action.action_type != ActionType.WAIT] - LOG.info("Skipping wait actions", wait_actions_to_skip=wait_actions_to_skip, actions=actions) + LOG.info( + "Skipping wait actions", + wait_actions_to_skip=wait_actions_to_skip, + actions=actions, + ) # initialize list of tuples and set actions as the first element of each tuple so that in the case # of an exception, we can still see all the actions @@ -436,13 +460,19 @@ class ForgeAgent: for action_idx, action in enumerate(actions): if isinstance(action, WebAction): if action.element_id in web_action_element_ids: - LOG.error("Duplicate action element id. Action handling stops", action=action) + LOG.error( + "Duplicate action element id. Action handling stops", + action=action, + ) break web_action_element_ids.add(action.element_id) self.async_operation_pool.run_operation(task.task_id, AgentPhase.action) results = await ActionHandler.handle_action(scraped_page, task, step, browser_state, action) - detailed_agent_step_output.actions_and_results[action_idx] = (action, results) + detailed_agent_step_output.actions_and_results[action_idx] = ( + action, + results, + ) # wait random time between actions to avoid detection await asyncio.sleep(random.uniform(1.0, 2.0)) await self.record_artifacts_after_action(task, step, browser_state) @@ -467,7 +497,10 @@ class ForgeAgent: # for now, we're being optimistic and assuming that # js call doesn't have impact on the following actions if results[-1].javascript_triggered: - LOG.info("Action triggered javascript. Stop executing reamaining actions.", action=action) + LOG.info( + "Action triggered javascript. Stop executing reamaining actions.", + action=action, + ) # stop executing the rest actions break else: @@ -484,7 +517,9 @@ class ForgeAgent: ) # if the action failed, don't execute the rest of the actions, mark the step as failed, and retry failed_step = await self.update_step( - step=step, status=StepStatus.failed, output=detailed_agent_step_output.to_agent_step_output() + step=step, + status=StepStatus.failed, + output=detailed_agent_step_output.to_agent_step_output(), ) return failed_step, detailed_agent_step_output @@ -498,7 +533,9 @@ class ForgeAgent: ) # If no action errors return the agent state and output completed_step = await self.update_step( - step=step, status=StepStatus.completed, output=detailed_agent_step_output.to_agent_step_output() + step=step, + status=StepStatus.completed, + output=detailed_agent_step_output.to_agent_step_output(), ) return completed_step, detailed_agent_step_output except Exception: @@ -510,7 +547,9 @@ class ForgeAgent: step_retry=step.retry_index, ) failed_step = await self.update_step( - step=step, status=StepStatus.failed, output=detailed_agent_step_output.to_agent_step_output() + step=step, + status=StepStatus.failed, + output=detailed_agent_step_output.to_agent_step_output(), ) return failed_step, detailed_agent_step_output @@ -645,7 +684,7 @@ class ForgeAgent: ): element_tree_format = ElementTreeFormat.HTML LOG.info( - f"Building element tree", + "Building element tree", task_id=task.task_id, workflow_run_id=task.workflow_run_id, format=element_tree_format, @@ -660,7 +699,7 @@ class ForgeAgent: elements=scraped_page.build_element_tree(element_tree_format), data_extraction_goal=task.data_extraction_goal, action_history=actions_and_results_str, - error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else None, + error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None), utc_datetime=datetime.utcnow(), ) @@ -791,7 +830,11 @@ class ForgeAgent: LOG.error("Failed to get task from db when sending task response") raise TaskNotFound(task_id=task.task_id) except Exception as e: - LOG.error("Failed to get task from db when sending task response", task_id=task.task_id, error=e) + LOG.error( + "Failed to get task from db when sending task response", + task_id=task.task_id, + error=e, + ) raise TaskNotFound(task_id=task.task_id) from e task = refreshed_task # log the task status as an event @@ -843,7 +886,11 @@ class ForgeAgent: await self.execute_task_webhook(task=task, last_step=last_step, api_key=api_key) async def execute_task_webhook( - self, task: Task, last_step: Step, api_key: str | None, skip_artifacts: bool = False + self, + task: Task, + last_step: Step, + api_key: str | None, + skip_artifacts: bool = False, ) -> None: if not api_key: LOG.warning( @@ -1106,7 +1153,11 @@ class ForgeAgent: ) last_step = await self.update_step(step, is_last=True) extracted_information = await self.get_extracted_information_for_task(task) - await self.update_task(task, status=TaskStatus.completed, extracted_information=extracted_information) + await self.update_task( + task, + status=TaskStatus.completed, + extracted_information=extracted_information, + ) return True, last_step, None if step.is_terminated(): LOG.info( @@ -1193,5 +1244,7 @@ class ForgeAgent: task_errors.extend([error.model_dump() for error in step_errors]) return await app.DATABASE.update_task( - task_id=task.task_id, organization_id=task.organization_id, errors=task_errors + task_id=task.task_id, + organization_id=task.organization_id, + errors=task_errors, ) diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.py index 744d4683..af5f72c2 100644 --- a/skyvern/forge/api_app.py +++ b/skyvern/forge/api_app.py @@ -102,7 +102,8 @@ def get_agent_app(router: APIRouter = base_router) -> FastAPI: LOG.info("Loading additional module to set up api app", module=module) __import__(module) LOG.info( - "Additional modules loaded to set up api app", modules=SettingsManager.get_settings().ADDITIONAL_MODULES + "Additional modules loaded to set up api app", + modules=SettingsManager.get_settings().ADDITIONAL_MODULES, ) if forge_app.setup_api_app: diff --git a/skyvern/forge/app.py b/skyvern/forge/app.py index 4d5525ef..76d07bf6 100644 --- a/skyvern/forge/app.py +++ b/skyvern/forge/app.py @@ -31,7 +31,8 @@ tracer.configure( setup_logger() SETTINGS_MANAGER = SettingsManager.get_settings() DATABASE = AgentDB( - SettingsManager.get_settings().DATABASE_STRING, debug_enabled=SettingsManager.get_settings().DEBUG_MODE + SettingsManager.get_settings().DATABASE_STRING, + debug_enabled=SettingsManager.get_settings().DEBUG_MODE, ) STORAGE = StorageFactory.get_storage() ARTIFACT_MANAGER = ArtifactManager() diff --git a/skyvern/forge/async_operations.py b/skyvern/forge/async_operations.py index 8b91aa70..05c41bbb 100644 --- a/skyvern/forge/async_operations.py +++ b/skyvern/forge/async_operations.py @@ -50,7 +50,7 @@ class AsyncOperation: def run(self) -> asyncio.Task | None: if self.aio_task is not None and not self.aio_task.done(): LOG.warning( - f"Task already running", + "Task already running", task_id=self.task_id, operation_type=self.type, agent_phase=self.agent_phase, @@ -113,7 +113,7 @@ class AsyncOperationPool: aio_task = self._aio_tasks[task_id][operation_type] if not aio_task.done(): LOG.info( - f"aio task already running", + "aio task already running", task_id=task_id, operation_type=operation_type, agent_phase=agent_phase, @@ -130,6 +130,9 @@ class AsyncOperationPool: async with asyncio.timeout(30): await asyncio.gather(*[aio_task for aio_task in self.get_aio_tasks(task_id) if not aio_task.done()]) except asyncio.TimeoutError: - LOG.error(f"Timeout (30s) while waiting for pending async tasks for task_id={task_id}", task_id=task_id) + LOG.error( + f"Timeout (30s) while waiting for pending async tasks for task_id={task_id}", + task_id=task_id, + ) self.remove_operations(task_id) diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py index e69b2c58..166795f5 100644 --- a/skyvern/forge/sdk/api/llm/api_handler_factory.py +++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py @@ -44,7 +44,7 @@ class LLMAPIHandlerFactory: ), num_retries=llm_config.num_retries, retry_after=llm_config.retry_delay_seconds, - set_verbose=False if SettingsManager.get_settings().is_cloud_environment() else llm_config.set_verbose, + set_verbose=(False if SettingsManager.get_settings().is_cloud_environment() else llm_config.set_verbose), enable_pre_call_checks=True, ) main_model_group = llm_config.main_model_group @@ -101,7 +101,11 @@ class LLMAPIHandlerFactory: except openai.OpenAIError as e: raise LLMProviderError(llm_key) from e except Exception as e: - LOG.exception("LLM request failed unexpectedly", llm_key=llm_key, model=main_model_group) + LOG.exception( + "LLM request failed unexpectedly", + llm_key=llm_key, + model=main_model_group, + ) raise LLMProviderError(llm_key) from e if step: diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py index d246aa2e..1a926b7c 100644 --- a/skyvern/forge/sdk/api/llm/config_registry.py +++ b/skyvern/forge/sdk/api/llm/config_registry.py @@ -58,35 +58,58 @@ if not any( if SettingsManager.get_settings().ENABLE_OPENAI: LLMConfigRegistry.register_config( "OPENAI_GPT4_TURBO", - LLMConfig("gpt-4-turbo", ["OPENAI_API_KEY"], supports_vision=False, add_assistant_prefix=False), + LLMConfig( + "gpt-4-turbo", + ["OPENAI_API_KEY"], + supports_vision=False, + add_assistant_prefix=False, + ), ) LLMConfigRegistry.register_config( - "OPENAI_GPT4V", LLMConfig("gpt-4-turbo", ["OPENAI_API_KEY"], supports_vision=True, add_assistant_prefix=False) + "OPENAI_GPT4V", + LLMConfig( + "gpt-4-turbo", + ["OPENAI_API_KEY"], + supports_vision=True, + add_assistant_prefix=False, + ), ) if SettingsManager.get_settings().ENABLE_ANTHROPIC: LLMConfigRegistry.register_config( "ANTHROPIC_CLAUDE3", LLMConfig( - "anthropic/claude-3-sonnet-20240229", ["ANTHROPIC_API_KEY"], supports_vision=True, add_assistant_prefix=True + "anthropic/claude-3-sonnet-20240229", + ["ANTHROPIC_API_KEY"], + supports_vision=True, + add_assistant_prefix=True, ), ) LLMConfigRegistry.register_config( "ANTHROPIC_CLAUDE3_OPUS", LLMConfig( - "anthropic/claude-3-opus-20240229", ["ANTHROPIC_API_KEY"], supports_vision=True, add_assistant_prefix=True + "anthropic/claude-3-opus-20240229", + ["ANTHROPIC_API_KEY"], + supports_vision=True, + add_assistant_prefix=True, ), ) LLMConfigRegistry.register_config( "ANTHROPIC_CLAUDE3_SONNET", LLMConfig( - "anthropic/claude-3-sonnet-20240229", ["ANTHROPIC_API_KEY"], supports_vision=True, add_assistant_prefix=True + "anthropic/claude-3-sonnet-20240229", + ["ANTHROPIC_API_KEY"], + supports_vision=True, + add_assistant_prefix=True, ), ) LLMConfigRegistry.register_config( "ANTHROPIC_CLAUDE3_HAIKU", LLMConfig( - "anthropic/claude-3-haiku-20240307", ["ANTHROPIC_API_KEY"], supports_vision=True, add_assistant_prefix=True + "anthropic/claude-3-haiku-20240307", + ["ANTHROPIC_API_KEY"], + supports_vision=True, + add_assistant_prefix=True, ), ) @@ -125,7 +148,12 @@ if SettingsManager.get_settings().ENABLE_AZURE: "AZURE_OPENAI_GPT4V", LLMConfig( f"azure/{SettingsManager.get_settings().AZURE_DEPLOYMENT}", - ["AZURE_DEPLOYMENT", "AZURE_API_KEY", "AZURE_API_BASE", "AZURE_API_VERSION"], + [ + "AZURE_DEPLOYMENT", + "AZURE_API_KEY", + "AZURE_API_BASE", + "AZURE_API_VERSION", + ], supports_vision=True, add_assistant_prefix=False, ), diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.py index 11e5ed14..da725482 100644 --- a/skyvern/forge/sdk/api/llm/utils.py +++ b/skyvern/forge/sdk/api/llm/utils.py @@ -33,7 +33,10 @@ async def llm_messages_builder( ) # Anthropic models seems to struggle to always output a valid json object so we need to prefill the response to force it: if add_assistant_prefix: - return [{"role": "user", "content": messages}, {"role": "assistant", "content": "{"}] + return [ + {"role": "user", "content": messages}, + {"role": "assistant", "content": "{"}, + ] return [{"role": "user", "content": messages}] diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.py index 8cd38144..7e6a4156 100644 --- a/skyvern/forge/sdk/artifact/manager.py +++ b/skyvern/forge/sdk/artifact/manager.py @@ -17,7 +17,11 @@ class ArtifactManager: upload_aiotasks_map: dict[str, list[asyncio.Task[None]]] = defaultdict(list) async def create_artifact( - self, step: Step, artifact_type: ArtifactType, data: bytes | None = None, path: str | None = None + self, + step: Step, + artifact_type: ArtifactType, + data: bytes | None = None, + path: str | None = None, ) -> str: # TODO (kerem): Which is better? # current: (disadvantage: we create the artifact_id UUID here) @@ -87,7 +91,10 @@ class ArtifactManager: duration=time.time() - st, ) except asyncio.TimeoutError: - LOG.error(f"Timeout (30s) while waiting for upload tasks for task_id={task_id}", task_id=task_id) + LOG.error( + f"Timeout (30s) while waiting for upload tasks for task_id={task_id}", + task_id=task_id, + ) del self.upload_aiotasks_map[task_id] @@ -109,7 +116,10 @@ class ArtifactManager: duration=time.time() - st, ) except asyncio.TimeoutError: - LOG.error(f"Timeout (30s) while waiting for upload tasks for task_ids={task_ids}", task_ids=task_ids) + LOG.error( + f"Timeout (30s) while waiting for upload tasks for task_ids={task_ids}", + task_ids=task_ids, + ) for task_id in task_ids: del self.upload_aiotasks_map[task_id] diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.py index 5fcb2800..950b953d 100644 --- a/skyvern/forge/sdk/artifact/storage/local.py +++ b/skyvern/forge/sdk/artifact/storage/local.py @@ -28,7 +28,11 @@ class LocalStorage(BaseStorage): with open(file_path, "wb") as f: f.write(data) except Exception: - LOG.exception("Failed to store artifact locally.", file_path=file_path, artifact=artifact) + LOG.exception( + "Failed to store artifact locally.", + file_path=file_path, + artifact=artifact, + ) async def store_artifact_from_path(self, artifact: Artifact, path: str) -> None: file_path = None @@ -37,7 +41,11 @@ class LocalStorage(BaseStorage): self._create_directories_if_not_exists(file_path) Path(path).replace(file_path) except Exception: - LOG.exception("Failed to store artifact locally.", file_path=file_path, artifact=artifact) + LOG.exception( + "Failed to store artifact locally.", + file_path=file_path, + artifact=artifact, + ) async def retrieve_artifact(self, artifact: Artifact) -> bytes | None: file_path = None @@ -46,7 +54,11 @@ class LocalStorage(BaseStorage): with open(file_path, "rb") as f: return f.read() except Exception: - LOG.exception("Failed to retrieve local artifact.", file_path=file_path, artifact=artifact) + LOG.exception( + "Failed to retrieve local artifact.", + file_path=file_path, + artifact=artifact, + ) return None async def get_share_link(self, artifact: Artifact) -> str: diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.py index a16e0611..3a2e74ec 100644 --- a/skyvern/forge/sdk/db/client.py +++ b/skyvern/forge/sdk/db/client.py @@ -184,7 +184,11 @@ class AgentDB: ).first(): return convert_to_task(task_obj, self.debug_enabled) else: - LOG.info("Task not found", task_id=task_id, organization_id=organization_id) + LOG.info( + "Task not found", + task_id=task_id, + organization_id=organization_id, + ) return None except SQLAlchemyError: LOG.error("SQLAlchemyError", exc_info=True) @@ -266,7 +270,11 @@ class AgentDB: ).first(): return convert_to_step(step, debug_enabled=self.debug_enabled) else: - LOG.info("Latest step not found", task_id=task_id, organization_id=organization_id) + LOG.info( + "Latest step not found", + task_id=task_id, + organization_id=organization_id, + ) return None except SQLAlchemyError: LOG.error("SQLAlchemyError", exc_info=True) @@ -812,7 +820,10 @@ class AgentDB: ) .where(WorkflowModel.organization_id == organization_id) .where(WorkflowModel.deleted_at.is_(None)) - .group_by(WorkflowModel.organization_id, WorkflowModel.workflow_permanent_id) + .group_by( + WorkflowModel.organization_id, + WorkflowModel.workflow_permanent_id, + ) .subquery() ) main_query = ( @@ -924,7 +935,10 @@ class AgentDB: await session.commit() await session.refresh(workflow_run) return convert_to_workflow_run(workflow_run) - LOG.error("WorkflowRun not found, nothing to update", workflow_run_id=workflow_run_id) + LOG.error( + "WorkflowRun not found, nothing to update", + workflow_run_id=workflow_run_id, + ) return None async def get_workflow_run(self, workflow_run_id: str) -> WorkflowRun | None: @@ -1066,7 +1080,10 @@ class AgentDB: raise async def create_workflow_run_output_parameter( - self, workflow_run_id: str, output_parameter_id: str, value: dict[str, Any] | list | str | None + self, + workflow_run_id: str, + output_parameter_id: str, + value: dict[str, Any] | list | str | None, ) -> WorkflowRunOutputParameter: try: async with self.Session() as session: @@ -1149,7 +1166,9 @@ class AgentDB: ( workflow_parameter, convert_to_workflow_run_parameter( - workflow_run_parameter, workflow_parameter, self.debug_enabled + workflow_run_parameter, + workflow_parameter, + self.debug_enabled, ), ) ) diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py index 5d80c6c6..fcae00f2 100644 --- a/skyvern/forge/sdk/db/models.py +++ b/skyvern/forge/sdk/db/models.py @@ -63,7 +63,11 @@ class TaskModel(Base): max_steps_per_run = Column(Integer, nullable=True) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False, index=True) modified_at = Column( - DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False, index=True + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + index=True, ) @@ -80,7 +84,12 @@ class StepModel(Base): is_last = Column(Boolean, default=False) retry_index = Column(Integer, default=0) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) input_token_count = Column(Integer, default=0) output_token_count = Column(Integer, default=0) step_cost = Column(Numeric, default=0) @@ -96,7 +105,12 @@ class OrganizationModel(Base): max_retries_per_step = Column(Integer, nullable=True) domain = Column(String, nullable=True, index=True) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime, + nullable=False, + ) class OrganizationAuthTokenModel(Base): @@ -115,7 +129,12 @@ class OrganizationAuthTokenModel(Base): valid = Column(Boolean, nullable=False, default=True) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime, + nullable=False, + ) deleted_at = Column(DateTime, nullable=True) @@ -130,13 +149,23 @@ class ArtifactModel(Base): artifact_type = Column(String) uri = Column(String) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) class WorkflowModel(Base): __tablename__ = "workflows" __table_args__ = ( - UniqueConstraint("organization_id", "workflow_permanent_id", "version", name="uc_org_permanent_id_version"), + UniqueConstraint( + "organization_id", + "workflow_permanent_id", + "version", + name="uc_org_permanent_id_version", + ), Index("permanent_id_version_idx", "workflow_permanent_id", "version"), ) @@ -149,7 +178,12 @@ class WorkflowModel(Base): webhook_callback_url = Column(String) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) deleted_at = Column(DateTime, nullable=True) workflow_permanent_id = Column(String, nullable=False, default=generate_workflow_permanent_id, index=True) @@ -166,7 +200,12 @@ class WorkflowRunModel(Base): webhook_callback_url = Column(String) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) class WorkflowParameterModel(Base): @@ -179,7 +218,12 @@ class WorkflowParameterModel(Base): workflow_id = Column(String, ForeignKey("workflows.workflow_id"), index=True, nullable=False) default_value = Column(String, nullable=True) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) deleted_at = Column(DateTime, nullable=True) @@ -191,7 +235,12 @@ class OutputParameterModel(Base): description = Column(String, nullable=True) workflow_id = Column(String, ForeignKey("workflows.workflow_id"), index=True, nullable=False) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) deleted_at = Column(DateTime, nullable=True) @@ -204,7 +253,12 @@ class AWSSecretParameterModel(Base): description = Column(String, nullable=True) aws_key = Column(String, nullable=False) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) deleted_at = Column(DateTime, nullable=True) @@ -212,7 +266,10 @@ class BitwardenLoginCredentialParameterModel(Base): __tablename__ = "bitwarden_login_credential_parameters" bitwarden_login_credential_parameter_id = Column( - String, primary_key=True, index=True, default=generate_bitwarden_login_credential_parameter_id + String, + primary_key=True, + index=True, + default=generate_bitwarden_login_credential_parameter_id, ) workflow_id = Column(String, ForeignKey("workflows.workflow_id"), index=True, nullable=False) key = Column(String, nullable=False) @@ -222,16 +279,29 @@ class BitwardenLoginCredentialParameterModel(Base): bitwarden_master_password_aws_secret_key = Column(String, nullable=False) url_parameter_key = Column(String, nullable=False) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) - modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False) + modified_at = Column( + DateTime, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow, + nullable=False, + ) deleted_at = Column(DateTime, nullable=True) class WorkflowRunParameterModel(Base): __tablename__ = "workflow_run_parameters" - workflow_run_id = Column(String, ForeignKey("workflow_runs.workflow_run_id"), primary_key=True, index=True) + workflow_run_id = Column( + String, + ForeignKey("workflow_runs.workflow_run_id"), + primary_key=True, + index=True, + ) workflow_parameter_id = Column( - String, ForeignKey("workflow_parameters.workflow_parameter_id"), primary_key=True, index=True + String, + ForeignKey("workflow_parameters.workflow_parameter_id"), + primary_key=True, + index=True, ) # Can be bool | int | float | str | dict | list depending on the workflow parameter type value = Column(String, nullable=False) @@ -241,9 +311,17 @@ class WorkflowRunParameterModel(Base): class WorkflowRunOutputParameterModel(Base): __tablename__ = "workflow_run_output_parameters" - workflow_run_id = Column(String, ForeignKey("workflow_runs.workflow_run_id"), primary_key=True, index=True) + workflow_run_id = Column( + String, + ForeignKey("workflow_runs.workflow_run_id"), + primary_key=True, + index=True, + ) output_parameter_id = Column( - String, ForeignKey("output_parameters.output_parameter_id"), primary_key=True, index=True + String, + ForeignKey("output_parameters.output_parameter_id"), + primary_key=True, + index=True, ) value = Column(JSON, nullable=False) created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False) diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py index 85546c28..528fef67 100644 --- a/skyvern/forge/sdk/db/utils.py +++ b/skyvern/forge/sdk/db/utils.py @@ -67,7 +67,7 @@ def convert_to_task(task_obj: TaskModel, debug_enabled: bool = False) -> Task: extracted_information=task_obj.extracted_information, failure_reason=task_obj.failure_reason, organization_id=task_obj.organization_id, - proxy_location=ProxyLocation(task_obj.proxy_location) if task_obj.proxy_location else None, + proxy_location=(ProxyLocation(task_obj.proxy_location) if task_obj.proxy_location else None), extracted_information_schema=task_obj.extracted_information_schema, workflow_run_id=task_obj.workflow_run_id, order=task_obj.order, @@ -112,7 +112,9 @@ def convert_to_organization(org_model: OrganizationModel) -> Organization: ) -def convert_to_organization_auth_token(org_auth_token: OrganizationAuthTokenModel) -> OrganizationAuthToken: +def convert_to_organization_auth_token( + org_auth_token: OrganizationAuthTokenModel, +) -> OrganizationAuthToken: return OrganizationAuthToken( id=org_auth_token.id, organization_id=org_auth_token.organization_id, @@ -126,7 +128,10 @@ def convert_to_organization_auth_token(org_auth_token: OrganizationAuthTokenMode def convert_to_artifact(artifact_model: ArtifactModel, debug_enabled: bool = False) -> Artifact: if debug_enabled: - LOG.debug("Converting ArtifactModel to Artifact", artifact_id=artifact_model.artifact_id) + LOG.debug( + "Converting ArtifactModel to Artifact", + artifact_id=artifact_model.artifact_id, + ) return Artifact( artifact_id=artifact_model.artifact_id, @@ -142,7 +147,10 @@ def convert_to_artifact(artifact_model: ArtifactModel, debug_enabled: bool = Fal def convert_to_workflow(workflow_model: WorkflowModel, debug_enabled: bool = False) -> Workflow: if debug_enabled: - LOG.debug("Converting WorkflowModel to Workflow", workflow_id=workflow_model.workflow_id) + LOG.debug( + "Converting WorkflowModel to Workflow", + workflow_id=workflow_model.workflow_id, + ) return Workflow( workflow_id=workflow_model.workflow_id, @@ -150,7 +158,7 @@ def convert_to_workflow(workflow_model: WorkflowModel, debug_enabled: bool = Fal title=workflow_model.title, workflow_permanent_id=workflow_model.workflow_permanent_id, webhook_callback_url=workflow_model.webhook_callback_url, - proxy_location=ProxyLocation(workflow_model.proxy_location) if workflow_model.proxy_location else None, + proxy_location=(ProxyLocation(workflow_model.proxy_location) if workflow_model.proxy_location else None), version=workflow_model.version, description=workflow_model.description, workflow_definition=WorkflowDefinition.model_validate(workflow_model.workflow_definition), @@ -162,13 +170,18 @@ def convert_to_workflow(workflow_model: WorkflowModel, debug_enabled: bool = Fal def convert_to_workflow_run(workflow_run_model: WorkflowRunModel, debug_enabled: bool = False) -> WorkflowRun: if debug_enabled: - LOG.debug("Converting WorkflowRunModel to WorkflowRun", workflow_run_id=workflow_run_model.workflow_run_id) + LOG.debug( + "Converting WorkflowRunModel to WorkflowRun", + workflow_run_id=workflow_run_model.workflow_run_id, + ) return WorkflowRun( workflow_run_id=workflow_run_model.workflow_run_id, workflow_id=workflow_run_model.workflow_id, status=WorkflowRunStatus[workflow_run_model.status], - proxy_location=ProxyLocation(workflow_run_model.proxy_location) if workflow_run_model.proxy_location else None, + proxy_location=( + ProxyLocation(workflow_run_model.proxy_location) if workflow_run_model.proxy_location else None + ), webhook_callback_url=workflow_run_model.webhook_callback_url, created_at=workflow_run_model.created_at, modified_at=workflow_run_model.modified_at, @@ -221,7 +234,8 @@ def convert_to_aws_secret_parameter( def convert_to_bitwarden_login_credential_parameter( - bitwarden_login_credential_parameter_model: BitwardenLoginCredentialParameterModel, debug_enabled: bool = False + bitwarden_login_credential_parameter_model: BitwardenLoginCredentialParameterModel, + debug_enabled: bool = False, ) -> BitwardenLoginCredentialParameter: if debug_enabled: LOG.debug( diff --git a/skyvern/forge/sdk/executor/async_executor.py b/skyvern/forge/sdk/executor/async_executor.py index 6be9ee80..e28cd258 100644 --- a/skyvern/forge/sdk/executor/async_executor.py +++ b/skyvern/forge/sdk/executor/async_executor.py @@ -91,7 +91,10 @@ class BackgroundTaskExecutor(AsyncExecutor): api_key: str | None, **kwargs: dict, ) -> None: - LOG.info("Executing workflow using background task executor", workflow_run_id=workflow_run_id) + LOG.info( + "Executing workflow using background task executor", + workflow_run_id=workflow_run_id, + ) background_tasks.add_task( app.WORKFLOW_SERVICE.execute_workflow, workflow_run_id=workflow_run_id, diff --git a/skyvern/forge/sdk/models.py b/skyvern/forge/sdk/models.py index 643bc999..75d307f4 100644 --- a/skyvern/forge/sdk/models.py +++ b/skyvern/forge/sdk/models.py @@ -53,7 +53,12 @@ class Step(BaseModel): output_token_count: int = 0 step_cost: float = 0 - def validate_update(self, status: StepStatus | None, output: AgentStepOutput | None, is_last: bool | None) -> None: + def validate_update( + self, + status: StepStatus | None, + output: AgentStepOutput | None, + is_last: bool | None, + ) -> None: old_status = self.status if status and not old_status.can_update_to(status): diff --git a/skyvern/forge/sdk/prompting.py b/skyvern/forge/sdk/prompting.py index 2d2c3fe0..48efc4f4 100644 --- a/skyvern/forge/sdk/prompting.py +++ b/skyvern/forge/sdk/prompting.py @@ -78,7 +78,12 @@ class PromptEngine: matches = get_close_matches(target, model_dirs, n=1, cutoff=0.1) return matches[0] except Exception: - LOG.error("Failed to get closest match.", target=target, model_dirs=model_dirs, exc_info=True) + LOG.error( + "Failed to get closest match.", + target=target, + model_dirs=model_dirs, + exc_info=True, + ) raise def load_prompt(self, template: str, **kwargs: Any) -> str: @@ -97,7 +102,12 @@ class PromptEngine: jinja_template = self.env.get_template(f"{template}.j2") return jinja_template.render(**kwargs) except Exception: - LOG.error("Failed to load prompt.", template=template, kwargs_keys=kwargs.keys(), exc_info=True) + LOG.error( + "Failed to load prompt.", + template=template, + kwargs_keys=kwargs.keys(), + exc_info=True, + ) raise def load_prompt_from_string(self, template: str, **kwargs: Any) -> str: @@ -115,5 +125,10 @@ class PromptEngine: jinja_template = self.env.from_string(template) return jinja_template.render(**kwargs) except Exception: - LOG.error("Failed to load prompt from string.", template=template, kwargs_keys=kwargs.keys(), exc_info=True) + LOG.error( + "Failed to load prompt from string.", + template=template, + kwargs_keys=kwargs.keys(), + exc_info=True, + ) raise diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py index e0c87110..2b401ff3 100644 --- a/skyvern/forge/sdk/routes/agent_protocol.py +++ b/skyvern/forge/sdk/routes/agent_protocol.py @@ -54,7 +54,10 @@ async def webhook( x_skyvern_timestamp=x_skyvern_timestamp, payload=payload, ) - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Missing webhook signature or timestamp") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing webhook signature or timestamp", + ) generated_signature = generate_skyvern_signature( payload.decode("utf-8"), @@ -82,7 +85,12 @@ async def check_server_status() -> Response: @base_router.post("/tasks", tags=["agent"], response_model=CreateTaskResponse) -@base_router.post("/tasks/", tags=["agent"], response_model=CreateTaskResponse, include_in_schema=False) +@base_router.post( + "/tasks/", + tags=["agent"], + response_model=CreateTaskResponse, + include_in_schema=False, +) async def create_agent_task( background_tasks: BackgroundTasks, task: TaskRequest, @@ -342,13 +350,21 @@ async def get_agent_tasks( """ analytics.capture("skyvern-oss-agent-tasks-get") tasks = await app.DATABASE.get_tasks( - page, page_size, task_status=task_status, organization_id=current_org.organization_id + page, + page_size, + task_status=task_status, + organization_id=current_org.organization_id, ) return ORJSONResponse([task.to_task_response().model_dump() for task in tasks]) @base_router.get("/internal/tasks", tags=["agent"], response_model=list[Task]) -@base_router.get("/internal/tasks/", tags=["agent"], response_model=list[Task], include_in_schema=False) +@base_router.get( + "/internal/tasks/", + tags=["agent"], + response_model=list[Task], + include_in_schema=False, +) async def get_agent_tasks_internal( page: int = Query(1, ge=1), page_size: int = Query(10, ge=1), @@ -367,7 +383,12 @@ async def get_agent_tasks_internal( @base_router.get("/tasks/{task_id}/steps", tags=["agent"], response_model=list[Step]) -@base_router.get("/tasks/{task_id}/steps/", tags=["agent"], response_model=list[Step], include_in_schema=False) +@base_router.get( + "/tasks/{task_id}/steps/", + tags=["agent"], + response_model=list[Step], + include_in_schema=False, +) async def get_agent_task_steps( task_id: str, current_org: Organization = Depends(org_auth_service.get_current_org), @@ -382,7 +403,11 @@ async def get_agent_task_steps( return ORJSONResponse([step.model_dump() for step in steps]) -@base_router.get("/tasks/{task_id}/steps/{step_id}/artifacts", tags=["agent"], response_model=list[Artifact]) +@base_router.get( + "/tasks/{task_id}/steps/{step_id}/artifacts", + tags=["agent"], + response_model=list[Artifact], +) @base_router.get( "/tasks/{task_id}/steps/{step_id}/artifacts/", tags=["agent"], @@ -412,7 +437,11 @@ async def get_agent_task_step_artifacts( for i, artifact in enumerate(artifacts): artifact.signed_url = signed_urls[i] else: - LOG.warning("Failed to get signed urls for artifacts", task_id=task_id, step_id=step_id) + LOG.warning( + "Failed to get signed urls for artifacts", + task_id=task_id, + step_id=step_id, + ) return ORJSONResponse([artifact.model_dump() for artifact in artifacts]) @@ -424,7 +453,11 @@ class ActionResultTmp(BaseModel): @base_router.get("/tasks/{task_id}/actions", response_model=list[ActionResultTmp]) -@base_router.get("/tasks/{task_id}/actions/", response_model=list[ActionResultTmp], include_in_schema=False) +@base_router.get( + "/tasks/{task_id}/actions/", + response_model=list[ActionResultTmp], + include_in_schema=False, +) async def get_task_actions( task_id: str, current_org: Organization = Depends(org_auth_service.get_current_org), @@ -441,7 +474,11 @@ async def get_task_actions( @base_router.post("/workflows/{workflow_id}/run", response_model=RunWorkflowResponse) -@base_router.post("/workflows/{workflow_id}/run/", response_model=RunWorkflowResponse, include_in_schema=False) +@base_router.post( + "/workflows/{workflow_id}/run/", + response_model=RunWorkflowResponse, + include_in_schema=False, +) async def execute_workflow( background_tasks: BackgroundTasks, workflow_id: str, @@ -476,7 +513,10 @@ async def execute_workflow( ) -@base_router.get("/workflows/{workflow_id}/runs/{workflow_run_id}", response_model=WorkflowRunStatusResponse) +@base_router.get( + "/workflows/{workflow_id}/runs/{workflow_run_id}", + response_model=WorkflowRunStatusResponse, +) @base_router.get( "/workflows/{workflow_id}/runs/{workflow_run_id}/", response_model=WorkflowRunStatusResponse, diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.py index ac136ca6..5051944f 100644 --- a/skyvern/forge/sdk/schemas/tasks.py +++ b/skyvern/forge/sdk/schemas/tasks.py @@ -82,13 +82,27 @@ class TaskStatus(StrEnum): completed = "completed" def is_final(self) -> bool: - return self in {TaskStatus.failed, TaskStatus.terminated, TaskStatus.completed, TaskStatus.timed_out} + return self in { + TaskStatus.failed, + TaskStatus.terminated, + TaskStatus.completed, + TaskStatus.timed_out, + } def can_update_to(self, new_status: TaskStatus) -> bool: allowed_transitions: dict[TaskStatus, set[TaskStatus]] = { - TaskStatus.created: {TaskStatus.queued, TaskStatus.running, TaskStatus.timed_out}, + TaskStatus.created: { + TaskStatus.queued, + TaskStatus.running, + TaskStatus.timed_out, + }, TaskStatus.queued: {TaskStatus.running, TaskStatus.timed_out}, - TaskStatus.running: {TaskStatus.completed, TaskStatus.failed, TaskStatus.terminated, TaskStatus.timed_out}, + TaskStatus.running: { + TaskStatus.completed, + TaskStatus.failed, + TaskStatus.terminated, + TaskStatus.timed_out, + }, TaskStatus.failed: set(), TaskStatus.terminated: set(), TaskStatus.completed: set(), diff --git a/skyvern/forge/sdk/services/bitwarden.py b/skyvern/forge/sdk/services/bitwarden.py index 09e9515b..0465307f 100644 --- a/skyvern/forge/sdk/services/bitwarden.py +++ b/skyvern/forge/sdk/services/bitwarden.py @@ -53,7 +53,11 @@ class BitwardenService: """ # Step 1: Set up environment variables and log in try: - env = {"BW_CLIENTID": client_id, "BW_CLIENTSECRET": client_secret, "BW_PASSWORD": master_password} + env = { + "BW_CLIENTID": client_id, + "BW_CLIENTSECRET": client_secret, + "BW_PASSWORD": master_password, + } login_command = ["bw", "login", "--apikey"] login_result = BitwardenService.run_command(login_command, env) @@ -81,7 +85,15 @@ class BitwardenService: raise BitwardenUnlockError("Session key is empty.") # Step 3: Retrieve the items - list_command = ["bw", "list", "items", "--url", url, "--session", session_key] + list_command = [ + "bw", + "list", + "items", + "--url", + url, + "--session", + session_key, + ] items_result = BitwardenService.run_command(list_command) if items_result.stderr and "Event post failed" not in items_result.stderr: @@ -100,7 +112,11 @@ class BitwardenService: totp_result = BitwardenService.run_command(totp_command) if totp_result.stderr and "Event post failed" not in totp_result.stderr: - LOG.warning("Bitwarden TOTP Error", error=totp_result.stderr, e=BitwardenTOTPError(totp_result.stderr)) + LOG.warning( + "Bitwarden TOTP Error", + error=totp_result.stderr, + e=BitwardenTOTPError(totp_result.stderr), + ) totp_code = totp_result.stdout credentials: list[dict[str, str]] = [ diff --git a/skyvern/forge/sdk/services/org_auth_service.py b/skyvern/forge/sdk/services/org_auth_service.py index 7f0c9461..6b48acf5 100644 --- a/skyvern/forge/sdk/services/org_auth_service.py +++ b/skyvern/forge/sdk/services/org_auth_service.py @@ -39,7 +39,9 @@ async def get_current_org( ) -async def get_current_org_with_api_key(x_api_key: Annotated[str | None, Header()] = None) -> Organization: +async def get_current_org_with_api_key( + x_api_key: Annotated[str | None, Header()] = None, +) -> Organization: if not x_api_key: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -48,7 +50,9 @@ async def get_current_org_with_api_key(x_api_key: Annotated[str | None, Header() return await _get_current_org_cached(x_api_key, app.DATABASE) -async def get_current_org_with_authentication(authorization: Annotated[str | None, Header()] = None) -> Organization: +async def get_current_org_with_authentication( + authorization: Annotated[str | None, Header()] = None, +) -> Organization: if not authorization: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, diff --git a/skyvern/forge/sdk/services/org_auth_token_service.py b/skyvern/forge/sdk/services/org_auth_token_service.py index 7f7227a6..f7faab18 100644 --- a/skyvern/forge/sdk/services/org_auth_token_service.py +++ b/skyvern/forge/sdk/services/org_auth_token_service.py @@ -35,5 +35,5 @@ async def create_org_api_token(org_id: str) -> OrganizationAuthToken: token=api_key, token_type=OrganizationAuthTokenType.api, ) - LOG.info(f"Created API token for organization", organization_id=org_id) + LOG.info("Created API token for organization", organization_id=org_id) return org_auth_token diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.py index 521f8acb..7599bbe8 100644 --- a/skyvern/forge/sdk/workflow/context_manager.py +++ b/skyvern/forge/sdk/workflow/context_manager.py @@ -93,7 +93,7 @@ class WorkflowRunContext: assume it's an actual parameter value and return it. """ - if type(secret_id_or_value) is str: + if isinstance(secret_id_or_value, str): return self.secrets.get(secret_id_or_value) return None @@ -149,7 +149,7 @@ class WorkflowRunContext: url = self.values[parameter.url_parameter_key] else: LOG.error(f"URL parameter {parameter.url_parameter_key} not found or has no value") - raise ValueError(f"URL parameter for Bitwarden login credentials not found or has no value") + raise ValueError("URL parameter for Bitwarden login credentials not found or has no value") try: secret_credentials = BitwardenService.get_secret_value_from_url( @@ -224,7 +224,9 @@ class WorkflowRunContext: await self.set_parameter_values_for_output_parameter_dependent_blocks(parameter, value) async def set_parameter_values_for_output_parameter_dependent_blocks( - self, output_parameter: OutputParameter, value: dict[str, Any] | list | str | None + self, + output_parameter: OutputParameter, + value: dict[str, Any] | list | str | None, ) -> None: for key, parameter in self.parameters.items(): if ( @@ -268,7 +270,7 @@ class WorkflowRunContext: isinstance(x, ContextParameter), # This makes sure that ContextParameters witha ContextParameter source are processed after all other # ContextParameters - isinstance(x.source, ContextParameter) if isinstance(x, ContextParameter) else False, + (isinstance(x.source, ContextParameter) if isinstance(x, ContextParameter) else False), isinstance(x, BitwardenLoginCredentialParameter), ) ) diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py index a80558a6..86e8e2d3 100644 --- a/skyvern/forge/sdk/workflow/models/block.py +++ b/skyvern/forge/sdk/workflow/models/block.py @@ -81,16 +81,20 @@ class Block(BaseModel, abc.ABC): value=value, ) LOG.info( - f"Registered output parameter value", + "Registered output parameter value", output_parameter_id=self.output_parameter.output_parameter_id, workflow_run_id=workflow_run_id, ) def build_block_result( - self, success: bool, output_parameter_value: dict[str, Any] | list | str | None = None + self, + success: bool, + output_parameter_value: dict[str, Any] | list | str | None = None, ) -> BlockResult: return BlockResult( - success=success, output_parameter=self.output_parameter, output_parameter_value=output_parameter_value + success=success, + output_parameter=self.output_parameter, + output_parameter_value=output_parameter_value, ) @classmethod @@ -236,11 +240,14 @@ class TaskBlock(Block): workflow_run=workflow_run, url=self.url ) if not browser_state.page: - LOG.error("BrowserState has no page", workflow_run_id=workflow_run.workflow_run_id) + LOG.error( + "BrowserState has no page", + workflow_run_id=workflow_run.workflow_run_id, + ) raise MissingBrowserStatePage(workflow_run_id=workflow_run.workflow_run_id) LOG.info( - f"Navigating to page", + "Navigating to page", url=self.url, workflow_run_id=workflow_run_id, task_id=task.task_id, @@ -253,7 +260,12 @@ class TaskBlock(Block): await browser_state.page.goto(self.url, timeout=settings.BROWSER_LOADING_TIMEOUT_MS) try: - await app.agent.execute_step(organization=organization, task=task, step=step, workflow_run=workflow_run) + await app.agent.execute_step( + organization=organization, + task=task, + step=step, + workflow_run=workflow_run, + ) except Exception as e: # Make sure the task is marked as failed in the database before raising the exception await app.DATABASE.update_task( @@ -273,7 +285,7 @@ class TaskBlock(Block): if updated_task.status == TaskStatus.completed or updated_task.status == TaskStatus.terminated: LOG.info( - f"Task completed", + "Task completed", task_id=updated_task.task_id, task_status=updated_task.status, workflow_run_id=workflow_run_id, @@ -400,7 +412,7 @@ class ForLoopBlock(Block): ) if not loop_over_values or len(loop_over_values) == 0: LOG.info( - f"No loop_over values found", + "No loop_over values found", block_type=self.block_type, workflow_run_id=workflow_run_id, num_loop_over_values=len(loop_over_values), @@ -519,7 +531,11 @@ class TextPromptBlock(Block): + json.dumps(self.json_schema, indent=2) + "\n```\n\n" ) - LOG.info("TextPromptBlock: Sending prompt to LLM", prompt=prompt, llm_key=self.llm_key) + LOG.info( + "TextPromptBlock: Sending prompt to LLM", + prompt=prompt, + llm_key=self.llm_key, + ) response = await llm_api_handler(prompt=prompt) LOG.info("TextPromptBlock: Received response from LLM", response=response) return response @@ -692,7 +708,12 @@ class SendEmailBlock(Block): workflow_run_id: str, ) -> list[PARAMETER_TYPE]: workflow_run_context = self.get_workflow_run_context(workflow_run_id) - parameters = [self.smtp_host, self.smtp_port, self.smtp_username, self.smtp_password] + parameters = [ + self.smtp_host, + self.smtp_port, + self.smtp_username, + self.smtp_password, + ] if self.file_attachments: for file_path in self.file_attachments: @@ -732,7 +753,12 @@ class SendEmailBlock(Block): if email_config_problems: raise InvalidEmailClientConfiguration(email_config_problems) - return smtp_host_value, smtp_port_value, smtp_username_value, smtp_password_value + return ( + smtp_host_value, + smtp_port_value, + smtp_username_value, + smtp_password_value, + ) def _get_file_paths(self, workflow_run_context: WorkflowRunContext, workflow_run_id: str) -> list[str]: file_paths = [] @@ -846,7 +872,12 @@ class SendEmailBlock(Block): subtype=subtype, ) with open(path, "rb") as fp: - msg.add_attachment(fp.read(), maintype=maintype, subtype=subtype, filename=attachment_filename) + msg.add_attachment( + fp.read(), + maintype=maintype, + subtype=subtype, + filename=attachment_filename, + ) finally: if path: os.unlink(path) @@ -884,6 +915,12 @@ class SendEmailBlock(Block): BlockSubclasses = Union[ - ForLoopBlock, TaskBlock, CodeBlock, TextPromptBlock, DownloadToS3Block, UploadToS3Block, SendEmailBlock + ForLoopBlock, + TaskBlock, + CodeBlock, + TextPromptBlock, + DownloadToS3Block, + UploadToS3Block, + SendEmailBlock, ] BlockTypeVar = Annotated[BlockSubclasses, Field(discriminator="block_type")] diff --git a/skyvern/forge/sdk/workflow/models/parameter.py b/skyvern/forge/sdk/workflow/models/parameter.py index adc71e2b..8d50469a 100644 --- a/skyvern/forge/sdk/workflow/models/parameter.py +++ b/skyvern/forge/sdk/workflow/models/parameter.py @@ -114,6 +114,10 @@ class OutputParameter(Parameter): ParameterSubclasses = Union[ - WorkflowParameter, ContextParameter, AWSSecretParameter, BitwardenLoginCredentialParameter, OutputParameter + WorkflowParameter, + ContextParameter, + AWSSecretParameter, + BitwardenLoginCredentialParameter, + OutputParameter, ] PARAMETER_TYPE = Annotated[ParameterSubclasses, Field(discriminator="parameter_type")] diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py index c9c886e1..183c62ec 100644 --- a/skyvern/forge/sdk/workflow/service.py +++ b/skyvern/forge/sdk/workflow/service.py @@ -166,7 +166,10 @@ class WorkflowService: wp_wps_tuples = await self.get_workflow_run_parameter_tuples(workflow_run_id=workflow_run.workflow_run_id) workflow_output_parameters = await self.get_workflow_output_parameters(workflow_id=workflow.workflow_id) app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context( - workflow_run_id, wp_wps_tuples, workflow_output_parameters, context_parameters + workflow_run_id, + wp_wps_tuples, + workflow_output_parameters, + context_parameters, ) # Execute workflow blocks blocks = workflow.workflow_definition.blocks @@ -203,7 +206,11 @@ class WorkflowService: ) else: await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run.workflow_run_id) - await self.send_workflow_response(workflow=workflow, workflow_run=workflow_run, api_key=api_key) + await self.send_workflow_response( + workflow=workflow, + workflow_run=workflow_run, + api_key=api_key, + ) return workflow_run except Exception: @@ -224,13 +231,21 @@ class WorkflowService: # Create a mapping of status to (action, log_func, log_message) status_action_mapping = { - TaskStatus.running: (None, LOG.error, "has running tasks, this should not happen"), + TaskStatus.running: ( + None, + LOG.error, + "has running tasks, this should not happen", + ), TaskStatus.terminated: ( self.mark_workflow_run_as_terminated, LOG.warning, "has terminated tasks, marking as terminated", ), - TaskStatus.failed: (self.mark_workflow_run_as_failed, LOG.warning, "has failed tasks, marking as failed"), + TaskStatus.failed: ( + self.mark_workflow_run_as_failed, + LOG.warning, + "has failed tasks, marking as failed", + ), TaskStatus.completed: ( self.mark_workflow_run_as_completed, LOG.info, @@ -333,7 +348,7 @@ class WorkflowService: title=title, organization_id=organization_id, description=description, - workflow_definition=workflow_definition.model_dump() if workflow_definition else None, + workflow_definition=(workflow_definition.model_dump() if workflow_definition else None), ) async def delete_workflow_by_permanent_id( @@ -529,7 +544,10 @@ class WorkflowService: for task in workflow_run_tasks[::-1]: screenshot_artifact = await app.DATABASE.get_latest_artifact( task_id=task.task_id, - artifact_types=[ArtifactType.SCREENSHOT_ACTION, ArtifactType.SCREENSHOT_FINAL], + artifact_types=[ + ArtifactType.SCREENSHOT_ACTION, + ArtifactType.SCREENSHOT_FINAL, + ], organization_id=organization_id, ) if screenshot_artifact: @@ -541,17 +559,19 @@ class WorkflowService: recording_url = None recording_artifact = await app.DATABASE.get_artifact_for_workflow_run( - workflow_run_id=workflow_run_id, artifact_type=ArtifactType.RECORDING, organization_id=organization_id + workflow_run_id=workflow_run_id, + artifact_type=ArtifactType.RECORDING, + organization_id=organization_id, ) if recording_artifact: recording_url = await app.ARTIFACT_MANAGER.get_share_link(recording_artifact) workflow_parameter_tuples = await app.DATABASE.get_workflow_run_parameters(workflow_run_id=workflow_run_id) parameters_with_value = {wfp.key: wfrp.value for wfp, wfrp in workflow_parameter_tuples} - output_parameter_tuples: list[tuple[OutputParameter, WorkflowRunOutputParameter]] = ( - await self.get_output_parameter_workflow_run_output_parameter_tuples( - workflow_id=workflow_id, workflow_run_id=workflow_run_id - ) + output_parameter_tuples: list[ + tuple[OutputParameter, WorkflowRunOutputParameter] + ] = await self.get_output_parameter_workflow_run_output_parameter_tuples( + workflow_id=workflow_id, workflow_run_id=workflow_run_id ) if output_parameter_tuples: outputs = {output_parameter.key: output.value for output_parameter, output in output_parameter_tuples} @@ -587,7 +607,9 @@ class WorkflowService: tasks = await self.get_tasks_by_workflow_run_id(workflow_run.workflow_run_id) all_workflow_task_ids = [task.task_id for task in tasks] browser_state = await app.BROWSER_MANAGER.cleanup_for_workflow_run( - workflow_run.workflow_run_id, all_workflow_task_ids, close_browser_on_completion + workflow_run.workflow_run_id, + all_workflow_task_ids, + close_browser_on_completion, ) if browser_state: await self.persist_video_data(browser_state, workflow, workflow_run) @@ -600,7 +622,10 @@ class WorkflowService: workflow_run_id=workflow_run.workflow_run_id, organization_id=workflow.organization_id, ) - LOG.info("Built workflow run status response", workflow_run_status_response=workflow_run_status_response) + LOG.info( + "Built workflow run status response", + workflow_run_status_response=workflow_run_status_response, + ) if not workflow_run.webhook_callback_url: LOG.warning( @@ -661,7 +686,8 @@ class WorkflowService: ) except Exception as e: raise FailedToSendWebhook( - workflow_id=workflow.workflow_id, workflow_run_id=workflow_run.workflow_run_id + workflow_id=workflow.workflow_id, + workflow_run_id=workflow_run.workflow_run_id, ) from e async def persist_video_data( @@ -681,10 +707,16 @@ class WorkflowService: ) async def persist_har_data( - self, browser_state: BrowserState, last_step: Step, workflow: Workflow, workflow_run: WorkflowRun + self, + browser_state: BrowserState, + last_step: Step, + workflow: Workflow, + workflow_run: WorkflowRun, ) -> None: har_data = await app.BROWSER_MANAGER.get_har_data( - workflow_id=workflow.workflow_id, workflow_run_id=workflow_run.workflow_run_id, browser_state=browser_state + workflow_id=workflow.workflow_id, + workflow_run_id=workflow_run.workflow_run_id, + browser_state=browser_state, ) if har_data: await app.ARTIFACT_MANAGER.create_artifact( @@ -703,7 +735,11 @@ class WorkflowService: await app.ARTIFACT_MANAGER.create_artifact(step=last_step, artifact_type=ArtifactType.TRACE, path=trace_path) async def persist_debug_artifacts( - self, browser_state: BrowserState, last_task: Task, workflow: Workflow, workflow_run: WorkflowRun + self, + browser_state: BrowserState, + last_task: Task, + workflow: Workflow, + workflow_run: WorkflowRun, ) -> None: last_step = await app.DATABASE.get_latest_step( task_id=last_task.task_id, organization_id=last_task.organization_id @@ -720,7 +756,11 @@ class WorkflowService: request: WorkflowCreateYAMLRequest, workflow_permanent_id: str | None = None, ) -> Workflow: - LOG.info("Creating workflow from request", organization_id=organization_id, title=request.title) + LOG.info( + "Creating workflow from request", + organization_id=organization_id, + title=request.title, + ) try: if workflow_permanent_id: existing_latest_workflow = await self.get_workflow_by_permanent_id( @@ -769,7 +809,8 @@ class WorkflowService: # Create output parameters for all blocks block_output_parameters = await WorkflowService._create_all_output_parameters_for_workflow( - workflow_id=workflow.workflow_id, block_yamls=request.workflow_definition.blocks + workflow_id=workflow.workflow_id, + block_yamls=request.workflow_definition.blocks, ) for block_output_parameter in block_output_parameters.values(): parameters[block_output_parameter.key] = block_output_parameter @@ -822,7 +863,8 @@ class WorkflowService: for context_parameter in context_parameter_yamls: if context_parameter.source_parameter_key not in parameters: raise ContextParameterSourceNotDefined( - context_parameter_key=context_parameter.key, source_key=context_parameter.source_parameter_key + context_parameter_key=context_parameter.key, + source_key=context_parameter.source_parameter_key, ) if context_parameter.key in parameters: @@ -901,7 +943,9 @@ class WorkflowService: @staticmethod async def block_yaml_to_block( - workflow: Workflow, block_yaml: BLOCK_YAML_TYPES, parameters: dict[str, Parameter] + workflow: Workflow, + block_yaml: BLOCK_YAML_TYPES, + parameters: dict[str, Parameter], ) -> BlockTypeVar: output_parameter = parameters[f"{block_yaml.label}_output"] if block_yaml.block_type == BlockType.TASK: diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py index 2ecb2799..f94ce382 100644 --- a/skyvern/webeye/actions/actions.py +++ b/skyvern/webeye/actions/actions.py @@ -157,7 +157,12 @@ def parse_actions(task: Task, json_response: List[Dict[str, Any]]) -> List[Actio reasoning=reasoning, actions=actions, ) - actions.append(TerminateAction(reasoning=reasoning, errors=action["errors"] if "errors" in action else [])) + actions.append( + TerminateAction( + reasoning=reasoning, + errors=action["errors"] if "errors" in action else [], + ) + ) elif action_type == ActionType.CLICK: file_url = action["file_url"] if "file_url" in action else None actions.append( @@ -173,11 +178,21 @@ def parse_actions(task: Task, json_response: List[Dict[str, Any]]) -> List[Actio elif action_type == ActionType.UPLOAD_FILE: # TODO: see if the element is a file input element. if it's not, convert this action into a click action - actions.append(UploadFileAction(element_id=element_id, file_url=action["file_url"], reasoning=reasoning)) + actions.append( + UploadFileAction( + element_id=element_id, + file_url=action["file_url"], + reasoning=reasoning, + ) + ) # This action is not used in the current implementation. Click actions are used instead. elif action_type == ActionType.DOWNLOAD_FILE: actions.append( - DownloadFileAction(element_id=element_id, file_name=action["file_name"], reasoning=reasoning) + DownloadFileAction( + element_id=element_id, + file_name=action["file_name"], + reasoning=reasoning, + ) ) elif action_type == ActionType.SELECT_OPTION: actions.append( @@ -192,7 +207,13 @@ def parse_actions(task: Task, json_response: List[Dict[str, Any]]) -> List[Actio ) ) elif action_type == ActionType.CHECKBOX: - actions.append(CheckboxAction(element_id=element_id, is_checked=action["is_checked"], reasoning=reasoning)) + actions.append( + CheckboxAction( + element_id=element_id, + is_checked=action["is_checked"], + reasoning=reasoning, + ) + ) elif action_type == ActionType.WAIT: actions.append(WaitAction(reasoning=reasoning)) elif action_type == ActionType.COMPLETE: diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index d5acb1b1..98f816c2 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -38,15 +38,18 @@ LOG = structlog.get_logger() class ActionHandler: _handled_action_types: dict[ - ActionType, Callable[[Action, Page, ScrapedPage, Task, Step], Awaitable[list[ActionResult]]] + ActionType, + Callable[[Action, Page, ScrapedPage, Task, Step], Awaitable[list[ActionResult]]], ] = {} _setup_action_types: dict[ - ActionType, Callable[[Action, Page, ScrapedPage, Task, Step], Awaitable[list[ActionResult]]] + ActionType, + Callable[[Action, Page, ScrapedPage, Task, Step], Awaitable[list[ActionResult]]], ] = {} _teardown_action_types: dict[ - ActionType, Callable[[Action, Page, ScrapedPage, Task, Step], Awaitable[list[ActionResult]]] + ActionType, + Callable[[Action, Page, ScrapedPage, Task, Step], Awaitable[list[ActionResult]]], ] = {} @classmethod @@ -111,10 +114,19 @@ class ActionHandler: return actions_result else: - LOG.error("Unsupported action type in handler", action=action, type=type(action)) + LOG.error( + "Unsupported action type in handler", + action=action, + type=type(action), + ) return [ActionFailure(Exception(f"Unsupported action type: {type(action)}"))] except MissingElement as e: - LOG.info("Known exceptions", action=action, exception_type=type(e), exception_message=str(e)) + LOG.info( + "Known exceptions", + action=action, + exception_type=type(e), + exception_message=str(e), + ) return [ActionFailure(e)] except MultipleElementsFound as e: LOG.exception( @@ -128,7 +140,11 @@ class ActionHandler: async def handle_solve_captcha_action( - action: actions.SolveCaptchaAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.SolveCaptchaAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: LOG.warning( "Please solve the captcha on the page, you have 30 seconds", @@ -139,14 +155,22 @@ async def handle_solve_captcha_action( async def handle_click_action( - action: actions.ClickAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.ClickAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: xpath = await validate_actions_in_dom(action, page, scraped_page) await asyncio.sleep(0.3) if action.download: return await handle_click_to_download_file_action(action, page, scraped_page) return await chain_click( - task, page, action, xpath, timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS + task, + page, + action, + xpath, + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, ) @@ -158,7 +182,9 @@ async def handle_click_to_download_file_action( xpath = await validate_actions_in_dom(action, page, scraped_page) try: await page.click( - f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, modifiers=["Alt"] + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + modifiers=["Alt"], ) except Exception as e: LOG.exception("ClickAction with download failed", action=action, exc_info=True) @@ -168,7 +194,11 @@ async def handle_click_to_download_file_action( async def handle_input_text_action( - action: actions.InputTextAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.InputTextAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: xpath = await validate_actions_in_dom(action, page, scraped_page) locator = page.locator(f"xpath={xpath}") @@ -184,7 +214,11 @@ async def handle_input_text_action( async def handle_upload_file_action( - action: actions.UploadFileAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.UploadFileAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: if not action.file_url: LOG.warning("InputFileAction has no file_url", action=action) @@ -209,7 +243,8 @@ async def handle_upload_file_action( LOG.info("Taking UploadFileAction. Found file input tag", action=action) if file_path: await page.locator(f"xpath={xpath}").set_input_files( - file_path, timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS + file_path, + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, ) # Sleep for 10 seconds after uploading a file to let the page process it @@ -222,13 +257,21 @@ async def handle_upload_file_action( # treat it as a click action action.is_upload_file_tag = False return await chain_click( - task, page, action, xpath, timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS + task, + page, + action, + xpath, + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, ) @deprecated("This function is deprecated. Downloads are handled by the click action handler now.") async def handle_download_file_action( - action: actions.DownloadFileAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.DownloadFileAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: xpath = await validate_actions_in_dom(action, page, scraped_page) file_name = f"{action.file_name or uuid.uuid4()}" @@ -238,7 +281,9 @@ async def handle_download_file_action( async with page.expect_download() as download_info: await asyncio.sleep(0.3) await page.click( - f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, modifiers=["Alt"] + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + modifiers=["Alt"], ) download = await download_info.value @@ -260,20 +305,33 @@ async def handle_download_file_action( async def handle_null_action( - action: actions.NullAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.NullAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: return [ActionSuccess()] async def handle_select_option_action( - action: actions.SelectOptionAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.SelectOptionAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: xpath = await validate_actions_in_dom(action, page, scraped_page) locator = page.locator(f"xpath={xpath}") tag_name = await get_tag_name_lowercase(locator) element_dict = scraped_page.id_to_element_dict[action.element_id] - LOG.info("SelectOptionAction", action=action, tag_name=tag_name, element_dict=element_dict) + LOG.info( + "SelectOptionAction", + action=action, + tag_name=tag_name, + element_dict=element_dict, + ) # if element is not a select option, prioritize clicking the linked element if any if tag_name != "select" and "linked_element" in element_dict: @@ -290,7 +348,11 @@ async def handle_select_option_action( linked_element=element_dict["linked_element"], ) return [ActionSuccess()] - LOG.warning("Failed to click linked element", action=action, linked_element=element_dict["linked_element"]) + LOG.warning( + "Failed to click linked element", + action=action, + linked_element=element_dict["linked_element"], + ) # check if the element is an a tag first. If yes, click it instead of selecting the option if tag_name == "label": @@ -360,7 +422,7 @@ async def handle_select_option_action( except Exception as e: LOG.error("Failed to click option", action=action, exc_info=True) return [ActionFailure(e)] - return [ActionFailure(Exception(f"SelectOption option index is missing"))] + return [ActionFailure(Exception("SelectOption option index is missing"))] elif role_attribute == "option": LOG.info( "SelectOptionAction on an option element. Clicking the option", @@ -373,7 +435,7 @@ async def handle_select_option_action( LOG.error( "SelectOptionAction on a non-listbox element. Cannot handle this action", ) - return [ActionFailure(Exception(f"Cannot handle SelectOptionAction on a non-listbox element"))] + return [ActionFailure(Exception("Cannot handle SelectOptionAction on a non-listbox element"))] elif tag_name == "input" and element_dict.get("attributes", {}).get("type", None) in ["radio", "checkbox"]: LOG.info( "SelectOptionAction is on checkbox/radio", @@ -387,13 +449,19 @@ async def handle_select_option_action( return [ActionSuccess()] try: # First click by label (if it matches) - await page.click(f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) await page.select_option( xpath, label=action.option.label, timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, ) - await page.click(f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) return [ActionSuccess()] except Exception as e: if action.option.index is not None: @@ -418,23 +486,35 @@ async def handle_select_option_action( if match: # This means we were trying to select an option xpath, click the option option_index = int(match.group(1)) - await page.click(f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) await page.select_option( xpath, index=option_index, timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, ) - await page.click(f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) return [ActionSuccess()] else: # This means the supplied index was for the select element, not a reference to the xpath dict - await page.click(f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) await page.select_option( xpath, index=action.option.index, timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, ) - await page.click(f"xpath={xpath}", timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) return [ActionSuccess()] except Exception as e: LOG.warning("Failed to click on the option by index", action=action, exc_info=True) @@ -442,7 +522,11 @@ async def handle_select_option_action( async def handle_checkbox_action( - self: actions.CheckboxAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + self: actions.CheckboxAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: """ ******* NOT REGISTERED ******* @@ -462,20 +546,32 @@ async def handle_checkbox_action( async def handle_wait_action( - action: actions.WaitAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.WaitAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: await asyncio.sleep(10) return [ActionFailure(exception=Exception("Wait action is treated as a failure"))] async def handle_terminate_action( - action: actions.TerminateAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.TerminateAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: return [ActionSuccess()] async def handle_complete_action( - action: actions.CompleteAction, page: Page, scraped_page: ScrapedPage, task: Task, step: Step + action: actions.CompleteAction, + page: Page, + scraped_page: ScrapedPage, + task: Task, + step: Step, ) -> list[ActionResult]: extracted_data = None if action.data_extraction_goal: @@ -526,7 +622,11 @@ async def validate_actions_in_dom(action: WebAction, page: Page, scraped_page: S num_elements = await locator.count() if num_elements < 1: - LOG.warning("No elements found with action xpath. Validation failed.", action=action, xpath=xpath) + LOG.warning( + "No elements found with action xpath. Validation failed.", + action=action, + xpath=xpath, + ) raise MissingElement(xpath=xpath, element_id=action.element_id) elif num_elements > 1: LOG.warning( @@ -560,10 +660,14 @@ async def chain_click( try: file = await download_file(file_url) except Exception: - LOG.exception("Failed to download file, continuing without it", action=action, file_url=file_url) + LOG.exception( + "Failed to download file, continuing without it", + action=action, + file_url=file_url, + ) file = [] - fc_func = lambda fc: fc.set_files(files=file) + fc_func = lambda fc: fc.set_files(files=file) # noqa: E731 page.on("filechooser", fc_func) LOG.info("Registered file chooser listener", action=action, path=file) @@ -585,13 +689,26 @@ async def chain_click( try: await page.click(f"xpath={xpath}", timeout=timeout) LOG.info("Chain click: main element click succeeded", action=action, xpath=xpath) - return [ActionSuccess(javascript_triggered=javascript_triggered, download_triggered=download_triggered)] + return [ + ActionSuccess( + javascript_triggered=javascript_triggered, + download_triggered=download_triggered, + ) + ] except Exception as e: action_results: list[ActionResult] = [ - ActionFailure(e, javascript_triggered=javascript_triggered, download_triggered=download_triggered) + ActionFailure( + e, + javascript_triggered=javascript_triggered, + download_triggered=download_triggered, + ) ] if await is_input_element(page.locator(xpath)): - LOG.info("Chain click: it's an input element. going to try sibling click", action=action, xpath=xpath) + LOG.info( + "Chain click: it's an input element. going to try sibling click", + action=action, + xpath=xpath, + ) sibling_action_result = await click_sibling_of_input(page.locator(xpath), timeout=timeout) sibling_action_result.download_triggered = download_triggered action_results.append(sibling_action_result) @@ -604,7 +721,11 @@ async def chain_click( javascript_triggered = javascript_triggered or parent_javascript_triggered parent_locator = page.locator(xpath).locator("..") await parent_locator.click(timeout=timeout) - LOG.info("Chain click: successfully clicked parent element", action=action, parent_xpath=parent_xpath) + LOG.info( + "Chain click: successfully clicked parent element", + action=action, + parent_xpath=parent_xpath, + ) action_results.append( ActionSuccess( javascript_triggered=javascript_triggered, @@ -613,9 +734,18 @@ async def chain_click( ) ) except Exception as pe: - LOG.warning("Failed to click parent element", action=action, parent_xpath=parent_xpath, exc_info=True) + LOG.warning( + "Failed to click parent element", + action=action, + parent_xpath=parent_xpath, + exc_info=True, + ) action_results.append( - ActionFailure(pe, javascript_triggered=javascript_triggered, interacted_with_parent=True) + ActionFailure( + pe, + javascript_triggered=javascript_triggered, + interacted_with_parent=True, + ) ) # We don't raise exception here because we do log the exception, and return ActionFailure as the last action @@ -765,7 +895,7 @@ async def extract_information_for_navigation_goal( extracted_information_schema=task.extracted_information_schema, current_url=scraped_page.url, extracted_text=scraped_page.extracted_text, - error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else None, + error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None), ) json_response = await app.LLM_API_HANDLER( @@ -804,7 +934,12 @@ async def click_listbox_option( await page.click(f"xpath={option_xpath}", timeout=1000) return True except Exception: - LOG.error("Failed to click on the option", action=action, option_xpath=option_xpath, exc_info=True) + LOG.error( + "Failed to click on the option", + action=action, + option_xpath=option_xpath, + exc_info=True, + ) if "children" in child: bfs_queue.extend(child["children"]) return False diff --git a/skyvern/webeye/actions/models.py b/skyvern/webeye/actions/models.py index ac353173..b480d7ea 100644 --- a/skyvern/webeye/actions/models.py +++ b/skyvern/webeye/actions/models.py @@ -63,6 +63,6 @@ class DetailedAgentStepOutput(BaseModel): def to_agent_step_output(self) -> AgentStepOutput: return AgentStepOutput( action_results=self.action_results if self.action_results else [], - actions_and_results=self.actions_and_results if self.actions_and_results else [], + actions_and_results=(self.actions_and_results if self.actions_and_results else []), errors=self.extract_errors(), ) diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py index e083050b..2cc5b162 100644 --- a/skyvern/webeye/browser_factory.py +++ b/skyvern/webeye/browser_factory.py @@ -62,7 +62,10 @@ class BrowserContextFactory: ], "record_har_path": har_dir, "record_video_dir": video_dir, - "viewport": {"width": settings.BROWSER_WIDTH, "height": settings.BROWSER_HEIGHT}, + "viewport": { + "width": settings.BROWSER_WIDTH, + "height": settings.BROWSER_HEIGHT, + }, } @staticmethod @@ -73,7 +76,10 @@ class BrowserContextFactory: traces_dir: str | None = None, ) -> BrowserArtifacts: return BrowserArtifacts( - video_path=video_path, har_path=har_path, video_artifact_id=video_artifact_id, traces_dir=traces_dir + video_path=video_path, + har_path=har_path, + video_artifact_id=video_artifact_id, + traces_dir=traces_dir, ) @classmethod @@ -156,7 +162,10 @@ class BrowserState: LOG.info("playwright is started") if self.browser_context is None: LOG.info("creating browser context") - browser_context, browser_artifacts = await BrowserContextFactory.create_browser_context(self.pw, url=url) + ( + browser_context, + browser_artifacts, + ) = await BrowserContextFactory.create_browser_context(self.pw, url=url) self.browser_context = browser_context self.browser_artifacts = browser_artifacts LOG.info("browser context is created") @@ -179,7 +188,11 @@ class BrowserState: start_time = time.time() await self.page.goto(url, timeout=settings.BROWSER_LOADING_TIMEOUT_MS) end_time = time.time() - LOG.info(f"Page loading time", loading_time=end_time - start_time, url=url) + LOG.info( + "Page loading time", + loading_time=end_time - start_time, + url=url, + ) except Error as playright_error: LOG.exception(f"Error while navigating to url: {str(playright_error)}") raise FailedToNavigateToUrl(url=url, error_message=str(playright_error)) @@ -239,7 +252,7 @@ class BrowserState: ) end_time = time.time() LOG.info( - f"Screenshot taking time", + "Screenshot taking time", screenshot_time=end_time - start_time, full_page=full_page, file_path=file_path, diff --git a/skyvern/webeye/browser_manager.py b/skyvern/webeye/browser_manager.py index e2c806a5..85fed399 100644 --- a/skyvern/webeye/browser_manager.py +++ b/skyvern/webeye/browser_manager.py @@ -27,7 +27,10 @@ class BrowserManager: task_id: str | None = None, ) -> BrowserState: pw = await async_playwright().start() - browser_context, browser_artifacts = await BrowserContextFactory.create_browser_context( + ( + browser_context, + browser_artifacts, + ) = await BrowserContextFactory.create_browser_context( pw, proxy_location=proxy_location, url=url, @@ -67,7 +70,10 @@ class BrowserManager: async def get_or_create_for_workflow_run(self, workflow_run: WorkflowRun, url: str | None = None) -> BrowserState: if workflow_run.workflow_run_id in self.pages: return self.pages[workflow_run.workflow_run_id] - LOG.info("Creating browser state for workflow run", workflow_run_id=workflow_run.workflow_run_id) + LOG.info( + "Creating browser state for workflow run", + workflow_run_id=workflow_run.workflow_run_id, + ) browser_state = await self._create_browser_state(workflow_run.proxy_location, url=url) # The URL here is only used when creating a new page, and not when using an existing page. @@ -102,7 +108,11 @@ class BrowserManager: raise MissingBrowserState(task_id=task.task_id) async def get_video_data( - self, browser_state: BrowserState, task_id: str = "", workflow_id: str = "", workflow_run_id: str = "" + self, + browser_state: BrowserState, + task_id: str = "", + workflow_id: str = "", + workflow_run_id: str = "", ) -> bytes: if browser_state: path = browser_state.browser_artifacts.video_path @@ -113,12 +123,19 @@ class BrowserManager: except FileNotFoundError: pass LOG.warning( - "Video data not found for task", task_id=task_id, workflow_id=workflow_id, workflow_run_id=workflow_run_id + "Video data not found for task", + task_id=task_id, + workflow_id=workflow_id, + workflow_run_id=workflow_run_id, ) return b"" async def get_har_data( - self, browser_state: BrowserState, task_id: str = "", workflow_id: str = "", workflow_run_id: str = "" + self, + browser_state: BrowserState, + task_id: str = "", + workflow_id: str = "", + workflow_run_id: str = "", ) -> bytes: if browser_state: path = browser_state.browser_artifacts.har_path @@ -126,7 +143,10 @@ class BrowserManager: with open(path, "rb") as f: return f.read() LOG.warning( - "HAR data not found for task", task_id=task_id, workflow_id=workflow_id, workflow_run_id=workflow_run_id + "HAR data not found for task", + task_id=task_id, + workflow_id=workflow_id, + workflow_run_id=workflow_run_id, ) return b"" @@ -154,7 +174,10 @@ class BrowserManager: return browser_state_to_close async def cleanup_for_workflow_run( - self, workflow_run_id: str, task_ids: list[str], close_browser_on_completion: bool = True + self, + workflow_run_id: str, + task_ids: list[str], + close_browser_on_completion: bool = True, ) -> BrowserState | None: LOG.info("Cleaning up for workflow run") browser_state_to_close = self.pages.pop(workflow_run_id, None) diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py index 7f4dcd4c..a18d03e4 100644 --- a/skyvern/webeye/scraper/scraper.py +++ b/skyvern/webeye/scraper/scraper.py @@ -241,7 +241,11 @@ async def scrape_web_unsafe( scroll_y_px_old = scroll_y_px LOG.info("Scrolling to next page", url=url, num_screenshots=len(screenshots)) scroll_y_px = await scroll_to_next_page(page, drow_boxes=True) - LOG.info("Scrolled to next page", scroll_y_px=scroll_y_px, scroll_y_px_old=scroll_y_px_old) + LOG.info( + "Scrolled to next page", + scroll_y_px=scroll_y_px, + scroll_y_px_old=scroll_y_px_old, + ) await remove_bounding_boxes(page) await scroll_to_top(page, drow_boxes=False) diff --git a/streamlit_app/visualizer/artifact_loader.py b/streamlit_app/visualizer/artifact_loader.py index 7f7b1654..aa74bd9a 100644 --- a/streamlit_app/visualizer/artifact_loader.py +++ b/streamlit_app/visualizer/artifact_loader.py @@ -64,6 +64,18 @@ def streamlit_show_recording(st_obj: Any, uri: str) -> None: content = read_artifact_safe(uri, is_webm=True) # type: ignore if content: random_key = "".join(random.choices(string.ascii_uppercase + string.digits, k=6)) - st_obj.download_button("Download recording", content, f"recording{uri.split('/')[-1]}.webm", key=random_key) + st_obj.download_button( + "Download recording", + content, + f"recording{uri.split('/')[-1]}.webm", + key=random_key, + ) - streamlit_content_safe(st_obj, st_obj.video, content, "No recording available.", format="video/webm", start_time=0) + streamlit_content_safe( + st_obj, + st_obj.video, + content, + "No recording available.", + format="video/webm", + start_time=0, + ) diff --git a/streamlit_app/visualizer/sample_data.py b/streamlit_app/visualizer/sample_data.py index cbaf9446..d4f5d3f1 100644 --- a/streamlit_app/visualizer/sample_data.py +++ b/streamlit_app/visualizer/sample_data.py @@ -199,9 +199,18 @@ geico_sample_data = SampleTaskRequest( "additionalProperties": False, "description": "The vehicle that the collision and comprehensive coverage is for", "properties": { - "make": {"description": "The make of the vehicle", "type": "string"}, - "model": {"description": "The model of the vehicle", "type": "string"}, - "year": {"description": "The year of the vehicle", "type": "string"}, + "make": { + "description": "The make of the vehicle", + "type": "string", + }, + "model": { + "description": "The model of the vehicle", + "type": "string", + }, + "year": { + "description": "The year of the vehicle", + "type": "string", + }, }, "type": "object", }, @@ -225,4 +234,9 @@ geico_sample_data = SampleTaskRequest( ) -supported_examples = [geico_sample_data, finditparts_sample_data, california_edd_sample_data, bci_seguros_sample_data] +supported_examples = [ + geico_sample_data, + finditparts_sample_data, + california_edd_sample_data, + bci_seguros_sample_data, +] diff --git a/streamlit_app/visualizer/streamlit.py b/streamlit_app/visualizer/streamlit.py index c94e47bb..1bf8fd2c 100644 --- a/streamlit_app/visualizer/streamlit.py +++ b/streamlit_app/visualizer/streamlit.py @@ -50,16 +50,18 @@ for config in CONFIGS_DICT: st.sidebar.markdown("#### **Settings**") select_env = st.sidebar.selectbox("Environment", list(SETTINGS.keys()), on_change=reset_session_state) select_org = st.sidebar.selectbox( - "Organization", list(SETTINGS[select_env]["orgs"].keys()), on_change=reset_session_state + "Organization", + list(SETTINGS[select_env]["orgs"].keys()), + on_change=reset_session_state, ) # Hack the sidebar size to be a little bit smaller st.markdown( - f""" + """ """, unsafe_allow_html=True, @@ -68,7 +70,8 @@ st.markdown( # Initialize session state if "client" not in st.session_state: st.session_state.client = SkyvernClient( - base_url=SETTINGS[select_env]["host"], credentials=SETTINGS[select_env]["orgs"][select_org] + base_url=SETTINGS[select_env]["host"], + credentials=SETTINGS[select_env]["orgs"][select_org], ) if "repository" not in st.session_state: st.session_state.repository = TaskRepository(st.session_state.client) @@ -133,7 +136,8 @@ def copy_curl_to_clipboard(task_request_body: TaskRequest) -> None: with execute_tab: # Streamlit doesn't support "focusing" on a tab, so this is a workaround to make the requested tab be the "first" tab sorted_supported_examples = sorted( - supported_examples, key=lambda x: (-1 if x.name.lower() == tab_name.lower() else 0) + supported_examples, + key=lambda x: (-1 if x.name.lower() == tab_name.lower() else 0), ) example_tabs = st.tabs([supported_example.name for supported_example in sorted_supported_examples]) @@ -157,7 +161,9 @@ with execute_tab: # Create all the fields to create a TaskRequest object st_url = st.text_input("URL*", value=example.url, key=f"url_{unique_key}") st_webhook_callback_url = st.text_input( - "Webhook Callback URL", key=f"webhook_{unique_key}", placeholder="Optional" + "Webhook Callback URL", + key=f"webhook_{unique_key}", + placeholder="Optional", ) st_navigation_goal = st.text_area( "Navigation Goal", @@ -252,11 +258,11 @@ with visualizer_tab: col_tasks, _, col_steps, _, col_artifacts = st.columns([4, 1, 6, 1, 18]) - col_tasks.markdown(f"#### Tasks") - col_steps.markdown(f"#### Steps") + col_tasks.markdown("#### Tasks") + col_steps.markdown("#### Steps") col_artifacts.markdown("#### Artifacts") tasks_response = repository.get_tasks(task_page_number) - if type(tasks_response) is not list: + if not isinstance(tasks_response, list): st.error("Failed to fetch tasks.") st.error(tasks_response) st.error( @@ -282,7 +288,7 @@ with visualizer_tab: on_click=select_task, args=(task,), use_container_width=True, - type="primary" if selected_task and task_id == selected_task["task_id"] else "secondary", + type=("primary" if selected_task and task_id == selected_task["task_id"] else "secondary"), ) for task_id, task in tasks.items() } @@ -339,10 +345,16 @@ with visualizer_tab: if task_steps: col_steps_prev, _, col_steps_next = col_steps.columns([3, 1, 3]) col_steps_prev.button( - "prev", on_click=go_to_previous_step, key="previous_step_button", use_container_width=True + "prev", + on_click=go_to_previous_step, + key="previous_step_button", + use_container_width=True, ) col_steps_next.button( - "next", on_click=go_to_next_step, key="next_step_button", use_container_width=True + "next", + on_click=go_to_next_step, + key="next_step_button", + use_container_width=True, ) step_id_buttons = { @@ -351,7 +363,7 @@ with visualizer_tab: on_click=select_step, args=(step,), use_container_width=True, - type="primary" if selected_step and step["step_id"] == selected_step["step_id"] else "secondary", + type=("primary" if selected_step and step["step_id"] == selected_step["step_id"] else "secondary"), ) for step in task_steps } @@ -439,7 +451,10 @@ with visualizer_tab: # tab_llm_prompt.text_area("collapsed", value=content, label_visibility="collapsed", height=1000) elif file_name.endswith("llm_request.json"): streamlit_content_safe( - tab_llm_request, tab_llm_request.json, read_artifact_safe(uri), "No LLM request available." + tab_llm_request, + tab_llm_request.json, + read_artifact_safe(uri), + "No LLM request available.", ) elif file_name.endswith("llm_response_parsed.json"): streamlit_content_safe( @@ -456,8 +471,18 @@ with visualizer_tab: "No raw LLM response available.", ) elif file_name.endswith("html_scrape.html"): - streamlit_content_safe(tab_html, tab_html.text, read_artifact_safe(uri), "No html available.") + streamlit_content_safe( + tab_html, + tab_html.text, + read_artifact_safe(uri), + "No html available.", + ) elif file_name.endswith("html_action.html"): - streamlit_content_safe(tab_html, tab_html.text, read_artifact_safe(uri), "No html available.") + streamlit_content_safe( + tab_html, + tab_html.text, + read_artifact_safe(uri), + "No html available.", + ) else: st.write(f"Artifact {file_name} not supported.")