diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 984f3e05..f3490731 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -12,7 +12,7 @@ body: id: version attributes: label: What version of eigent are you using? - placeholder: E.g., 0.0.85 + placeholder: E.g., 0.0.86 validations: required: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0eecb122..11c5aeca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -116,7 +116,7 @@ jobs: fi ulimit -n 65536 2>/dev/null || ulimit -n 10240 2>/dev/null || true echo "File descriptor limit: $(ulimit -n) (hard: $(ulimit -Hn 2>/dev/null || echo 'N/A'))" - npm run build -- --arch ${{ matrix.arch }} + npm run build -- --${{ matrix.arch }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} CSC_LINK: ${{ secrets.CERT_P12 }} @@ -125,6 +125,7 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} VITE_BASE_URL: ${{ secrets.VITE_BASE_URL }} + VITE_PROXY_URL: ${{ secrets.VITE_PROXY_URL }} VITE_STACK_PROJECT_ID: ${{ secrets.VITE_STACK_PROJECT_ID }} VITE_STACK_PUBLISHABLE_CLIENT_KEY: ${{ secrets.VITE_STACK_PUBLISHABLE_CLIENT_KEY }} VITE_STACK_SECRET_SERVER_KEY: ${{ secrets.VITE_STACK_SECRET_SERVER_KEY }} @@ -134,10 +135,11 @@ jobs: - name: Build Release Files (Windows without signing) if: runner.os == 'Windows' timeout-minutes: 90 - run: npm run build -- --arch ${{ matrix.arch }} + run: npm run build -- --${{ matrix.arch }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VITE_BASE_URL: ${{ secrets.VITE_BASE_URL }} + VITE_PROXY_URL: ${{ secrets.VITE_PROXY_URL }} VITE_STACK_PROJECT_ID: ${{ secrets.VITE_STACK_PROJECT_ID }} VITE_STACK_PUBLISHABLE_CLIENT_KEY: ${{ secrets.VITE_STACK_PUBLISHABLE_CLIENT_KEY }} VITE_STACK_SECRET_SERVER_KEY: ${{ secrets.VITE_STACK_SECRET_SERVER_KEY }} @@ -151,6 +153,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VITE_BASE_URL: ${{ secrets.VITE_BASE_URL }} + VITE_PROXY_URL: ${{ secrets.VITE_PROXY_URL }} VITE_STACK_PROJECT_ID: ${{ secrets.VITE_STACK_PROJECT_ID }} VITE_STACK_PUBLISHABLE_CLIENT_KEY: ${{ secrets.VITE_STACK_PUBLISHABLE_CLIENT_KEY }} VITE_STACK_SECRET_SERVER_KEY: ${{ secrets.VITE_STACK_SECRET_SERVER_KEY }} @@ -344,7 +347,7 @@ jobs: files: | gh-release-assets/* - # Extract version from tag (e.g., v0.0.85 -> 0.0.85) + # Extract version from tag (e.g., v0.0.85 -> 0.0.86) - name: Extract version if: startsWith(github.ref, 'refs/tags/') id: version diff --git a/.github/workflows/lint-markdown.yml b/.github/workflows/lint-markdown.yml index a40dd6e3..d55666ae 100644 --- a/.github/workflows/lint-markdown.yml +++ b/.github/workflows/lint-markdown.yml @@ -33,6 +33,6 @@ jobs: - name: Lint markdown if: steps.changed-files.outputs.any_changed == 'true' - run: npx markdownlint-cli@0.43.0 --ignore 'resources/**' $CHANGED_FILES + run: npx markdownlint-cli@0.43.0 --ignore 'resources/**' --ignore 'backend/benchmark/harbor/template/**' $CHANGED_FILES env: CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.gitignore b/.gitignore index b90aa1bb..371f8962 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,5 @@ resources/prebuilt/cache/ *storybook.log storybook-static nul + +backend/benchmark/jobs/ diff --git a/README.md b/README.md index 65b33bfe..a7575756 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Built on [CAMEL-AI][camel-site]'s acclaimed open-source project, our system intr - ✅ **Zero Setup** - No technical configuration required - ✅ **Multi-Agent Coordination** - Handle complex multi-agent workflows - ✅ **Enterprise Feature** - SSO/Access control -- ✅ **Local Deploymen**t +- ✅ **Local Deployment** - ✅ **Open Source** - ✅ **Custom Model Support** - ✅ **MCP Integration** @@ -70,7 +70,7 @@ Built on [CAMEL-AI][camel-site]'s acclaimed open-source project, our system intr - [🗺️ Roadmap - Open Source Cowork](#%EF%B8%8F-roadmap---open-source-Cowork) - [📖 Contributing](#-contributing) - [Main Contributors](#main-contributors) - - [Distinguished amabssador](#distinguished-amabssador) + - [Distinguished ambassador](#distinguished-ambassador) - [Ecosystem](#ecosystem) - [📄 Open Source License](#-open-source-license) - [🌐 Community & contact](#-community--contact) diff --git a/backend/.gitignore b/backend/.gitignore index 2cc4d0e6..71ed26e2 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -20,3 +20,4 @@ img uv_installing.lock uv_installed.lock benchmark/*_results.csv +benchmark/harbor/jobs/ diff --git a/backend/.pre-commit-config.yaml b/backend/.pre-commit-config.yaml index c16219b6..5c400e84 100644 --- a/backend/.pre-commit-config.yaml +++ b/backend/.pre-commit-config.yaml @@ -21,6 +21,7 @@ repos: exclude: '\.md$' - id: check-toml name: Check toml + exclude: 'benchmark/harbor/template/' - id: check-yaml name: Check yaml diff --git a/backend/app/agent/factory/multi_modal.py b/backend/app/agent/factory/multi_modal.py index e4ce6fdf..f01756f5 100644 --- a/backend/app/agent/factory/multi_modal.py +++ b/backend/app/agent/factory/multi_modal.py @@ -23,11 +23,11 @@ from app.agent.listen_chat_agent import logger from app.agent.prompt import MULTI_MODAL_SYS_PROMPT from app.agent.toolkit.audio_analysis_toolkit import AudioAnalysisToolkit from app.agent.toolkit.human_toolkit import HumanToolkit -from app.agent.toolkit.image_analysis_toolkit import ImageAnalysisToolkit # TODO: Remove NoteTakingToolkit and use TerminalToolkit instead from app.agent.toolkit.note_taking_toolkit import NoteTakingToolkit from app.agent.toolkit.openai_image_toolkit import OpenAIImageToolkit +from app.agent.toolkit.screenshot_toolkit import ScreenshotToolkit from app.agent.toolkit.search_toolkit import SearchToolkit from app.agent.toolkit.skill_toolkit import SkillToolkit from app.agent.toolkit.terminal_toolkit import TerminalToolkit @@ -56,9 +56,15 @@ def multi_modal_agent(options: Chat): video_download_toolkit = message_integration.register_toolkits( video_download_toolkit ) - image_analysis_toolkit = ImageAnalysisToolkit(options.project_id) - image_analysis_toolkit = message_integration.register_toolkits( - image_analysis_toolkit + screenshot_toolkit = ScreenshotToolkit( + options.project_id, + working_directory=working_directory, + agent_name=Agents.multi_modal_agent, + ) + # Save reference before registering for toolkits_to_register_agent + screenshot_toolkit_for_agent_registration = screenshot_toolkit + screenshot_toolkit = message_integration.register_toolkits( + screenshot_toolkit ) terminal_toolkit = TerminalToolkit( @@ -95,7 +101,7 @@ def multi_modal_agent(options: Chat): tools = [ *video_download_toolkit.get_tools(), - *image_analysis_toolkit.get_tools(), + *screenshot_toolkit.get_tools(), *HumanToolkit.get_can_use_tools( options.project_id, Agents.multi_modal_agent ), @@ -161,7 +167,7 @@ def multi_modal_agent(options: Chat): tool_names=[ VideoDownloaderToolkit.toolkit_name(), AudioAnalysisToolkit.toolkit_name(), - ImageAnalysisToolkit.toolkit_name(), + ScreenshotToolkit.toolkit_name(), OpenAIImageToolkit.toolkit_name(), HumanToolkit.toolkit_name(), TerminalToolkit.toolkit_name(), @@ -169,4 +175,7 @@ def multi_modal_agent(options: Chat): SearchToolkit.toolkit_name(), SkillToolkit.toolkit_name(), ], + toolkits_to_register_agent=[ + screenshot_toolkit_for_agent_registration, + ], ) diff --git a/backend/app/agent/prompt.py b/backend/app/agent/prompt.py index 77bd9e2f..40f8d1bc 100644 --- a/backend/app/agent/prompt.py +++ b/backend/app/agent/prompt.py @@ -177,10 +177,11 @@ Your capabilities include: - Handle various audio formats including MP3, WAV, and OGG - Image Analysis & Understanding: + - Use `read_image` to analyze images from local file paths + - Use `take_screenshot_and_read_image` to capture and analyze the screen - Generate detailed descriptions of image content - Answer specific questions about images - Identify objects, text, people, and scenes in images - - Process images from both local files and URLs - Image Generation: - Create high-quality images based on detailed text prompts using DALL-E diff --git a/backend/app/agent/toolkit/terminal_toolkit.py b/backend/app/agent/toolkit/terminal_toolkit.py index e22567d2..bdae15a7 100644 --- a/backend/app/agent/toolkit/terminal_toolkit.py +++ b/backend/app/agent/toolkit/terminal_toolkit.py @@ -41,7 +41,7 @@ logger = logging.getLogger("terminal_toolkit") # App version - should match electron app version # TODO: Consider getting this from a shared config -APP_VERSION = "0.0.85" +APP_VERSION = "0.0.86" def get_terminal_base_venv_path() -> str: diff --git a/backend/app/agent/tools.py b/backend/app/agent/tools.py index f91f4413..c0880b13 100644 --- a/backend/app/agent/tools.py +++ b/backend/app/agent/tools.py @@ -26,7 +26,6 @@ from app.agent.toolkit.github_toolkit import GithubToolkit from app.agent.toolkit.google_calendar_toolkit import GoogleCalendarToolkit from app.agent.toolkit.google_drive_mcp_toolkit import GoogleDriveMCPToolkit from app.agent.toolkit.google_gmail_mcp_toolkit import GoogleGmailMCPToolkit -from app.agent.toolkit.image_analysis_toolkit import ImageAnalysisToolkit from app.agent.toolkit.lark_toolkit import LarkToolkit from app.agent.toolkit.linkedin_toolkit import LinkedInToolkit from app.agent.toolkit.mcp_search_toolkit import McpSearchToolkit @@ -62,7 +61,6 @@ async def get_toolkits(tools: list[str], agent_name: str, api_task_id: str): "google_calendar_toolkit": GoogleCalendarToolkit, "google_drive_mcp_toolkit": GoogleDriveMCPToolkit, "google_gmail_mcp_toolkit": GoogleGmailMCPToolkit, - "image_analysis_toolkit": ImageAnalysisToolkit, "linkedin_toolkit": LinkedInToolkit, "lark_toolkit": LarkToolkit, "mcp_search_toolkit": McpSearchToolkit, diff --git a/backend/benchmark/harbor/.gitignore b/backend/benchmark/harbor/.gitignore new file mode 100644 index 00000000..3133a130 --- /dev/null +++ b/backend/benchmark/harbor/.gitignore @@ -0,0 +1 @@ +datasets/ diff --git a/backend/benchmark/harbor/README.md b/backend/benchmark/harbor/README.md new file mode 100644 index 00000000..92ba3e64 --- /dev/null +++ b/backend/benchmark/harbor/README.md @@ -0,0 +1,155 @@ +# Eigent Benchmark — Harbor Adapter + +Converts eigent benchmark tasks into [Harbor](https://harborframework.com/) task format for standardized agent evaluation. + +## Prerequisites + +```bash +# Install Harbor CLI +uv tool install harbor + +# Set your API key +export ANTHROPIC_API_KEY=your-key-here +``` + +## Quick Start + +```bash +cd backend/benchmark/harbor + +# 1. Generate Harbor tasks +python run_adapter.py + +# 2. Verify with oracle (should score 1.0) +harbor run \ + -p datasets/eigent-bench \ + -a oracle \ + --env docker + +# 3. Run with an agent +harbor run \ + -p datasets/eigent-bench \ + -a claude-code \ + -m anthropic/claude-sonnet-4-20250514 \ + --env docker +``` + +## Generating Tasks + +```bash +cd backend/benchmark/harbor + +# Default (all tasks, 50/50 weights) +python run_adapter.py + +# Custom weights (70% checker, 30% grader) +python run_adapter.py --checker-weight 0.7 --grader-weight 0.3 + +# Limit number of tasks +python run_adapter.py --limit 2 +``` + +## Running Benchmarks + +Harbor builds a Docker image from each task's `environment/Dockerfile` automatically on first run. Subsequent runs reuse the cached image. + +### Run a specific task + +```bash +# Run only task 0 (hello world) +harbor run \ + -p datasets/eigent-bench \ + -a claude-code \ + -m anthropic/claude-haiku-4-5-20251001 \ + -t "eigent-bench-0000" \ + --env docker +``` + +### Run all tasks + +```bash +harbor run \ + -p datasets/eigent-bench \ + -a claude-code \ + -m anthropic/claude-sonnet-4-20250514 \ + --env docker +``` + +### Run with different models + +```bash +# Claude Haiku +harbor run -p datasets/eigent-bench -a claude-code \ + -m anthropic/claude-haiku-4-5-20251001 --env docker + +``` + +### Run with concurrency + +```bash +harbor run \ + -p datasets/eigent-bench \ + -a claude-code \ + -m anthropic/claude-sonnet-4-20250514 \ + --env docker \ + --n-concurrent 3 +``` + +## Rebuild Docker Image + +Harbor caches Docker images. To force a rebuild (e.g., after changing the Dockerfile): + +```bash +# Clear Harbor's Docker build cache +docker builder prune -f + +# Then rerun — Harbor will rebuild the image +harbor run -p datasets/eigent-bench -a oracle --env docker +``` + +## Reward Computation + +``` +reward = checker_weight × checker_score + grader_weight × grader_score +``` + +- `checker_score`: 1.0 if checker passes, 0.0 if fails +- `grader_score`: completed_milestones / total_milestones (0.0–1.0) +- Default weights: 0.5 / 0.5 (configurable via CLI flags) + +## Results + +Results are written to `jobs//result.json`. Each trial directory contains: + +``` +jobs// +├── result.json # Overall results +├── job.log # Build + run logs +└── eigent-bench-NNNN__/ + ├── agent/ # Agent logs and trajectory + │ ├── claude-code.txt # Raw agent output + │ └── trajectory.json # ATIF trajectory + └── verifier/ + ├── metrics.json # Detailed checker/grader results + └── test-stdout.txt # Verifier output +``` + +## Generated Task Structure + +``` +eigent-bench-NNNN/ +├── task.toml # Harbor metadata +├── instruction.md # Task question +├── environment/ +│ ├── Dockerfile # Eigent stack + evaluation deps +│ └── workspace/ +│ └── .env.development +├── tests/ +│ ├── test.sh # Entry point → evaluate.py +│ ├── evaluate.py # Checker+grader → reward.txt +│ ├── checker_N.py # Copied from benchmark/checker/ +│ ├── grader_N.py # Copied from benchmark/grader/ +│ └── config.json # Task config + weights +└── solution/ + └── solve.sh # Oracle solution +``` diff --git a/backend/app/agent/toolkit/image_analysis_toolkit.py b/backend/benchmark/harbor/__init__.py similarity index 50% rename from backend/app/agent/toolkit/image_analysis_toolkit.py rename to backend/benchmark/harbor/__init__.py index ada3090f..fa7455a0 100644 --- a/backend/app/agent/toolkit/image_analysis_toolkit.py +++ b/backend/benchmark/harbor/__init__.py @@ -11,24 +11,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= - -from camel.models import BaseModelBackend -from camel.toolkits import ImageAnalysisToolkit as BaseImageAnalysisToolkit - -from app.agent.toolkit.abstract_toolkit import AbstractToolkit -from app.service.task import Agents -from app.utils.listen.toolkit_listen import auto_listen_toolkit - - -@auto_listen_toolkit(BaseImageAnalysisToolkit) -class ImageAnalysisToolkit(BaseImageAnalysisToolkit, AbstractToolkit): - agent_name: str = Agents.multi_modal_agent - - def __init__( - self, - api_task_id: str, - model: BaseModelBackend | None = None, - timeout: float | None = None, - ): - super().__init__(model, timeout) - self.api_task_id = api_task_id diff --git a/backend/benchmark/harbor/adapter.py b/backend/benchmark/harbor/adapter.py new file mode 100644 index 00000000..6e511bbc --- /dev/null +++ b/backend/benchmark/harbor/adapter.py @@ -0,0 +1,281 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +from __future__ import annotations + +import json +import logging +import re +import shutil +from pathlib import Path + +logger = logging.getLogger(__name__) + +ADAPTER_DIR = Path(__file__).parent +TEMPLATE_DIR = ADAPTER_DIR / "template" +EVALUATE_SCRIPT = ADAPTER_DIR / "evaluate.py" +BENCHMARK_DIR = ADAPTER_DIR.parent # backend/benchmark/ + + +class EigentBenchAdapter: + """Converts eigent benchmark dataset JSONs into Harbor task directories.""" + + NAME = "eigent-bench" + + def __init__( + self, + task_dir: Path, + benchmark_dir: Path | None = None, + checker_weight: float = 0.5, + grader_weight: float = 0.5, + ) -> None: + self.task_dir = Path(task_dir) + self.benchmark_dir = ( + Path(benchmark_dir) if benchmark_dir else BENCHMARK_DIR + ) + self.checker_weight = checker_weight + self.grader_weight = grader_weight + self.configs = self._load_configs() + logger.info( + "EigentBenchAdapter initialized: %d tasks, weights=(%s/%s)", + len(self.configs), + self.checker_weight, + self.grader_weight, + ) + + def _load_configs(self) -> list[dict]: + """Load all dataset JSON configs.""" + dataset_dir = self.benchmark_dir / "dataset" + configs = [] + for json_file in sorted(dataset_dir.glob("*.json")): + configs.append(json.loads(json_file.read_text())) + if not configs: + raise ValueError(f"No benchmark configs found in {dataset_dir}") + return configs + + def generate_task( + self, index: int, local_task_id: str | None = None + ) -> None: + """Generate a single Harbor task from a benchmark config.""" + if index < 0 or index >= len(self.configs): + raise IndexError( + f"Index {index} out of range (0..{len(self.configs) - 1})" + ) + + config = self.configs[index] + if local_task_id is None: + local_task_id = f"eigent-bench-{index:04d}" + + self._prepare_task(config, local_task_id, index) + + def generate_all_tasks(self, limit: int | None = None) -> None: + """Generate all tasks.""" + total = ( + len(self.configs) + if limit is None + else min(limit, len(self.configs)) + ) + success_count = 0 + fail_count = 0 + for index in range(total): + try: + self.generate_task(index) + success_count += 1 + except Exception: + fail_count += 1 + logger.exception("Failed to generate task %d", index) + logger.info( + "Generation complete: %d succeeded, %d failed out of %d", + success_count, + fail_count, + total, + ) + + def _prepare_task( + self, config: dict, local_task_id: str, index: int + ) -> None: + """Generate the complete Harbor task directory.""" + output_dir = self.task_dir / local_task_id + + # 1. Copy template + if output_dir.exists(): + shutil.rmtree(output_dir) + shutil.copytree(TEMPLATE_DIR, output_dir) + + # 2. Fill in task.toml + metadata = config.get("metadata", {}) + self._write_task_toml(output_dir, metadata) + + # 3. Fill in instruction.md + data = config["data"] + self._write_instruction(output_dir, data) + + # 4. Copy checker/grader scripts into tests/ + tests_dir = output_dir / "tests" + tests_dir.mkdir(parents=True, exist_ok=True) + tests_config = config.get("tests", {}) + + checker_paths = [] + for checker_rel in tests_config.get("checker", []): + src = ( + self.benchmark_dir.parent / checker_rel + ) # relative to backend/ + dest = tests_dir / f"checker_{src.stem}.py" + shutil.copy2(src, dest) + checker_paths.append(f"/tests/{dest.name}") + + grader_paths = [] + for grader_rel in tests_config.get("grader", []): + src = self.benchmark_dir.parent / grader_rel + dest = tests_dir / f"grader_{src.stem}.py" + self._copy_and_patch_grader(src, dest) + grader_paths.append(f"/tests/{dest.name}") + + # 5. Copy answer files into tests/answer/{name}/ for graders that need them + task_name = data.get("name", str(index)) + answer_src = self.benchmark_dir / "answer" / task_name + if answer_src.exists(): + answer_dest = tests_dir / "answer" / task_name + if answer_dest.exists(): + shutil.rmtree(answer_dest) + shutil.copytree(answer_src, answer_dest) + + # 6. Write config.json for evaluate.py + task_config = { + "task_id": task_name, + "question": data["question"], + "checkers": checker_paths, + "graders": grader_paths, + "checker_weight": self.checker_weight, + "grader_weight": self.grader_weight, + "difficulty": metadata.get("difficulty", ""), + "tags": metadata.get("tags", []), + } + (tests_dir / "config.json").write_text( + json.dumps(task_config, indent=2) + "\n" + ) + + # 7. Copy evaluate.py into tests/ + shutil.copy2(EVALUATE_SCRIPT, tests_dir / "evaluate.py") + + # 8. Write solution/solve.sh from answer files + solution_dir = output_dir / "solution" + solution_dir.mkdir(parents=True, exist_ok=True) + answer_dir = self.benchmark_dir / "answer" / task_name + if answer_dir.exists(): + self._write_solution(solution_dir, answer_dir) + + # 9. Make test.sh executable + test_sh = tests_dir / "test.sh" + if test_sh.exists(): + test_sh.chmod(0o755) + + logger.info("Generated task: %s", local_task_id) + + def _copy_and_patch_grader(self, src: Path, dest: Path) -> None: + """Copy grader script and patch for Harbor container. + + Patches applied: + - Removes browser log checks (milestone #1) and adjusts total count, + since browser logs are eigent-specific and not available for other agents. + - ANSWER_CSV (grader 2): replaces path to resolve relative to copied script. + """ + content = src.read_text() + + # Remove BROWSER_LOG_DIR and _visited_urls function entirely + content = re.sub( + r"^BROWSER_LOG_DIR\s*=.*$", + "# BROWSER_LOG_DIR removed — browser checks disabled for Harbor", + content, + flags=re.MULTILINE, + ) + content = re.sub( + r"^def _visited_urls\(\).*?^(?=\ndef |\nclass |\n[A-Z])", + "# _visited_urls removed — browser checks disabled for Harbor\n\n", + content, + flags=re.MULTILINE | re.DOTALL, + ) + + # Comment out the browser URL check block (milestone #1) and reduce total + # Pattern: " # 1. Visited..." through the else/print block ending + content = re.sub( + r"^(\s+# 1\. Visited.*?)(?=\n\s+(?:# 2\.|[a-z_]+ = Path|if not ))", + lambda m: "\n".join( + " # " + line if line.strip() else line + for line in m.group(1).splitlines() + ), + content, + flags=re.MULTILINE | re.DOTALL, + ) + # Remove "visited = _visited_urls()" if still present + content = re.sub( + r"^\s+visited = _visited_urls\(\)\s*$", + "", + content, + flags=re.MULTILINE, + ) + + # Reduce total by 1 to account for removed browser milestone + content = re.sub( + r"^(\s+total\s*=\s*)(\d+)", + lambda m: f"{m.group(1)}{int(m.group(2)) - 1}", + content, + flags=re.MULTILINE, + ) + + # Patch ANSWER_CSV to use path relative to the script in container + content = re.sub( + r'Path\(__file__\)\.resolve\(\)\.parents\[\d+\]\s*/\s*"answer"', + 'Path(__file__).resolve().parent / "answer"', + content, + ) + + # Remove unused imports (urlparse, json for _visited_urls) + # Keep them only if still used elsewhere in the file + for module in ["from urllib.parse import urlparse"]: + if ( + module in content + and "urlparse" not in content.split(module, 1)[1] + ): + content = content.replace(module + "\n", "") + + dest.write_text(content) + + def _write_task_toml(self, output_dir: Path, metadata: dict) -> None: + task_toml = output_dir / "task.toml" + content = task_toml.read_text() + difficulty = metadata.get("difficulty", "medium") + tags_list = metadata.get("tags", []) + tags_str = ", ".join(f'"{t}"' for t in ["eigent-bench"] + tags_list) + content = content.replace("{difficulty}", difficulty) + content = content.replace("{tags}", tags_str) + task_toml.write_text(content) + + def _write_instruction(self, output_dir: Path, data: dict) -> None: + instruction = output_dir / "instruction.md" + content = instruction.read_text() + content = content.replace("{question}", data["question"]) + instruction.write_text(content) + + def _write_solution(self, solution_dir: Path, answer_dir: Path) -> None: + """Create solve.sh that copies answer files to workspace.""" + lines = ["#!/bin/bash", ""] + for f in sorted(answer_dir.iterdir()): + if f.is_file(): + shutil.copy2(f, solution_dir / f.name) + lines.append(f"cp /solution/{f.name} /workspace/{f.name}") + lines.append("") + solve_sh = solution_dir / "solve.sh" + solve_sh.write_text("\n".join(lines)) + solve_sh.chmod(0o755) diff --git a/backend/benchmark/harbor/evaluate.py b/backend/benchmark/harbor/evaluate.py new file mode 100644 index 00000000..bb75bf0b --- /dev/null +++ b/backend/benchmark/harbor/evaluate.py @@ -0,0 +1,131 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +import argparse +import importlib.util +import json +from pathlib import Path + + +def load_module(path: str, module_name: str): + """Dynamically load a Python module from a file path.""" + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run_checker(checker_path: str, working_dir: str) -> bool: + """Run checker script and return pass/fail.""" + module = load_module(checker_path, "checker") + return module.check(working_dir) + + +def run_grader(grader_path: str, working_dir: str) -> tuple[int, int]: + """Run grader script and return (completed, total).""" + module = load_module(grader_path, "grader") + return module.grade(working_dir) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, help="Path to config.json") + parser.add_argument( + "--working-dir", required=True, help="Agent working directory" + ) + parser.add_argument( + "--output-dir", default="/logs/verifier", help="Output directory" + ) + args = parser.parse_args() + + config = json.loads(Path(args.config).read_text()) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + checker_weight = config.get("checker_weight", 0.5) + grader_weight = config.get("grader_weight", 0.5) + + # Run checker(s) + checker_count = len(config.get("checkers", [])) + checker_passes = 0 + checker_results = [] + for checker_path in config.get("checkers", []): + try: + passed = run_checker(checker_path, args.working_dir) + checker_results.append({"script": checker_path, "passed": passed}) + if passed: + checker_passes += 1 + except Exception as e: + checker_results.append( + {"script": checker_path, "passed": False, "error": str(e)} + ) + checker_score = ( + checker_passes / checker_count if checker_count > 0 else 0.0 + ) + + # Run grader(s) + grader_completed = 0 + grader_total = 0 + grader_results = [] + for grader_path in config.get("graders", []): + try: + completed, total = run_grader(grader_path, args.working_dir) + grader_results.append( + { + "script": grader_path, + "completed": completed, + "total": total, + } + ) + grader_completed += completed + grader_total += total + except Exception as e: + grader_results.append( + { + "script": grader_path, + "completed": 0, + "total": 0, + "error": str(e), + } + ) + grader_score = grader_completed / grader_total if grader_total > 0 else 0.0 + + # Compute weighted reward + reward = checker_weight * checker_score + grader_weight * grader_score + + # Write Harbor output + (output_dir / "reward.txt").write_text(f"{reward:.4f}\n") + (output_dir / "metrics.json").write_text( + json.dumps( + { + "reward": reward, + "checker_score": checker_score, + "grader_score": grader_score, + "checker_weight": checker_weight, + "grader_weight": grader_weight, + "checker_results": checker_results, + "grader_results": grader_results, + }, + indent=2, + ) + + "\n" + ) + + print( + f"Reward: {reward:.4f} (checker={checker_score:.2f}*{checker_weight} + grader={grader_score:.2f}*{grader_weight})" + ) + + +if __name__ == "__main__": + main() diff --git a/backend/benchmark/harbor/run_adapter.py b/backend/benchmark/harbor/run_adapter.py new file mode 100644 index 00000000..8a786e21 --- /dev/null +++ b/backend/benchmark/harbor/run_adapter.py @@ -0,0 +1,78 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +from adapter import EigentBenchAdapter + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + +HARBOR_DATASETS_DIR = Path(__file__).resolve().parent / "datasets" + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Convert eigent benchmark tasks into Harbor task directories." + ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help=f"Output directory (default: {HARBOR_DATASETS_DIR / 'eigent-bench'})", + ) + parser.add_argument( + "--benchmark-dir", + type=Path, + default=None, + help="Path to eigent benchmark/ directory (default: auto-detect)", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Maximum number of tasks to generate.", + ) + parser.add_argument( + "--checker-weight", + type=float, + default=0.5, + help="Weight for checker score in reward (default: 0.5)", + ) + parser.add_argument( + "--grader-weight", + type=float, + default=0.5, + help="Weight for grader score in reward (default: 0.5)", + ) + args = parser.parse_args() + + output_dir = args.output_dir or (HARBOR_DATASETS_DIR / "eigent-bench") + + adapter = EigentBenchAdapter( + task_dir=output_dir, + benchmark_dir=args.benchmark_dir, + checker_weight=args.checker_weight, + grader_weight=args.grader_weight, + ) + adapter.generate_all_tasks(limit=args.limit) + logger.info("Done. Tasks written to %s", output_dir) + + +if __name__ == "__main__": + main() diff --git a/backend/benchmark/harbor/template/environment/Dockerfile b/backend/benchmark/harbor/template/environment/Dockerfile new file mode 100644 index 00000000..bbab9259 --- /dev/null +++ b/backend/benchmark/harbor/template/environment/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-lc"] + +# System dependencies +RUN apt-get update && apt-get install -y \ + sudo git curl \ + && rm -rf /var/lib/apt/lists/* + +# Set up directories +RUN mkdir -p /tests /logs/verifier /solution /workspace + +WORKDIR /workspace diff --git a/backend/benchmark/harbor/template/instruction.md b/backend/benchmark/harbor/template/instruction.md new file mode 100644 index 00000000..253e9650 --- /dev/null +++ b/backend/benchmark/harbor/template/instruction.md @@ -0,0 +1 @@ +{question} diff --git a/backend/benchmark/harbor/template/task.toml b/backend/benchmark/harbor/template/task.toml new file mode 100644 index 00000000..d1f9647e --- /dev/null +++ b/backend/benchmark/harbor/template/task.toml @@ -0,0 +1,20 @@ +version = "1.0" + +[metadata] +author_name = "eigent-ai" +difficulty = "{difficulty}" +category = "agent_benchmark" +tags = [{tags}] +source = "eigent-bench" + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 2 +memory_mb = 4096 +storage_mb = 5120 diff --git a/backend/benchmark/harbor/template/tests/test.sh b/backend/benchmark/harbor/template/tests/test.sh new file mode 100755 index 00000000..7456f564 --- /dev/null +++ b/backend/benchmark/harbor/template/tests/test.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +echo "=========================================" +echo "Running Eigent Benchmark Evaluation" +echo "=========================================" + +WORKING_DIR="${WORKING_DIR:-/workspace}" + +set +e +python3 /tests/evaluate.py \ + --config /tests/config.json \ + --working-dir "$WORKING_DIR" \ + --output-dir /logs/verifier +eval_exit=$? +set -e + +echo "=========================================" +echo "Evaluation completed with exit code: $eval_exit" +echo "=========================================" + +if [ $eval_exit -ne 0 ] && [ ! -f /logs/verifier/reward.txt ]; then + echo "0.0" > /logs/verifier/reward.txt +fi + +exit 0 diff --git a/backend/tests/app/agent/factory/test_multi_modal.py b/backend/tests/app/agent/factory/test_multi_modal.py index 8e3d145c..df070191 100644 --- a/backend/tests/app/agent/factory/test_multi_modal.py +++ b/backend/tests/app/agent/factory/test_multi_modal.py @@ -18,6 +18,7 @@ import pytest from app.agent.factory import multi_modal_agent from app.model.chat import Chat +from app.service.task import Agents pytestmark = pytest.mark.unit @@ -41,7 +42,7 @@ def test_multi_modal_agent_creation(sample_chat_data): patch("asyncio.create_task"), patch(f"{_mod}.HumanToolkit") as mock_human_toolkit, patch(f"{_mod}.VideoDownloaderToolkit") as mock_video_toolkit, - patch(f"{_mod}.ImageAnalysisToolkit") as mock_image_toolkit, + patch(f"{_mod}.ScreenshotToolkit") as mock_screenshot_toolkit, patch(f"{_mod}.OpenAIImageToolkit") as mock_openai_image_toolkit, patch(f"{_mod}.AudioAnalysisToolkit") as mock_audio_toolkit, patch(f"{_mod}.TerminalToolkit") as mock_terminal_toolkit, @@ -52,7 +53,7 @@ def test_multi_modal_agent_creation(sample_chat_data): # Mock all toolkit instances mock_human_toolkit.get_can_use_tools.return_value = [] mock_video_toolkit.return_value.get_tools.return_value = [] - mock_image_toolkit.return_value.get_tools.return_value = [] + mock_screenshot_toolkit.return_value.get_tools.return_value = [] mock_openai_image_toolkit.return_value.get_tools.return_value = [] mock_audio_toolkit.return_value.get_tools.return_value = [] mock_terminal_toolkit.return_value.get_tools.return_value = [] @@ -66,6 +67,11 @@ def test_multi_modal_agent_creation(sample_chat_data): assert result is mock_agent mock_agent_model.assert_called_once() + mock_screenshot_toolkit.assert_called_once_with( + options.project_id, + working_directory="/tmp/test_workdir", + agent_name=Agents.multi_modal_agent, + ) # Check that it was called with multi-modal agent configuration call_args = mock_agent_model.call_args diff --git a/docs/get_started/quick_start.md b/docs/get_started/quick_start.md index 0e154089..6252c5d9 100644 --- a/docs/get_started/quick_start.md +++ b/docs/get_started/quick_start.md @@ -153,12 +153,6 @@ Click the gear icon in the top-right corner to open Settings. Here’s a brief o ![General](/docs/images/quickstart_settings_general.png) -### **Privacy** - -Your privacy is important to us! Because Eigent agents operate on your desktop, they may need permissions to take screenshots or access local files to complete tasks. You can enable and manage all these permissions here. - -![Privacy](/docs/images/quickstart_settings_privacy.png) - ### **Models**