mirror of
https://github.com/zed-industries/zed.git
synced 2026-08-21 15:04:51 +00:00
10f501d700
Summary: - Add the `zed-eval` Python CLI for Modal/Harbor/Pier benchmark orchestration, including content-addressed remote builds, run/suite management, reporting, rejudge, baseline, and cleanup workflows. - Extend `eval-cli` for remote evals with provider/model overrides and step/tool-call metrics in `result.json`. - Add install/source-run helper scripts so `zed-eval` can be installed or run from the checkout without manually setting `PYTHONPATH`. - Harden the remote harness wrappers around exit-code preservation, archive extraction, custom secret wiring, and Harbor/Pier option parity, with regression coverage. Testing: - Using the CLI for two weeks - `PYTHONPATH=crates/eval_cli python3 -m compileall -q crates/eval_cli/zed_eval` - `uv run --project crates/eval_cli/zed_eval python -m unittest discover -s crates/eval_cli/zed_eval/tests` - `bash -n crates/eval_cli/script/install-zed-eval crates/eval_cli/script/zed-eval` - `cargo check -p eval_cli` - `cargo fmt --package eval_cli -- --check` - `cargo test -p eval_cli --no-run` - `./script/clippy -p eval_cli` Release Notes: - N/A
146 lines
5 KiB
Python
146 lines
5 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import io
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from zed_eval import config
|
|
from zed_eval.cli import build_parser
|
|
from zed_eval.launch import prepare_benchmark_suite, prepare_shared_build, prepare_suite
|
|
|
|
|
|
class LaunchPlanTests(unittest.TestCase):
|
|
def parse(self, argv: list[str]) -> argparse.Namespace:
|
|
return build_parser().parse_args(argv)
|
|
|
|
def test_swe_atlas_defaults_judges_by_part(self) -> None:
|
|
args = self.parse(
|
|
[
|
|
"swe-atlas",
|
|
"--parts",
|
|
"qna,rf",
|
|
"--build",
|
|
"bld-test",
|
|
"--plan",
|
|
"--allow-untracked",
|
|
"--n-tasks",
|
|
"1",
|
|
"--yes",
|
|
]
|
|
)
|
|
|
|
prepared = prepare_suite(args)
|
|
|
|
self.assertEqual([part for part, *_ in prepared], ["qna", "rf"])
|
|
self.assertEqual(prepared[0][1]["build_id"], "bld-test")
|
|
self.assertEqual(prepared[0][2]["build_id"], "bld-test")
|
|
self.assertEqual(prepared[1][1]["build_id"], "bld-test")
|
|
self.assertIsNone(prepared[1][2])
|
|
self.assertEqual(prepared[0][1]["judge_preset"], "deepseek-v4-pro")
|
|
self.assertEqual(prepared[1][1]["judge_preset"], "kimi-k2.7-code")
|
|
self.assertEqual(prepared[0][1]["agent_model"], config.DEFAULT_MODEL)
|
|
|
|
def test_baseten_model_preset_generates_provider_json(self) -> None:
|
|
args = self.parse(
|
|
[
|
|
"swe-atlas",
|
|
"--parts",
|
|
"tw",
|
|
"--build",
|
|
"bld-test",
|
|
"--plan",
|
|
"--allow-untracked",
|
|
"--model",
|
|
"baseten:kimi-k2.7-code",
|
|
"--yes",
|
|
]
|
|
)
|
|
|
|
prepared = prepare_suite(args)
|
|
run_request = prepared[0][1]
|
|
|
|
self.assertEqual([part for part, *_ in prepared], ["tw"])
|
|
self.assertEqual(run_request["benchmark"]["id"], "swe-atlas-tw")
|
|
self.assertEqual(
|
|
run_request["agent_model"], "baseten/moonshotai/Kimi-K2.7-Code"
|
|
)
|
|
self.assertIn(
|
|
"moonshotai/Kimi-K2.7-Code", run_request["openai_compatible_provider_json"]
|
|
)
|
|
self.assertIn(
|
|
"https://inference.baseten.co/v1",
|
|
run_request["openai_compatible_provider_json"],
|
|
)
|
|
|
|
def test_interactive_suite_can_include_non_swe_atlas_benchmarks(self) -> None:
|
|
args = self.parse(
|
|
[
|
|
"swe-atlas",
|
|
"--build",
|
|
"bld-test",
|
|
"--plan",
|
|
"--allow-untracked",
|
|
"--yes",
|
|
]
|
|
)
|
|
|
|
prepared = prepare_benchmark_suite(
|
|
args, ["qna", "terminal-bench-2.1", "deepswe"]
|
|
)
|
|
|
|
self.assertEqual(
|
|
[
|
|
run_request["benchmark"]["id"]
|
|
for _label, run_request, _build in prepared
|
|
],
|
|
["swe-atlas-qna", "terminal-bench-2.1", "deepswe"],
|
|
)
|
|
self.assertEqual(
|
|
[label for label, *_ in prepared], ["qna", "terminal-bench-2.1", "deepswe"]
|
|
)
|
|
self.assertEqual(prepared[0][1]["judge_preset"], "deepseek-v4-pro")
|
|
self.assertIsNone(prepared[1][1]["judge_preset"])
|
|
self.assertIsNone(prepared[2][1]["judge_preset"])
|
|
self.assertEqual(prepared[0][1]["build_id"], "bld-test")
|
|
self.assertEqual(prepared[1][1]["build_id"], "bld-test")
|
|
self.assertEqual(prepared[2][1]["build_id"], "bld-test")
|
|
self.assertEqual(prepared[0][2]["build_id"], "bld-test")
|
|
self.assertIsNone(prepared[1][2])
|
|
self.assertIsNone(prepared[2][2])
|
|
|
|
def test_explicit_existing_build_skips_source_preparation(self) -> None:
|
|
args = self.parse(["run", "rf", "--build", "custom-build"])
|
|
|
|
with (
|
|
contextlib.redirect_stdout(io.StringIO()),
|
|
patch("zed_eval.launch.build_ready_on_volume", return_value=True),
|
|
patch("zed_eval.launch.resolve_source") as resolve_source,
|
|
):
|
|
build_id, build_request = prepare_shared_build(args)
|
|
|
|
self.assertEqual(build_id, "custom-build")
|
|
self.assertIsNone(build_request)
|
|
resolve_source.assert_not_called()
|
|
|
|
def test_explicit_missing_build_uses_from_source(self) -> None:
|
|
args = self.parse(
|
|
["run", "rf", "--build", "custom-build", "--from", "v0.210.0"]
|
|
)
|
|
|
|
with (
|
|
patch("zed_eval.launch.build_ready_on_volume", return_value=False),
|
|
patch("zed_eval.source.resolve_remote_ref", return_value="a" * 40),
|
|
):
|
|
build_id, build_request = prepare_shared_build(args)
|
|
|
|
self.assertEqual(build_id, "custom-build")
|
|
self.assertEqual(build_request["build_id"], "custom-build")
|
|
self.assertEqual(build_request["source"]["base_sha"], "a" * 40)
|
|
self.assertEqual(build_request["source"]["base_ref"], "v0.210.0")
|
|
self.assertTrue(build_request["source"]["clean_source"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|