mirror of
https://github.com/zed-industries/zed.git
synced 2026-08-21 15:04:51 +00:00
10f501d700
Summary: - Add the `zed-eval` Python CLI for Modal/Harbor/Pier benchmark orchestration, including content-addressed remote builds, run/suite management, reporting, rejudge, baseline, and cleanup workflows. - Extend `eval-cli` for remote evals with provider/model overrides and step/tool-call metrics in `result.json`. - Add install/source-run helper scripts so `zed-eval` can be installed or run from the checkout without manually setting `PYTHONPATH`. - Harden the remote harness wrappers around exit-code preservation, archive extraction, custom secret wiring, and Harbor/Pier option parity, with regression coverage. Testing: - Using the CLI for two weeks - `PYTHONPATH=crates/eval_cli python3 -m compileall -q crates/eval_cli/zed_eval` - `uv run --project crates/eval_cli/zed_eval python -m unittest discover -s crates/eval_cli/zed_eval/tests` - `bash -n crates/eval_cli/script/install-zed-eval crates/eval_cli/script/zed-eval` - `cargo check -p eval_cli` - `cargo fmt --package eval_cli -- --check` - `cargo test -p eval_cli --no-run` - `./script/clippy -p eval_cli` Release Notes: - N/A
36 lines
1.1 KiB
Python
36 lines
1.1 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from zed_eval import baseline
|
|
|
|
|
|
class BaselineRecordTests(unittest.TestCase):
|
|
def test_record_uses_base_ref_from_source_info(self) -> None:
|
|
record, has_build_info = baseline._record_from_provenance(
|
|
namespace="alice",
|
|
fallback_experiment="swe-atlas-rf",
|
|
run_id="run-1",
|
|
provenance={
|
|
"request": {
|
|
"experiment_name": "swe-atlas-rf",
|
|
"agent_model": "anthropic/claude-sonnet-4-6",
|
|
"judge_preset": "kimi-k2.7-code",
|
|
"build_id": "bld-test",
|
|
},
|
|
"summary": {"status": "completed", "trial_count": 10},
|
|
"build_info": {
|
|
"build_id": "bld-test",
|
|
"base_sha": "a" * 40,
|
|
"patch_sha256": None,
|
|
"source": {"base_ref": "v0.210.0"},
|
|
},
|
|
},
|
|
)
|
|
|
|
self.assertTrue(has_build_info)
|
|
self.assertEqual(record["base_ref"], "v0.210.0")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|