mirror of
https://github.com/zed-industries/zed.git
synced 2026-08-29 13:12:39 +00:00
10f501d700
Summary: - Add the `zed-eval` Python CLI for Modal/Harbor/Pier benchmark orchestration, including content-addressed remote builds, run/suite management, reporting, rejudge, baseline, and cleanup workflows. - Extend `eval-cli` for remote evals with provider/model overrides and step/tool-call metrics in `result.json`. - Add install/source-run helper scripts so `zed-eval` can be installed or run from the checkout without manually setting `PYTHONPATH`. - Harden the remote harness wrappers around exit-code preservation, archive extraction, custom secret wiring, and Harbor/Pier option parity, with regression coverage. Testing: - Using the CLI for two weeks - `PYTHONPATH=crates/eval_cli python3 -m compileall -q crates/eval_cli/zed_eval` - `uv run --project crates/eval_cli/zed_eval python -m unittest discover -s crates/eval_cli/zed_eval/tests` - `bash -n crates/eval_cli/script/install-zed-eval crates/eval_cli/script/zed-eval` - `cargo check -p eval_cli` - `cargo fmt --package eval_cli -- --check` - `cargo test -p eval_cli --no-run` - `./script/clippy -p eval_cli` Release Notes: - N/A
79 lines
2.5 KiB
Python
79 lines
2.5 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from zed_eval import cleanup
|
|
|
|
|
|
def age_path(path: Path, days: float) -> None:
|
|
past = time.time() - days * 86400.0
|
|
os.utime(path, (past, past))
|
|
|
|
|
|
class PruneArtifactsTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self._tmp.name)
|
|
self.addCleanup(self._tmp.cleanup)
|
|
|
|
def _make_build(self, build_id: str, age_days: float) -> Path:
|
|
build_dir = self.root / "builds" / build_id
|
|
build_dir.mkdir(parents=True)
|
|
(build_dir / "eval-cli").write_text("binary")
|
|
ready = build_dir / "READY"
|
|
ready.write_text("done")
|
|
age_path(ready, age_days)
|
|
return build_dir
|
|
|
|
def test_eval_results_are_never_touched(self) -> None:
|
|
runs = self.root / "runs" / "anant" / "swe-atlas-rf" / "run-1"
|
|
runs.mkdir(parents=True)
|
|
result_file = runs / "summary.json"
|
|
result_file.write_text("{}")
|
|
age_path(result_file, 999)
|
|
|
|
cleanup.prune_artifacts(self.root, dry_run=False, build_retention_days=14)
|
|
|
|
self.assertTrue(result_file.exists())
|
|
|
|
def test_removes_old_builds_keeps_recent(self) -> None:
|
|
self._make_build("bld-old", age_days=30)
|
|
self._make_build("bld-new", age_days=1)
|
|
|
|
result = cleanup.prune_artifacts(
|
|
self.root, dry_run=False, build_retention_days=14
|
|
)
|
|
|
|
self.assertFalse((self.root / "builds" / "bld-old").exists())
|
|
self.assertTrue((self.root / "builds" / "bld-new").exists())
|
|
self.assertEqual(result["counts"]["builds"], 1)
|
|
|
|
def test_dry_run_deletes_nothing(self) -> None:
|
|
self._make_build("bld-old", age_days=30)
|
|
result = cleanup.prune_artifacts(
|
|
self.root, dry_run=True, build_retention_days=14
|
|
)
|
|
self.assertTrue((self.root / "builds" / "bld-old").exists())
|
|
self.assertEqual(result["counts"]["builds"], 1)
|
|
|
|
def test_stale_lock_removed(self) -> None:
|
|
locks = self.root / "build-locks"
|
|
locks.mkdir(parents=True)
|
|
stale = locks / "bld-x.json"
|
|
stale.write_text("{}")
|
|
age_path(stale, 1) # 1 day > 6h ttl
|
|
fresh = locks / "bld-y.json"
|
|
fresh.write_text("{}")
|
|
|
|
cleanup.prune_artifacts(self.root, dry_run=False, lock_ttl_hours=6)
|
|
|
|
self.assertFalse(stale.exists())
|
|
self.assertTrue(fresh.exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|