mirror of
https://github.com/zed-industries/zed.git
synced 2026-08-17 04:54:29 +00:00
10f501d700
Summary: - Add the `zed-eval` Python CLI for Modal/Harbor/Pier benchmark orchestration, including content-addressed remote builds, run/suite management, reporting, rejudge, baseline, and cleanup workflows. - Extend `eval-cli` for remote evals with provider/model overrides and step/tool-call metrics in `result.json`. - Add install/source-run helper scripts so `zed-eval` can be installed or run from the checkout without manually setting `PYTHONPATH`. - Harden the remote harness wrappers around exit-code preservation, archive extraction, custom secret wiring, and Harbor/Pier option parity, with regression coverage. Testing: - Using the CLI for two weeks - `PYTHONPATH=crates/eval_cli python3 -m compileall -q crates/eval_cli/zed_eval` - `uv run --project crates/eval_cli/zed_eval python -m unittest discover -s crates/eval_cli/zed_eval/tests` - `bash -n crates/eval_cli/script/install-zed-eval crates/eval_cli/script/zed-eval` - `cargo check -p eval_cli` - `cargo fmt --package eval_cli -- --check` - `cargo test -p eval_cli --no-run` - `./script/clippy -p eval_cli` Release Notes: - N/A
59 lines
2.2 KiB
Python
59 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
from harbor.models.verifier.result import VerifierResult
|
|
from harbor.utils.env import resolve_env_vars
|
|
from harbor.verifier.verifier import Verifier
|
|
|
|
from .judge_proxy import JUDGE_PROXY_ENSURE_SCRIPT, JUDGE_PROXY_SCRIPT
|
|
|
|
|
|
class ZedJudgeProxyVerifier(Verifier):
|
|
async def verify(self) -> VerifierResult:
|
|
await self._ensure_judge_proxy()
|
|
return await super().verify()
|
|
|
|
def _merged_verifier_env(self) -> dict[str, str]:
|
|
return {
|
|
**self.task.config.verifier.env,
|
|
**(self.verifier_env or {}),
|
|
**self.override_env,
|
|
}
|
|
|
|
async def _ensure_judge_proxy(self) -> None:
|
|
merged_env = self._merged_verifier_env()
|
|
if not merged_env.get("ZED_JUDGE_UPSTREAM"):
|
|
return
|
|
|
|
env = resolve_env_vars(merged_env)
|
|
result = await self.environment.exec(
|
|
command=(
|
|
"set -e\n"
|
|
"mkdir -p /usr/local/lib /usr/local/bin\n"
|
|
"cat > /usr/local/lib/zed_judge_proxy.py <<'ZED_PROXY_EOF'\n"
|
|
f"{JUDGE_PROXY_SCRIPT}"
|
|
"ZED_PROXY_EOF\n"
|
|
"cat > /usr/local/bin/ensure-judge-proxy <<'ZED_ENSURE_EOF'\n"
|
|
f"{JUDGE_PROXY_ENSURE_SCRIPT}"
|
|
"ZED_ENSURE_EOF\n"
|
|
"chmod +x /usr/local/bin/ensure-judge-proxy\n"
|
|
"ensure-judge-proxy\n"
|
|
"python3 - <<'ZED_PROXY_CHECK_EOF'\n"
|
|
"import os, socket\n"
|
|
"port = int(os.environ.get('ZED_JUDGE_PROXY_PORT', '8089'))\n"
|
|
"with socket.create_connection(('127.0.0.1', port), timeout=5):\n"
|
|
" pass\n"
|
|
"ZED_PROXY_CHECK_EOF\n"
|
|
"if [ -f /tmp/zed-judge-proxy.log ]; then "
|
|
"tail -2 /tmp/zed-judge-proxy.log; fi\n"
|
|
),
|
|
user="root",
|
|
env=env,
|
|
)
|
|
if result.return_code != 0:
|
|
output = "\n".join(
|
|
part for part in (result.stdout, result.stderr) if part
|
|
).strip()
|
|
detail = f": {output}" if output else ""
|
|
raise RuntimeError(
|
|
f"Failed to install or start judge proxy (exit {result.return_code}){detail}"
|
|
)
|