mirror of
https://github.com/zed-industries/zed.git
synced 2026-08-04 05:13:28 +00:00
10f501d700
Summary: - Add the `zed-eval` Python CLI for Modal/Harbor/Pier benchmark orchestration, including content-addressed remote builds, run/suite management, reporting, rejudge, baseline, and cleanup workflows. - Extend `eval-cli` for remote evals with provider/model overrides and step/tool-call metrics in `result.json`. - Add install/source-run helper scripts so `zed-eval` can be installed or run from the checkout without manually setting `PYTHONPATH`. - Harden the remote harness wrappers around exit-code preservation, archive extraction, custom secret wiring, and Harbor/Pier option parity, with regression coverage. Testing: - Using the CLI for two weeks - `PYTHONPATH=crates/eval_cli python3 -m compileall -q crates/eval_cli/zed_eval` - `uv run --project crates/eval_cli/zed_eval python -m unittest discover -s crates/eval_cli/zed_eval/tests` - `bash -n crates/eval_cli/script/install-zed-eval crates/eval_cli/script/zed-eval` - `cargo check -p eval_cli` - `cargo fmt --package eval_cli -- --check` - `cargo test -p eval_cli --no-run` - `./script/clippy -p eval_cli` Release Notes: - N/A
85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
from typing import Any
|
|
|
|
from . import source
|
|
|
|
|
|
def validate_build_id(build_id: str | None) -> None:
|
|
if build_id and source.sanitize_namespace(build_id) != build_id:
|
|
raise ValueError(
|
|
"build ids may only contain lowercase letters, numbers, '.', '_', and '-'"
|
|
)
|
|
|
|
|
|
def prepare_build_request(
|
|
*,
|
|
base_sha: str | None,
|
|
patch_path: str | None,
|
|
build_id: str | None,
|
|
allow_untracked: bool,
|
|
require_clean: bool,
|
|
repo_url: str | None,
|
|
clean_source: bool = False,
|
|
source_label: str | None = None,
|
|
pre_resolved_base_sha: str | None = None,
|
|
) -> dict[str, Any]:
|
|
validate_build_id(build_id)
|
|
source_info, patch = source.prepare_build_source(
|
|
base_sha=base_sha,
|
|
patch_path=patch_path,
|
|
allow_untracked=allow_untracked,
|
|
require_clean=require_clean,
|
|
repo_url=repo_url,
|
|
clean=clean_source,
|
|
source_label=source_label,
|
|
pre_resolved_base_sha=pre_resolved_base_sha,
|
|
)
|
|
resolved_build_id = build_id or source.compute_build_id(source_info=source_info)
|
|
return {
|
|
"build_id": resolved_build_id,
|
|
"base_sha": source_info["base_sha"],
|
|
"patch": patch,
|
|
"patch_sha256": source_info.get("patch_sha256"),
|
|
"source": source_info,
|
|
"toolchain": source.build_toolchain_info(),
|
|
}
|
|
|
|
|
|
def resolve_source(
|
|
args: argparse.Namespace,
|
|
) -> tuple[str | None, bool, str | None, str | None]:
|
|
"""Unify the source selectors into
|
|
`(base_sha, clean_source, source_label, pre_resolved_base_sha)`.
|
|
|
|
Precedence: `--from` wins over `--zed-version` / `--base-sha`.
|
|
|
|
--from local -> current HEAD + tracked patch (dev iteration)
|
|
--from <ref/tag/sha> -> clean build of that ref, resolved canonically
|
|
against the remote so everyone shares one build
|
|
"""
|
|
repo_url = getattr(args, "repo_url", None) or source.DEFAULT_REPO_URL
|
|
from_source = getattr(args, "from_source", None)
|
|
if from_source:
|
|
if from_source.strip().lower() == "local":
|
|
return (
|
|
getattr(args, "base_sha", None),
|
|
bool(getattr(args, "clean_source", False)),
|
|
None,
|
|
None,
|
|
)
|
|
resolved = source.resolve_remote_ref(repo_url, from_source)
|
|
return None, True, from_source, resolved
|
|
|
|
zed_version = getattr(args, "zed_version", None)
|
|
if zed_version:
|
|
resolved = source.resolve_remote_ref(repo_url, zed_version)
|
|
return None, True, zed_version, resolved
|
|
|
|
return (
|
|
getattr(args, "base_sha", None),
|
|
bool(getattr(args, "clean_source", False)),
|
|
None,
|
|
None,
|
|
)
|