mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-03 04:07:04 +00:00
Merge pull request #698 from razzant/fixfwd/sk1-product-gate
e2e_live: SK1 by the product gate, absorb wait proves no pending cycle, absorb-scoped reservation Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
This commit is contained in:
Ouroboros
commit
b6ac8c13c9
9 changed files with 357 additions and 91 deletions
20
.github/workflows/ci.yml
vendored
20
.github/workflows/ci.yml
vendored
|
|
@ -594,24 +594,24 @@ jobs:
|
|||
#
|
||||
# Run size — the stand's own admission arithmetic, computed from the code at
|
||||
# this base (devtools/e2e_live/run_live_lanes.py `RunBudget.reservation`):
|
||||
# reservation(attempt) = max(0.01, per_task_usd x (root_tasks + 1 if --self-mod else root_tasks))
|
||||
# reservation(attempt) = max(0.01, per_task_usd x (root_tasks + 1 if --self-mod and the scenario absorbs else root_tasks))
|
||||
# root_tasks = 1 for SM1 and SW1, 2 for SK1 (scenarios.py); --self-mod
|
||||
# adds one root for the evolution cycle (rc.14: up to two
|
||||
# cycles per lane, all under the same lane fence)
|
||||
# adds one root for the post-task evolution cycle of the
|
||||
# scenario that absorbs (SM1; SW1/SK1 pin promotion off)
|
||||
# and an attempt is admitted only while spent + reserved(in flight) +
|
||||
# reservation <= --total-budget; one that can never fit is recorded not_run
|
||||
# and fails the verdict. Worst case (every attempt spends its whole
|
||||
# reservation) the full operator set SM1,SW1,SK1 x3 with --self-mod needs
|
||||
# per_task x (2 + 2 + 3) x 3 = 21 x per_task, and the product's default
|
||||
# per_task x (2 + 1 + 2) x 3 = 15 x per_task, and the product's default
|
||||
# per-task cap (settings_defaults OUROBOROS_PER_TASK_COST_USD = 50) makes a
|
||||
# SINGLE SM1 attempt reserve 50 x (1 + 1) = $100 > $30: nothing would be
|
||||
# admissible. So the fence is --per-task-usd 15 = 30 / 2, the largest value
|
||||
# under which one single-root self-mod attempt fits: SM1 x1 reserves exactly
|
||||
# 15 x (1 + 1) = $30 = the cap (the full set: 21 x 15 = $315). A second
|
||||
# attempt of any scenario (SW1 $30, SK1 15 x (2 + 1) = $45) cannot fit by
|
||||
# that rule, hence ONE lane, ONE attempt, SM1 only — the largest feasible
|
||||
# subset, named in the job title. Only if the self-mod root ever left the
|
||||
# rule would SM1 + SW1 (15 + 15 = 30) become feasible together: revisit
|
||||
# 15 x (1 + 1) = $30 = the cap (the full set: 15 x 15 = $225). Any second
|
||||
# attempt (SW1 $15, SK1 15 x 2 = $30) cannot fit beside it by that rule,
|
||||
# hence ONE lane, ONE attempt, SM1 only — the largest feasible subset,
|
||||
# named in the job title. Only if SM1's evolution root ever left the rule
|
||||
# would SM1 + SW1 (15 + 15 = 30) become feasible together: revisit
|
||||
# --scenarios then, keep the arithmetic here (the feasibility test and the
|
||||
# summary-header pin re-derive it from the code and trip on a rule change).
|
||||
# --min-credit-usd defaults to --total-budget: the key needs >= $30 headroom
|
||||
|
|
@ -697,7 +697,7 @@ jobs:
|
|||
manifest = pathlib.Path(os.environ["RUNNER_TEMP"]) / "e2e_live" / "run_manifest.json"
|
||||
lines = ["## e2e-live: SM1 x1 — the largest subset feasible under the $30 cap "
|
||||
"(per-task $15 x (roots + the self-mod evolution root): SM1 x1 = $30; "
|
||||
"the full SM1/SW1/SK1 x3 set needs $315)"]
|
||||
"the full SM1/SW1/SK1 x3 set needs $225)"]
|
||||
try:
|
||||
if not manifest.exists():
|
||||
lines.append("no run_manifest.json: the stand never reached admission (see the step log)")
|
||||
|
|
|
|||
|
|
@ -310,6 +310,46 @@ def seed_owner_state(data_root: pathlib.Path, *, evolution_enabled: bool = False
|
|||
state_path.write_text(json.dumps(st), encoding="utf-8")
|
||||
|
||||
|
||||
def campaign_summary(data_root: pathlib.Path) -> dict:
|
||||
"""The durable campaign facts an absorb wait reasons about: evolution_campaign.json (presence, status,
|
||||
source, a pending ``active_transaction``, the newest transaction outcome, the absorbed counter) and the
|
||||
post-task promotion counter (``post_task_evolution_counter.json``: the decision ran at least once)."""
|
||||
state_dir = pathlib.Path(data_root) / "state"
|
||||
try:
|
||||
campaign = json.loads((state_dir / "evolution_campaign.json").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
campaign = {}
|
||||
campaign = campaign if isinstance(campaign, dict) else {}
|
||||
try:
|
||||
counter = int(json.loads((state_dir / "post_task_evolution_counter.json").read_text(encoding="utf-8"))["n"])
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
counter = 0
|
||||
history = [tx for tx in (campaign.get("transaction_history") or []) if isinstance(tx, dict)]
|
||||
return {"present": bool(campaign), "status": str(campaign.get("status") or ""),
|
||||
"source": str(campaign.get("source") or ""),
|
||||
"active_transaction": isinstance(campaign.get("active_transaction"), dict),
|
||||
"absorbed_cycles_done": int(campaign.get("absorbed_cycles_done") or 0),
|
||||
"history_len": len(history),
|
||||
"newest_outcome": str(history[-1].get("cycle_outcome") or "") if history else "",
|
||||
"post_task_counter": counter}
|
||||
|
||||
|
||||
def absorb_idle_reason(campaign: dict, history_len_at_start: int = 0) -> str:
|
||||
"""Typed non-confirmation of an idle lane (``IsolatedServer.wait_for_absorb``), relative to the wait's
|
||||
start so a resumed campaign's OLDER cycles never speak for this boundary: ``campaign_<status>`` (paused/
|
||||
stopped/completed wins), ``no_promotion`` (no campaign although an ``every_n`` post-task tick was
|
||||
recorded — the decision may still have declined), ``no_decision`` (no campaign, no tick recorded: ``llm``
|
||||
cadences write none), ``cycle_no_op`` / ``cycle_not_absorbed`` (a cycle newer than the wait ended without
|
||||
an absorb) or ``cycle_not_enqueued`` (a campaign that attached no new cycle)."""
|
||||
if campaign.get("status") in ("paused", "stopped", "completed"):
|
||||
return f"campaign_{campaign['status']}"
|
||||
if not campaign.get("present"):
|
||||
return "no_promotion" if campaign.get("post_task_counter") else "no_decision"
|
||||
if int(campaign.get("history_len") or 0) > int(history_len_at_start or 0):
|
||||
return "cycle_no_op" if campaign.get("newest_outcome") == "no_op" else "cycle_not_absorbed"
|
||||
return "cycle_not_enqueued"
|
||||
|
||||
|
||||
def absorbed_cycles_done(data_root: pathlib.Path) -> int:
|
||||
"""Read absorbed self-evolution cycle count from evolution_campaign.json."""
|
||||
path = pathlib.Path(data_root) / "state" / "evolution_campaign.json"
|
||||
|
|
@ -647,33 +687,44 @@ class IsolatedServer:
|
|||
return False
|
||||
|
||||
def wait_for_absorb(self, prev_sha: str, prev_absorbed: int, timeout: float = 1800,
|
||||
idle_grace: float = 90) -> dict:
|
||||
"""Between instances, wait for an absorbed self-evolution cycle: the server
|
||||
re-execs onto a new SHA and `absorbed_cycles_done` increments. Returns
|
||||
{absorbed, new_sha, cycles, reason}. When the LLM legitimately declines to
|
||||
promote (the common path), this returns absorbed=False EARLY — once the queue
|
||||
is idle, no post_task_evolution_request.json is pending, and no cycle absorbed
|
||||
within a short grace — instead of stalling the full timeout."""
|
||||
idle_grace: float = 90, idle_polls: int = 6) -> dict:
|
||||
"""Between instances, wait for an absorbed self-evolution cycle: the server re-execs onto a
|
||||
new SHA and ``absorbed_cycles_done`` increments. Returns ``{absorbed, new_sha, cycles, reason,
|
||||
campaign}``. An EARLY ``absorbed=False`` needs PROOF that no cycle is pending, held on
|
||||
``idle_polls`` consecutive polls after ``idle_grace``: the queue idle AND ``supervisor_ready``
|
||||
AND no ``post_task_evolution_request.json`` AND no campaign ``active_transaction``. One idle
|
||||
sample is not proof: a cycle that committed keeps its transaction as ``waiting_for_restart``
|
||||
while the supervisor restarts synchronously (queue empty, counter unchanged), and the re-exec'd
|
||||
server answers ``/api/state`` with zero counts before its supervisor is up — the counter moves
|
||||
only when the worker boot verifies the restart (rc.15 stand, adversarial finding of 2026-09-06).
|
||||
The typed reason is what the durable campaign state proves (``absorb_idle_reason``)."""
|
||||
deadline = time.time() + timeout
|
||||
start = time.time()
|
||||
request_path = self.data_root / "state" / "post_task_evolution_request.json"
|
||||
idle_streak, history_at_start = 0, campaign_summary(self.data_root)["history_len"]
|
||||
while time.time() < deadline:
|
||||
cycles = absorbed_cycles_done(self.data_root)
|
||||
sha = self.current_sha()
|
||||
if cycles > prev_absorbed and sha and sha != prev_sha:
|
||||
self.wait_for_health(timeout=180)
|
||||
return {"absorbed": True, "new_sha": sha, "cycles": cycles, "reason": "absorbed"}
|
||||
return {"absorbed": True, "new_sha": sha, "cycles": cycles, "reason": "absorbed",
|
||||
"campaign": campaign_summary(self.data_root)}
|
||||
if time.time() - start > idle_grace and cycles == prev_absorbed:
|
||||
campaign = campaign_summary(self.data_root)
|
||||
try:
|
||||
st = self._state(timeout=5)
|
||||
idle = int(st.get("pending_count") or 0) == 0 and int(st.get("running_count") or 0) == 0
|
||||
idle = (int(st.get("pending_count") or 0) == 0 and int(st.get("running_count") or 0) == 0
|
||||
and bool(st.get("supervisor_ready")))
|
||||
except (urllib.error.URLError, OSError, ValueError):
|
||||
idle = False
|
||||
if idle and not request_path.exists():
|
||||
return {"absorbed": False, "new_sha": sha, "cycles": cycles, "reason": "no_promotion"}
|
||||
idle = idle and not request_path.exists() and not campaign["active_transaction"]
|
||||
idle_streak = idle_streak + 1 if idle else 0
|
||||
if idle_streak >= max(1, int(idle_polls)):
|
||||
return {"absorbed": False, "new_sha": sha, "cycles": cycles,
|
||||
"reason": absorb_idle_reason(campaign, history_at_start), "campaign": campaign}
|
||||
time.sleep(5)
|
||||
return {"absorbed": False, "new_sha": self.current_sha(),
|
||||
"cycles": absorbed_cycles_done(self.data_root), "reason": "timeout"}
|
||||
return {"absorbed": False, "new_sha": self.current_sha(), "cycles": absorbed_cycles_done(self.data_root),
|
||||
"reason": "timeout", "campaign": campaign_summary(self.data_root)}
|
||||
|
||||
def stop(self) -> None:
|
||||
if self.proc is not None and self.proc.poll() is None:
|
||||
|
|
|
|||
|
|
@ -91,10 +91,9 @@ PROCFS_AVAILABLE = os.path.isdir("/proc") # the orphan scan reads /proc enviro
|
|||
# A lane's TOTAL_BUDGET must stay POSITIVE: the runtime reads a non-positive value as "no finite
|
||||
# global budget" (``settings_setup_contract.resolve_total_budget_usd``), the opposite of a cap.
|
||||
LANE_BUDGET_FLOOR_USD = 0.01
|
||||
RESERVATION_RULE = (f"max({LANE_BUDGET_FLOOR_USD:g}, per_task_usd x (root_tasks + 1 if --self-mod else root_tasks)) — the "
|
||||
"runtime fences each root task tree at OUROBOROS_PER_TASK_COST_USD and --self-mod adds one root for the "
|
||||
"one post-task evolution cycle (the lane seeds no campaign: the promotion enables a one-shot one). "
|
||||
"The lane's TOTAL_BUDGET is that reservation — the true fence — so settled spend + in-flight ceilings <= cap")
|
||||
RESERVATION_RULE = (f"max({LANE_BUDGET_FLOOR_USD:g}, per_task_usd x (root_tasks + 1 if --self-mod and the scenario absorbs else root_tasks)) "
|
||||
"— the runtime fences each root task tree at OUROBOROS_PER_TASK_COST_USD; --self-mod adds one root for the post-task "
|
||||
"cycle of a lane that promotes (SM1; SW1/SK1 pin it off). The lane's TOTAL_BUDGET is that reservation — the true fence")
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
|
|
@ -254,10 +253,10 @@ def lane_spend(data_root: pathlib.Path) -> tuple[float, int]:
|
|||
class RunBudget:
|
||||
"""The RUN-WIDE ledger behind ``--total-budget`` (the first paid run gave every lane the whole cap).
|
||||
|
||||
Reservation rule, per attempt: ``per_task_usd x (root_tasks + int(self_mod))`` — the runtime fences each ROOT
|
||||
task tree at ``OUROBOROS_PER_TASK_COST_USD`` and children spend under their root's ceiling, so SW1 (one root,
|
||||
two scouts) reserves for one root, SK1 (author + dispatch) for two, and ``--self-mod`` adds one root for the
|
||||
one post-task evolution cycle, all under the lane fence — the true fence. Admission asks, PER attempt:
|
||||
Reservation rule, per attempt: ``per_task_usd x (root_tasks + int(self_mod and absorbs))`` — the runtime fences
|
||||
each ROOT task tree at ``OUROBOROS_PER_TASK_COST_USD`` and children spend under their root's ceiling, so SW1 (one
|
||||
root, two scouts) reserves for one root, SK1 (author + dispatch) for two, and ``--self-mod`` adds one root for
|
||||
the post-task cycle of a lane that promotes (SM1 only), all under the lane fence. Admission asks, PER attempt:
|
||||
``spent + reservation > cap`` — it can NEVER fit, refused and recorded ``not_run`` (a later, smaller attempt is
|
||||
asked on its own; nothing halts the run); otherwise it reserves only when no earlier-dispatched attempt is
|
||||
still asking (FIFO by ``dispatch_index``, see ``admit``) and ``spent + reserved(in flight) + reservation <=
|
||||
|
|
@ -280,10 +279,10 @@ class RunBudget:
|
|||
self.refusals: list[dict] = []
|
||||
self.not_run: list[str] = []
|
||||
|
||||
def reservation(self, root_tasks: int) -> float:
|
||||
"""The ONE effective ceiling of an attempt (admission, the lane's TOTAL_BUDGET and the reports
|
||||
carry this same number): ``per_task_usd x (root_tasks + int(self_mod))``, floored, never rounded up."""
|
||||
return max(LANE_BUDGET_FLOOR_USD, self.per_task * (max(1, int(root_tasks or 1)) + int(self.self_mod)))
|
||||
def reservation(self, root_tasks: int, absorbs: bool = False) -> float:
|
||||
"""The ONE effective ceiling of an attempt (admission, the lane's TOTAL_BUDGET and the reports carry this
|
||||
same number): ``per_task_usd x (root_tasks + int(self_mod and absorbs))``, floored, never rounded up."""
|
||||
return max(LANE_BUDGET_FLOOR_USD, self.per_task * (max(1, int(root_tasks or 1)) + int(self.self_mod and absorbs)))
|
||||
|
||||
def _spent_locked(self) -> tuple[float, int]:
|
||||
rows = list(self._final.values()) + [self._read(root) for root, _reserved in self._live.values()]
|
||||
|
|
@ -293,7 +292,7 @@ class RunBudget:
|
|||
return sum(reserved for _root, reserved in self._live.values())
|
||||
|
||||
def admit(self, job: tuple, root_tasks: int, data_root: pathlib.Path, *, dispatch_index: int,
|
||||
on_wait: Callable[[str], None] | None = None) -> tuple[bool, dict]:
|
||||
on_wait: Callable[[str], None] | None = None, absorbs: bool = False) -> tuple[bool, dict]:
|
||||
"""FIFO by ``dispatch_index`` (``dispatch_order``'s position): an attempt reserves only when no earlier-
|
||||
dispatched attempt is still asking and the cap has room beside the reservations in flight; one that can
|
||||
never fit is refused at once and leaves the line. Without the line the freed lane's NEXT job took the lock
|
||||
|
|
@ -304,7 +303,7 @@ class RunBudget:
|
|||
and settle wake every waiter, the line predicate re-parks the later ones. Cost: a large head can idle lanes
|
||||
a smaller attempt would use. ``on_wait`` runs UNDER the budget lock (it must not touch the budget), told
|
||||
once, with the reason the wait begins with; ``facts["waited_sec"]`` is how long."""
|
||||
need, name, waited_from = self.reservation(root_tasks), f"{job[0]}_a{job[1]}", None
|
||||
need, name, waited_from = self.reservation(root_tasks, absorbs), f"{job[0]}_a{job[1]}", None
|
||||
with self._lock:
|
||||
self._pending[dispatch_index] = name
|
||||
try:
|
||||
|
|
@ -368,7 +367,7 @@ def budget_preflight(budget: RunBudget, scenario_ids: list[str], attempts: int,
|
|||
can never be admitted after any spend). No override flag: the operator changes the flags."""
|
||||
rows = []
|
||||
for sid in scenario_ids:
|
||||
need = budget.reservation(SCENARIOS[sid].root_tasks)
|
||||
need = budget.reservation(SCENARIOS[sid].root_tasks, SCENARIOS[sid].expects_absorb)
|
||||
rows.append({"scenario": sid, "root_tasks": SCENARIOS[sid].root_tasks, "reservation_usd": need,
|
||||
"attempts": int(attempts), "worst_case_usd": round(need * attempts, 4),
|
||||
"unreachable": need > budget.cap or (attempts >= 2 and need >= budget.cap)})
|
||||
|
|
@ -383,7 +382,7 @@ def dispatch_order(budget: RunBudget, requested: list[tuple[str, int]]) -> list[
|
|||
"""Round-robin by attempt (a1 of every scenario, then a2, ...), largest reservation first within a round
|
||||
(stable among equals); admission keeps this order (``RunBudget.admit``). The verdict is pass-of PER
|
||||
scenario, so the MINIMUM admitted attempts per scenario is what the order protects, not the sum."""
|
||||
return sorted(requested, key=lambda job: (job[1], -budget.reservation(SCENARIOS[job[0]].root_tasks)))
|
||||
return sorted(requested, key=lambda job: (job[1], -budget.reservation(SCENARIOS[job[0]].root_tasks, SCENARIOS[job[0]].expects_absorb)))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -517,7 +516,7 @@ def _newest_transaction(data_root: pathlib.Path) -> dict:
|
|||
def confirm_absorb(server: IsolatedServer, clone: pathlib.Path, data_root: pathlib.Path, pre: dict, *,
|
||||
timeout: float, ready_timeout: float) -> dict:
|
||||
"""POSITIVE evidence of an absorbed post-task evolution, or a typed non-confirmation.
|
||||
``wait_for_absorb`` answers ``absorbed=False`` on ``no_promotion``/``timeout``, and a runner that
|
||||
``wait_for_absorb`` answers ``absorbed=False`` typed (``absorb_idle_reason`` or ``timeout``), and a runner that
|
||||
only checks liveness afterwards lets ``--self-mod`` PASS with no restart at all. Confirmed means
|
||||
ALL of: the campaign's absorbed-cycle counter advanced past the pre-task snapshot, the served sha
|
||||
moved off the snapshot (``wait_for_absorb``'s own condition), the server re-exec'd (``/api/state``
|
||||
|
|
@ -642,7 +641,8 @@ def run_attempt(job: tuple[str, int], args: argparse.Namespace, out: pathlib.Pat
|
|||
_log(f"{sid}_a{attempt}: {msg}")
|
||||
states[job] = ("waiting (budget)", time.time())
|
||||
|
||||
admitted, facts = budget.admit(job, SCENARIOS[sid].root_tasks, lane / "data", dispatch_index=dispatch_index, on_wait=waiting)
|
||||
admitted, facts = budget.admit(job, SCENARIOS[sid].root_tasks, lane / "data", dispatch_index=dispatch_index,
|
||||
on_wait=waiting, absorbs=SCENARIOS[sid].expects_absorb)
|
||||
if not admitted:
|
||||
row = {**_lane_row(job, args), "status": "not_run", "reason_code": "budget_cap", "budget": facts,
|
||||
"refusal": {"type": "RunBudgetCap", "code": "budget_cap", "message": "run-wide budget cap reached"},
|
||||
|
|
@ -701,7 +701,7 @@ def run_lane(job: tuple[str, int], args: argparse.Namespace, out: pathlib.Path,
|
|||
cfg["OUROBOROS_REVIEW_ENFORCEMENT"] = "advisory"
|
||||
# The lane's ceiling is its own reservation: disjoint from the other lanes', never the whole cap.
|
||||
cfg["TOTAL_BUDGET"] = budget.ceiling(job)
|
||||
row["budget"] = {"reservation_usd": budget.reservation(scenario.root_tasks),
|
||||
row["budget"] = {"reservation_usd": budget.reservation(scenario.root_tasks, scenario.expects_absorb),
|
||||
"lane_total_budget_usd": cfg["TOTAL_BUDGET"], "per_task_usd": float(args.per_task_usd)}
|
||||
sha = write_settings(settings_path, cfg)
|
||||
# Owner id only, never a campaign: the task's post-task promotion enables the one-shot one (rc.15 run2 pin).
|
||||
|
|
|
|||
|
|
@ -765,11 +765,34 @@ def sk1_prompt() -> str:
|
|||
|
||||
|
||||
def _skill_entry(base_url: str, name: str) -> dict:
|
||||
listing = _api(base_url, "GET", "/api/extensions", timeout=30)
|
||||
"""The skill's ``/api/extensions`` row, or ``{}`` — a listing that fails is a failed check with facts
|
||||
(``review_executable``/``live_loaded`` absent), never an infra_error that aborts the lifecycle."""
|
||||
resp = _api_status(base_url, "GET", "/api/extensions", None, timeout=30)
|
||||
listing = resp["body"] if resp["status"] == 200 else {}
|
||||
rows = listing if isinstance(listing, list) else (listing.get("extensions") or listing.get("skills") or [])
|
||||
return next((row for row in rows if isinstance(row, dict) and row.get("name") == name), {})
|
||||
|
||||
|
||||
def sk1_review_gate(review: dict, entry: dict, findings: list) -> tuple[bool, dict]:
|
||||
"""The SK1 review criterion is the PRODUCT gate (owner decision 2026-09-06): the review ran (HTTP 200 with
|
||||
recorded findings) and the ``/api/extensions`` row says ``executable_review`` — clean, warnings, or blockers
|
||||
under advisory enforcement by operator choice (``skill_review_gate``). A clean all-PASS review is a recorded
|
||||
FACT, not the verdict: the rc.15 SK1 rerun on 560f7d71 authored one clean, one warnings and one blockers
|
||||
payload with every other lifecycle check green, so all-PASS measured the author model, not the product.
|
||||
The verdict also needs the review call itself to answer 200 with its own ``executable_review`` (the
|
||||
lifecycle's gate; a ``pending`` duplicate job never passes) and persisted findings (a review really ran).
|
||||
The SK1 lane sets no enforcement, so it runs the tree default (advisory today) under both profiles: the
|
||||
``blockers under blocking enforcement`` branch is the product's rule, not a path the stand exercises."""
|
||||
gate = entry.get("review_gate") if isinstance(entry.get("review_gate"), dict) else {}
|
||||
failed = [f.get("item") for f in findings if str(f.get("verdict") or "") != "PASS"]
|
||||
ok = (review["status"] == 200 and review["body"].get("executable_review") is True
|
||||
and entry.get("executable_review") is True and bool(findings))
|
||||
return ok, {"review_status": review["body"].get("status"), "review_executable": entry.get("executable_review"),
|
||||
"review_body_executable": review["body"].get("executable_review"),
|
||||
"review_enforcement": gate.get("review_enforcement"), "review_blocking_reason": gate.get("blocking_reason"),
|
||||
"findings": len(findings), "findings_failed": failed, "review_clean": bool(findings) and not failed}
|
||||
|
||||
|
||||
def run_sk1(ctx: LaneContext) -> None:
|
||||
from ouroboros.extension_surface_names import extension_surface_name
|
||||
|
||||
|
|
@ -785,11 +808,8 @@ def run_sk1(ctx: LaneContext) -> None:
|
|||
review = _api_status(ctx.server.base_url, "POST", f"/api/skills/{SK1_SKILL}/review", {}, timeout=900)
|
||||
review_state = ctx.oracle._json(f"state/skills/{SK1_SKILL}/review.json")
|
||||
findings = [f for f in (review_state.get("findings") or []) if isinstance(f, dict)]
|
||||
ctx.check("review_all_pass",
|
||||
review["status"] == 200 and review["body"].get("status") == "clean" and bool(findings)
|
||||
and all(str(f.get("verdict") or "") == "PASS" for f in findings),
|
||||
review_status=review["body"].get("status"), findings=len(findings),
|
||||
findings_failed=[f.get("item") for f in findings if str(f.get("verdict") or "") != "PASS"])
|
||||
ok, review_facts = sk1_review_gate(review, _skill_entry(ctx.server.base_url, SK1_SKILL), findings)
|
||||
ctx.check("review_executable", ok, **review_facts)
|
||||
grants = _api_status(ctx.server.base_url, "POST", f"/api/skills/{SK1_SKILL}/grants", {"items": SK1_GRANTS}, timeout=120)
|
||||
granted = ctx.oracle._json(f"state/skills/{SK1_SKILL}/grants.json").get("granted_permissions")
|
||||
ctx.check("grants_exactly_requested", (grants["body"].get("grants") or {}).get("all_granted") is True
|
||||
|
|
@ -869,6 +889,11 @@ class Scenario:
|
|||
out = dict(self.settings_overrides)
|
||||
if self.id == "SW1":
|
||||
out["OUROBOROS_SUBAGENTS"] = sw1_roster(model)
|
||||
if not self.expects_absorb:
|
||||
# A lane that commits nothing must not promote either: under --self-mod its one-shot cycle could
|
||||
# commit and re-exec the server in the middle of the lifecycle under test (SK1 review/grants/
|
||||
# dispatch), turning an unrelated restart into the lane's verdict. Only SM1 exercises evolution.
|
||||
out["OUROBOROS_POST_TASK_EVOLUTION"] = "false"
|
||||
return out
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1274,7 +1274,11 @@ computed style read by a browser from the COMMITTED CSS after a restart), SW1
|
|||
(the Swarm button arms `force_plan`; at least two children with causal lineage,
|
||||
the `swarm_fanout` receipt, the with-children cost rollup, no orphan process by
|
||||
the `/proc` environ scan), SK1 (the model authors `SKILL.md`+`plugin.py` and runs
|
||||
`skill_preflight`; the runner reviews, grants exactly the manifest's one
|
||||
`skill_preflight`; the runner reviews — the verdict is the product gate: the review call
|
||||
answers 200 with its own `executable_review`, the `/api/extensions` row says
|
||||
`executable_review` (clean, warnings, or blockers under advisory enforcement — the SK1 lane
|
||||
sets none, so it runs the tree default) and findings were persisted; the status, enforcement,
|
||||
blocking reason, non-PASS items and the clean state are recorded facts — grants exactly the manifest's one
|
||||
privileged permission, enables, dispatches, deletes; the author and dispatch
|
||||
tasks keep separate `author_*`/`dispatch_*` terminal checks, and the dispatch
|
||||
counts only on a tools.jsonl row with typed status `ok` and the extension's exact
|
||||
|
|
@ -1294,9 +1298,9 @@ only in each lane's 0600 settings file, disclosed by fingerprint as the runtime
|
|||
grant); the preflight takes `min(key limit remaining, account credits)` and
|
||||
refuses below `--min-credit-usd`. `--total-budget` (default 100) is the RUN-WIDE
|
||||
cap: a ledger sums the lanes' durable `llm_usage` costs, reserves
|
||||
`max(0.01, --per-task-usd × (root tasks + 1 with --self-mod))` per attempt (SM1 and SW1 one root — scouts spend
|
||||
under their root's `OUROBOROS_PER_TASK_COST_USD` fence — SK1 two; with `--self-mod` the
|
||||
one post-task evolution cycle is one more root (the rc.14 paid run showed a second, generic
|
||||
`max(0.01, --per-task-usd × (root tasks + 1 with --self-mod for the scenario that absorbs))` per attempt (SM1 and SW1 one root — scouts spend
|
||||
under their root's `OUROBOROS_PER_TASK_COST_USD` fence — SK1 two; with `--self-mod` SM1's
|
||||
one post-task evolution cycle is one more root (SW1/SK1 pin promotion off) (the rc.14 paid run showed a second, generic
|
||||
cycle at t=0 from the benchmark campaign the lane pre-seeded then), all under the lane fence — SM1_a1
|
||||
task $3.84 + cycles $12.40 + $2.84 of $20; the lane's TOTAL_BUDGET is the true fence and the
|
||||
+1 root is its reservation), admits an
|
||||
|
|
@ -1332,17 +1336,16 @@ can never fit is refused at once and leaves the line. Ordering the pool alone di
|
|||
admission — the dominant race is the woken waiter against the freed lane's NEXT job (settle in
|
||||
`run_attempt`'s finally → return → the executor takes the next job → `admit` on the same thread),
|
||||
which the newcomer won 300/300 on CPython, so a round that overflows the cap (the owner
|
||||
configuration: 150 + 100 + 100 = 350 > 300) let later attempts leapfrog the waiter and could
|
||||
configuration under the earlier +1-for-every-lane rule: 150 + 100 + 100 = 350 > 300) let later attempts leapfrog the waiter and could
|
||||
leave SW1 at 0/3 under pessimistic spends. FIFO admission removes that race at the cost of
|
||||
possible head-of-line idle lanes (a waiting large reservation holds back a smaller attempt that
|
||||
would fit). The verdict is pass-of PER scenario, so the order protects the MINIMUM admitted
|
||||
attempts per scenario rather than the sum: at cap 300 / per-task 50 / `--self-mod` / 3 lanes
|
||||
with realistic spends (SM1 30, SW1 8, SK1 15) all nine attempts are admitted in dispatch order
|
||||
for $159; with pessimistic spends (SM1 45, SW1 8, SK1 30) the admitted order is SK1_a1, SM1_a1,
|
||||
SW1_a1, SK1_a2, SM1_a2, SW1_a2, SM1_a3 — SK1_a3 refused when SM1_a2 settles ($158 + 150 > 300),
|
||||
SW1_a3 when SM1_a3 settles ($211 + 100 > 300) — 7/9 at $211 with every scenario keeping two,
|
||||
whereas largest-first dispatch (the rejected order) under the same admission runs SK1 ×3 and
|
||||
SM1 ×3 first and refuses every SW1 attempt at $225 (SW1 0/3). For a given spend model the
|
||||
(SM1_a1, SK1_a1, SW1_a1, …: SM1 and SK1 reserve 100, SW1 50) for $159; with pessimistic spends
|
||||
(SM1 45, SW1 8, SK1 30) eight are admitted and SK1_a3 is refused ($219 + 100 > 300) — 8/9 at
|
||||
$219 with every scenario keeping two, whereas largest-first dispatch (the rejected order, traced
|
||||
under the earlier rule) refused every SW1 attempt (SW1 0/3). For a given spend model the
|
||||
admitted sequence is exact (pinned in `tests/test_e2e_live_runner.py`); on a live run only the
|
||||
lanes' actual spend and its timing move it, and the first `--lanes` attempts, which enter
|
||||
admission within microseconds of each other, line up in the thread scheduler's order (in
|
||||
|
|
@ -1375,10 +1378,18 @@ t=0, gets the promotion refused (evolution already enabled) and its kept request
|
|||
`expects_absorb` (SM1, the one that lands a commit): a pre-task snapshot
|
||||
(clone HEAD, served sha, uptime, absorbed-cycle counter) and, afterwards, the
|
||||
counter advanced, the served sha moved, the uptime reset and the server ready;
|
||||
anything less is a typed `self_mod_absorb_confirmed=false` and the run fails.
|
||||
anything less is a typed `self_mod_absorb_confirmed=false` and the run fails. The
|
||||
wait (`IsolatedServer.wait_for_absorb`) ends early only with proof that no cycle is
|
||||
pending — six consecutive polls with the queue idle, `supervisor_ready`, no promotion
|
||||
request and no campaign `active_transaction` (a committed cycle keeps its transaction
|
||||
as `waiting_for_restart` through the synchronous restart and the re-exec boot, when a
|
||||
single idle sample looks exactly like a declined promotion) — and types the reason from
|
||||
the durable campaign state (`no_promotion`, `no_decision`, `cycle_no_op`,
|
||||
`cycle_not_absorbed`, `campaign_<status>`, `cycle_not_enqueued`).
|
||||
SW1/SK1 commit nothing, so under `--self-mod` they stop their server right after
|
||||
the scenario with `self_mod_absorb: {"expected": false}` and no absorb check
|
||||
(evolution stays on in their settings; the stand just does not wait for it) —
|
||||
the scenario with `self_mod_absorb: {"expected": false}` and no absorb check, and their
|
||||
lane settings pin `OUROBOROS_POST_TASK_EVOLUTION=false` (a one-shot cycle promoted from
|
||||
their own roots could re-exec the server inside the lifecycle under test) —
|
||||
the rc.15 paid stand (2026-09-05) had SK1_a1 pass twelve of its thirteen lifecycle
|
||||
checks (the thirteenth was a real reviewer finding on the model-authored plugin),
|
||||
then wait about 27 minutes for a promotion nothing had committed before the
|
||||
|
|
|
|||
|
|
@ -182,7 +182,8 @@ def test_the_run_size_is_feasible_under_the_cap_by_the_worst_case_reservation_ru
|
|||
# The stand's own rule (per-task x (roots + the self-mod evolution root)), asked from
|
||||
# the real ledger with the job's --self-mod, so a rule change re-derives this pin itself.
|
||||
budget = RunBudget(TOTAL_BUDGET_USD, per_task, self_mod=args["--self-mod"] is None)
|
||||
reservations = [budget.reservation(SCENARIOS[sid].root_tasks) for sid in scenarios for _ in range(attempts)]
|
||||
reservations = [budget.reservation(SCENARIOS[sid].root_tasks, SCENARIOS[sid].expects_absorb)
|
||||
for sid in scenarios for _ in range(attempts)]
|
||||
assert sum(reservations) <= TOTAL_BUDGET_USD, (reservations, per_task)
|
||||
# ...and the set is MAXIMAL at this fence: one more single-root attempt would
|
||||
# not fit. When the reservation factor changes this trips on purpose — the
|
||||
|
|
@ -200,13 +201,14 @@ def test_the_summary_header_and_the_job_comment_state_the_current_reservation_ar
|
|||
OWN reservations are rendered from the manifest's budget_preflight."""
|
||||
args = _stand_args()
|
||||
budget = RunBudget(TOTAL_BUDGET_USD, float(args["--per-task-usd"]), self_mod=args["--self-mod"] is None)
|
||||
chosen = budget.reservation(SCENARIOS["SM1"].root_tasks)
|
||||
full_set = sum(budget.reservation(SCENARIOS[sid].root_tasks) for sid in ("SM1", "SW1", "SK1")) * 3
|
||||
chosen = budget.reservation(SCENARIOS["SM1"].root_tasks, SCENARIOS["SM1"].expects_absorb)
|
||||
full_set = sum(budget.reservation(SCENARIOS[sid].root_tasks, SCENARIOS[sid].expects_absorb)
|
||||
for sid in ("SM1", "SW1", "SK1")) * 3
|
||||
run = _summary_step()["run"]
|
||||
assert f"SM1 x1 = ${chosen:.0f}" in run and f"x3 set needs ${full_set:.0f}" in run, run
|
||||
assert "budget_preflight" in run, run
|
||||
ci = CI_PATH.read_text(encoding="utf-8")
|
||||
for retired in ("HARD_STOP_INVERSE", "$360", "2 x per"):
|
||||
for retired in ("HARD_STOP_INVERSE", "$360", "2 x per", "$315", "(2 + 2 + 3)"):
|
||||
assert retired not in ci, retired
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -391,17 +391,17 @@ def test_reservation_counts_roots_plus_the_evolution_root_and_is_the_lane_total_
|
|||
"""EQUALITY pins of the rc.14/rc.15 finding: the reservation is per-task x root tasks, +1 with --self-mod (the one
|
||||
post-task cycle; rc.14: SM1_a1 task $3.84 + cycles $12.40 + $2.84 of $20 — the lane's TOTAL_BUDGET is the fence); the 2x
|
||||
factor and its product import are gone and no bench budget profile is projected. Per-task $20 and one root reserve
|
||||
$20 ($40 with --self-mod, $60 for SK1 + evolution) and that exact number reaches the lane's settings file as
|
||||
$20 ($40 for the absorbing SM1 root with --self-mod; SK1's two roots stay $40, it does not promote) and that exact number reaches the lane's settings file as
|
||||
TOTAL_BUDGET through ``run_lane`` (never the run-wide cap)."""
|
||||
_short_tmp(monkeypatch)
|
||||
rule = run_live_lanes.RESERVATION_RULE
|
||||
assert not hasattr(run_live_lanes, "HARD_STOP_INVERSE") and rule == run_live_lanes.RunBudget(1, 1).snapshot()["reservation_rule"]
|
||||
assert rule.startswith("max(0.01, per_task_usd x (root_tasks + 1 if --self-mod else root_tasks))")
|
||||
assert "one post-task evolution cycle" in rule and "the true fence" in rule and "cost_hard_stop" not in rule
|
||||
assert rule.startswith("max(0.01, per_task_usd x (root_tasks + 1 if --self-mod and the scenario absorbs else root_tasks))")
|
||||
assert "post-task cycle of a lane that promotes" in rule and "the true fence" in rule and "cost_hard_stop" not in rule
|
||||
budget = run_live_lanes.RunBudget(100.0, 20.0, reader=lambda root: (0.0, 0))
|
||||
assert budget.reservation(1) == 20.0 and budget.reservation(2) == 40.0 and not budget.self_mod
|
||||
evolving = run_live_lanes.RunBudget(100.0, 20.0, reader=lambda root: (0.0, 0), self_mod=True)
|
||||
assert evolving.reservation(1) == 40.0 and evolving.reservation(2) == 60.0 and evolving.self_mod
|
||||
assert evolving.reservation(1, absorbs=True) == 40.0 and evolving.reservation(2) == 40.0 and evolving.reservation(1) == 20.0
|
||||
seed = _git_seed(tmp_path)
|
||||
out, job = tmp_path / "out", ("SM1", 1)
|
||||
ok, facts = budget.admit(job, 1, out / "lanes" / "SM1_a1" / "data", dispatch_index=0)
|
||||
|
|
@ -445,31 +445,31 @@ def test_budget_preflight_refuses_reservations_that_can_never_all_be_admitted(tm
|
|||
refused every SK1 attempt by construction. The preflight refuses BEFORE any spend a reservation above the cap, or
|
||||
equal to it with attempts >= 2 (the second can never be admitted after any spend), in the credit preflight's typed
|
||||
shape, before the key, the seed or a lane; no override. The per-ROUND worst case is the --lanes largest reservations,
|
||||
ONE attempt per scenario: the owner's cap 300 / per-task 50 / --self-mod / 3 lanes = $350 > cap, so the third lane WAITS."""
|
||||
ONE attempt per scenario: the owner's cap 300 / per-task 50 / --self-mod / 3 lanes = $250 (SM1 100, SK1 100, SW1 50)."""
|
||||
def rows(budget, attempts, ids=("SM1", "SW1", "SK1"), lanes=3):
|
||||
pre = run_live_lanes.budget_preflight(budget, list(ids), attempts, lanes)
|
||||
return ({r["scenario"]: r["reservation_usd"] for r in pre["scenarios"]}, pre["worst_case_usd"], pre["unreachable"],
|
||||
pre["round_worst_case_usd"])
|
||||
|
||||
reader = lambda root: (0.0, 0) # noqa: E731 - a stub reader
|
||||
assert rows(run_live_lanes.RunBudget(300.0, 50.0, reader, self_mod=True), 3) == ({"SM1": 100.0, "SW1": 100.0, "SK1": 150.0}, 1050.0, [], 350.0)
|
||||
assert rows(run_live_lanes.RunBudget(300.0, 50.0, reader, self_mod=True), 3, lanes=2)[3] == 250.0
|
||||
assert rows(run_live_lanes.RunBudget(100.0, 50.0, reader, self_mod=True), 1) == ({"SM1": 100.0, "SW1": 100.0, "SK1": 150.0}, 350.0, ["SK1"], 350.0)
|
||||
at_cap = run_live_lanes.RunBudget(150.0, 50.0, reader, self_mod=True) # SK1 == cap: one attempt fits at $0, never a second
|
||||
assert rows(at_cap, 1)[2] == [] and rows(at_cap, 2)[2] == ["SK1"]
|
||||
assert rows(run_live_lanes.RunBudget(300.0, 50.0, reader, self_mod=True), 3) == ({"SM1": 100.0, "SW1": 50.0, "SK1": 100.0}, 750.0, [], 250.0)
|
||||
assert rows(run_live_lanes.RunBudget(300.0, 50.0, reader, self_mod=True), 3, lanes=2)[3] == 200.0
|
||||
assert rows(run_live_lanes.RunBudget(90.0, 50.0, reader, self_mod=True), 1) == ({"SM1": 100.0, "SW1": 50.0, "SK1": 100.0}, 250.0, ["SM1", "SK1"], 250.0)
|
||||
at_cap = run_live_lanes.RunBudget(100.0, 50.0, reader, self_mod=True) # SM1/SK1 == cap: one attempt fits at $0, never a second
|
||||
assert rows(at_cap, 1)[2] == [] and rows(at_cap, 2)[2] == ["SM1", "SK1"]
|
||||
assert rows(run_live_lanes.RunBudget(200.0, 100.0, reader), 3)[2] == ["SK1"] # the shipped 2x rule's SK1 = 200 of 200
|
||||
pre = run_live_lanes.budget_preflight(run_live_lanes.RunBudget(200.0, 50.0, reader), ["SK1"], 2, 4)
|
||||
assert pre == {"cap_usd": 200.0, "per_task_usd": 50.0, "self_mod": False, "reservation_rule": run_live_lanes.RESERVATION_RULE,
|
||||
"scenarios": [{"scenario": "SK1", "root_tasks": 2, "reservation_usd": 100.0, "attempts": 2,
|
||||
"worst_case_usd": 200.0, "unreachable": False}], "worst_case_usd": 200.0, "lanes": 4,
|
||||
"round_worst_case_usd": 100.0, "unreachable": []}
|
||||
out, manifest = _fake_run(tmp_path, monkeypatch, ["--total-budget", "100", "--per-task-usd", "50", "--self-mod",
|
||||
out, manifest = _fake_run(tmp_path, monkeypatch, ["--total-budget", "90", "--per-task-usd", "50", "--self-mod",
|
||||
"--scenarios", "SM1,SW1,SK1"],
|
||||
lane=lambda *a, **k: pytest.fail("a lane started after a budget refusal"), expect_rc=3)
|
||||
refusal = manifest["extra"]["refusal"]
|
||||
assert refusal["stage"] == "budget_preflight" and refusal["reason"] == "reservation_unreachable"
|
||||
assert refusal["unreachable"] == ["SK1"] and refusal["cap_usd"] == 100.0 and refusal["self_mod"] is True
|
||||
assert manifest["extra"]["budget_preflight"]["unreachable"] == ["SK1"] and manifest["extra"]["exit_code"] == 3
|
||||
assert refusal["unreachable"] == ["SM1", "SK1"] and refusal["cap_usd"] == 90.0 and refusal["self_mod"] is True
|
||||
assert manifest["extra"]["budget_preflight"]["unreachable"] == ["SM1", "SK1"] and manifest["extra"]["exit_code"] == 3
|
||||
assert "credential_fingerprint" not in manifest["extra"] and not (out / "seed").exists() and not (out / "lanes").exists()
|
||||
assert manifest["requested_task_ids"] == ["SM1_a1", "SW1_a1", "SK1_a1"] # SM1/SW1 = $100 = cap: one attempt fits
|
||||
|
||||
|
|
@ -524,9 +524,9 @@ class _Driver:
|
|||
lock.wait = park
|
||||
|
||||
def _ask(self, index: int, job) -> None:
|
||||
name, box = f"{job[0]}_a{job[1]}", {}
|
||||
name, box, row = f"{job[0]}_a{job[1]}", {}, scenarios.SCENARIOS[job[0]]
|
||||
thread = threading.Thread(name=name, daemon=True, target=lambda: box.__setitem__("r", self.budget.admit(
|
||||
job, scenarios.SCENARIOS[job[0]].root_tasks, pathlib.Path("/x") / name, dispatch_index=index)))
|
||||
job, row.root_tasks, pathlib.Path("/x") / name, dispatch_index=index, absorbs=row.expects_absorb)))
|
||||
self.threads[name] = (index, job, thread, box)
|
||||
thread.start()
|
||||
self._settle_thread(name)
|
||||
|
|
@ -564,19 +564,19 @@ class _Driver:
|
|||
REALISTIC_SPEND = {"SM1": 30.0, "SW1": 8.0, "SK1": 15.0} # assumed per-attempt spends: rc.14 SM1 lanes, rc.11 SW1/SK1
|
||||
PESSIMISTIC_SPEND = {"SM1": 45.0, "SW1": 8.0, "SK1": 30.0}
|
||||
OWNER_CONFIGURATION = dict(cap=300.0, per_task=50.0, scenario_ids=["SM1", "SW1", "SK1"], attempts=3, lanes=3, self_mod=True)
|
||||
DISPATCH_ORDER = ["SK1_a1", "SM1_a1", "SW1_a1", "SK1_a2", "SM1_a2", "SW1_a2", "SK1_a3", "SM1_a3", "SW1_a3"]
|
||||
DISPATCH_ORDER = ["SM1_a1", "SK1_a1", "SW1_a1", "SM1_a2", "SK1_a2", "SW1_a2", "SM1_a3", "SK1_a3", "SW1_a3"]
|
||||
|
||||
|
||||
def test_owner_configuration_cap_300_per_task_50_three_attempts_self_mod_is_exact_under_fifo_admission():
|
||||
"""The live configuration (cap 300, per-task 50, attempts 3, pass-of 2, 3 lanes, --self-mod: SM1/SW1 reserve 100,
|
||||
SK1 150; round 1 = 350 > cap, so SW1_a1 waits from t=0) — EXACT sequences, no wake-order range. Realistic spends:
|
||||
all nine admitted in dispatch order, $159. Pessimistic: SK1_a3 refused when SM1_a2 settles ($158 + 150 > 300),
|
||||
SW1_a3 when SM1_a3 settles ($211 + 100 > 300): 7/9 at $211, every scenario keeping two = pass-of. Largest-first
|
||||
dispatch under the same admission refuses all of SW1 ($225, 0/3): the handbook's traced reason, prose, not a pin."""
|
||||
"""The live configuration (cap 300, per-task 50, attempts 3, pass-of 2, 3 lanes, --self-mod: SM1 reserves 100 — its
|
||||
root plus the post-task cycle only it promotes — SK1 100 for two roots, SW1 50; round 1 = 250 fits) — EXACT
|
||||
sequences, no wake-order range. Realistic spends: all nine admitted in dispatch order, $159. Pessimistic: SK1_a3
|
||||
refused ($219 + 100 > 300): 8/9 at $219, every scenario keeping two = pass-of. Largest-first dispatch under the
|
||||
earlier +1-for-every-lane rule refused all of SW1 ($225, 0/3): the handbook's traced reason, prose, not a pin."""
|
||||
assert _Driver(spends=REALISTIC_SPEND, **OWNER_CONFIGURATION).run() == (DISPATCH_ORDER, [], 159.0)
|
||||
admitted, refused, spent = _Driver(spends=PESSIMISTIC_SPEND, **OWNER_CONFIGURATION).run()
|
||||
assert (admitted, refused, spent) == (DISPATCH_ORDER[:6] + ["SM1_a3"], ["SK1_a3", "SW1_a3"], 211.0)
|
||||
assert {s: sum(n.startswith(s) for n in admitted) for s in ("SM1", "SW1", "SK1")} == {"SM1": 3, "SW1": 2, "SK1": 2}
|
||||
assert (admitted, refused, spent) == (DISPATCH_ORDER[:7] + ["SW1_a3"], ["SK1_a3"], 219.0)
|
||||
assert {s: sum(n.startswith(s) for n in admitted) for s in ("SM1", "SW1", "SK1")} == {"SM1": 3, "SW1": 3, "SK1": 2}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -874,7 +874,7 @@ def test_absorb_wait_and_check_follow_the_scenarios_expects_absorb(tmp_path, mon
|
|||
"""The rc.15 paid stand (2026-09-05, SK1_a1): every ``--self-mod`` lane waited ``--task-timeout`` for an absorb
|
||||
only SM1's commit could trigger, then failed ``self_mod_absorb_confirmed`` by construction. Now SM1 waits and
|
||||
carries the check; SW1/SK1 stop right after the scenario with ``{"expected": False}``, no check, post-task
|
||||
evolution ON in their settings; every lane seeds ``owner_chat_id`` ONLY, never a campaign (run2's t=0 cycles)."""
|
||||
evolution OFF in their settings; every lane seeds ``owner_chat_id`` ONLY, never a campaign (run2's t=0 cycles)."""
|
||||
waits: list = []
|
||||
monkeypatch.setattr(run_live_lanes, "resolve_ui_client", lambda base_url: (None, "ui_unavailable:test"))
|
||||
monkeypatch.setattr(run_live_lanes, "self_mod_snapshot", lambda server, clone, data_root: {"pre": True})
|
||||
|
|
@ -890,7 +890,7 @@ def test_absorb_wait_and_check_follow_the_scenarios_expects_absorb(tmp_path, mon
|
|||
assert row["self_mod_absorb"] == {"expected": False} and row["self_mod"] is True and waits == [{"pre": True}]
|
||||
lane = tmp_path / sid / "out" / "lanes" / f"{sid}_a1" / "data"
|
||||
state = json.loads((lane / "state" / "state.json").read_text(encoding="utf-8"))
|
||||
assert json.loads((lane / "settings.json").read_text())["OUROBOROS_POST_TASK_EVOLUTION"] == "true"
|
||||
assert json.loads((lane / "settings.json").read_text())["OUROBOROS_POST_TASK_EVOLUTION"] == ("true" if sid == "SM1" else "false")
|
||||
assert state["owner_chat_id"] == 1 and "evolution_mode_enabled" not in state, state
|
||||
assert not (lane / "state" / "evolution_campaign.json").exists(), sid
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
"""The live stand's SK1 probe plugin presents the host token only to the loopback Host Service
|
||||
(docs/CHECKLISTS.md skill item 12, host_token_handling): a base URL naming any other host or
|
||||
scheme is refused before a request is built. Pinned after the rc.15 paid stand, where the skill
|
||||
review blocked the stand's own plugin for reading HOST_SERVICE_URL unvalidated."""
|
||||
review blocked the stand's own plugin for reading HOST_SERVICE_URL unvalidated. The SK1 review criterion
|
||||
is the product gate (owner decision 2026-09-06): executable review, with the clean/all-PASS state a fact."""
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
|
|
@ -47,3 +48,51 @@ def test_the_probe_accepts_the_loopback_base_and_only_then_reaches_the_transport
|
|||
with pytest.raises(Exception) as excinfo:
|
||||
_echo_tool()(None, "x")
|
||||
assert not isinstance(excinfo.value, RuntimeError) or "loopback" not in str(excinfo.value)
|
||||
|
||||
|
||||
_FINDINGS = [{"item": "manifest_schema", "verdict": "PASS"}, {"item": "bug_hunting", "verdict": "FAIL"}]
|
||||
_CLEAN = [{"item": "a", "verdict": "PASS"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,http,body_exec,executable,findings,expected", [
|
||||
("clean", 200, True, True, _CLEAN, True),
|
||||
("warnings", 200, True, True, _FINDINGS, True), # rc.15 SK1_a2: warnings are executable
|
||||
("blockers", 200, True, True, _FINDINGS, True), # rc.15 SK1_a3: blockers executable under advisory
|
||||
("blockers", 200, False, False, _FINDINGS, False), # the same review under blocking enforcement
|
||||
("pending", 200, False, True, _CLEAN, False), # duplicate job: the call's own gate says no
|
||||
("clean", 500, True, True, _CLEAN, False), # the review call itself failed
|
||||
("clean", 200, True, True, [], False), # no recorded findings: no review actually ran
|
||||
("clean", 200, True, None, _CLEAN, False), # the /api/extensions row carries no gate fact
|
||||
])
|
||||
def test_sk1_review_verdict_is_the_product_gate_and_records_the_clean_state_as_a_fact(status, http, body_exec,
|
||||
executable, findings, expected):
|
||||
review = {"status": http, "body": {"status": status, "executable_review": body_exec}}
|
||||
entry = {"executable_review": executable,
|
||||
"review_gate": {"review_enforcement": "advisory", "blocking_reason": "x"}}
|
||||
ok, facts = scenarios.sk1_review_gate(review, entry, findings)
|
||||
assert ok is expected
|
||||
failed = [f["item"] for f in findings if f["verdict"] != "PASS"]
|
||||
assert facts == {"review_status": status, "review_executable": executable, "review_body_executable": body_exec,
|
||||
"review_enforcement": "advisory", "review_blocking_reason": "x", "findings": len(findings),
|
||||
"findings_failed": failed, "review_clean": bool(findings) and not failed}
|
||||
|
||||
|
||||
def test_the_product_gate_the_stand_relies_on_executes_warnings_and_advisory_blockers_only():
|
||||
"""The contract behind ``sk1_review_gate``: ``skill_review_gate`` (the same rule the ``/api/extensions``
|
||||
row and the review call project as ``executable_review``)."""
|
||||
from ouroboros.skill_review_status import skill_review_gate
|
||||
|
||||
assert skill_review_gate("clean", enforcement="blocking")["executable_review"] is True
|
||||
assert skill_review_gate("warnings", enforcement="blocking")["executable_review"] is True
|
||||
assert skill_review_gate("blockers", enforcement="advisory")["executable_review"] is True
|
||||
assert skill_review_gate("blockers", enforcement="blocking")["executable_review"] is False
|
||||
assert skill_review_gate("pending", enforcement="advisory")["executable_review"] is False
|
||||
|
||||
|
||||
def test_only_the_absorbing_scenario_promotes_post_task_evolution():
|
||||
"""Under ``--self-mod`` SW1/SK1 would otherwise promote from their own roots and their one-shot cycle could
|
||||
re-exec the server inside the lifecycle under test; the lane overrides pin post-task evolution off for
|
||||
every scenario that commits nothing, and leave SM1 (``expects_absorb``) to the run-level setting."""
|
||||
for sid, row in scenarios.SCENARIOS.items():
|
||||
applied = row.overrides("stub/model").get("OUROBOROS_POST_TASK_EVOLUTION")
|
||||
assert applied == (None if row.expects_absorb else "false"), (sid, applied)
|
||||
|
|
|
|||
128
tests/test_server_runner_absorb_wait.py
Normal file
128
tests/test_server_runner_absorb_wait.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""``IsolatedServer.wait_for_absorb`` may answer ``absorbed=False`` early only with PROOF that no cycle is
|
||||
pending (rc.15 stand, adversarial finding of 2026-09-06): a cycle that committed keeps its campaign transaction
|
||||
as ``waiting_for_restart`` through the synchronous supervisor restart (queue idle, counter unchanged) and the
|
||||
re-exec'd server answers ``/api/state`` with zero counts before its supervisor is ready; one idle sample used to
|
||||
end the wait as ``no_promotion`` right there. The typed idle reasons come from the durable campaign state."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
from devtools.benchmarks.common import server_runner
|
||||
|
||||
|
||||
class _Clock:
|
||||
def __init__(self) -> None:
|
||||
self.now = 0.0
|
||||
|
||||
def time(self) -> float:
|
||||
return self.now
|
||||
|
||||
def sleep(self, seconds: float) -> None:
|
||||
self.now += float(seconds)
|
||||
|
||||
|
||||
def _server(tmp_path: pathlib.Path, states: list, monkeypatch, *, sha: str = "aaaaaaaa") -> server_runner.IsolatedServer:
|
||||
"""``states`` feeds the idle poll only (one read per poll after the grace); the served sha is fixed."""
|
||||
clock = _Clock()
|
||||
monkeypatch.setattr(server_runner, "time", clock)
|
||||
srv = object.__new__(server_runner.IsolatedServer)
|
||||
srv.data_root = tmp_path
|
||||
srv.base_url = "http://127.0.0.1:1"
|
||||
calls = {"n": 0}
|
||||
|
||||
def _state(timeout: float = 5) -> dict:
|
||||
st = states[min(calls["n"], len(states) - 1)]
|
||||
calls["n"] += 1
|
||||
if isinstance(st, Exception):
|
||||
raise st
|
||||
return st
|
||||
|
||||
srv._state = _state
|
||||
srv.current_sha = lambda: sha
|
||||
srv.wait_for_health = lambda timeout=180: True
|
||||
return srv
|
||||
|
||||
|
||||
def _campaign(root: pathlib.Path, **fields) -> None:
|
||||
(root / "state").mkdir(parents=True, exist_ok=True)
|
||||
(root / "state" / "evolution_campaign.json").write_text(json.dumps({"schema_version": 1, **fields}), encoding="utf-8")
|
||||
|
||||
|
||||
IDLE = {"sha": "aaaaaaaa", "pending_count": 0, "running_count": 0, "supervisor_ready": True}
|
||||
BOOTING = {"sha": "", "pending_count": 0, "running_count": 0, "supervisor_ready": False}
|
||||
BUSY = {"sha": "aaaaaaaa", "pending_count": 0, "running_count": 1, "supervisor_ready": True}
|
||||
|
||||
|
||||
def test_an_idle_queue_with_a_transaction_waiting_for_restart_is_not_a_declined_promotion(tmp_path, monkeypatch):
|
||||
_campaign(tmp_path, status="active", source="post_task", absorbed_cycles_done=0,
|
||||
active_transaction={"commit_sha": "c" * 40, "cycle_outcome": "waiting_for_restart"})
|
||||
srv = _server(tmp_path, [IDLE], monkeypatch)
|
||||
out = srv.wait_for_absorb("aaaaaaaa", 0, timeout=200, idle_grace=10, idle_polls=2)
|
||||
assert out["reason"] == "timeout" and out["absorbed"] is False and out["campaign"]["active_transaction"] is True
|
||||
|
||||
|
||||
def test_the_booting_reexecd_server_and_a_single_idle_sample_do_not_end_the_wait(tmp_path, monkeypatch):
|
||||
_campaign(tmp_path, status="active", source="post_task", absorbed_cycles_done=0, transaction_history=[])
|
||||
# after the grace: booting, idle, busy, idle, idle -> the streak of two idle polls only forms on the fifth poll
|
||||
srv = _server(tmp_path, [BOOTING, IDLE, BUSY, IDLE, IDLE], monkeypatch)
|
||||
out = srv.wait_for_absorb("aaaaaaaa", 0, timeout=10_000, idle_grace=10, idle_polls=2)
|
||||
assert out["reason"] == "cycle_not_enqueued" and out["absorbed"] is False
|
||||
assert server_runner.time.now == 35 # polls at t=15,20,25,30,35: the streak completes on the fifth
|
||||
|
||||
|
||||
def test_a_pending_request_file_blocks_the_early_exit(tmp_path, monkeypatch):
|
||||
(tmp_path / "state").mkdir()
|
||||
(tmp_path / "state" / "post_task_evolution_request.json").write_text("{}", encoding="utf-8")
|
||||
srv = _server(tmp_path, [IDLE], monkeypatch)
|
||||
assert srv.wait_for_absorb("aaaaaaaa", 0, timeout=100, idle_grace=10, idle_polls=1)["reason"] == "timeout"
|
||||
|
||||
|
||||
def test_the_absorb_is_confirmed_when_the_counter_and_the_served_sha_move(tmp_path, monkeypatch):
|
||||
_campaign(tmp_path, status="active", source="post_task", absorbed_cycles_done=1,
|
||||
transaction_history=[{"cycle_outcome": "absorbed"}])
|
||||
srv = _server(tmp_path, [IDLE], monkeypatch, sha="bbbbbbbb")
|
||||
out = srv.wait_for_absorb("aaaaaaaa", 0, timeout=100, idle_grace=10)
|
||||
assert out["absorbed"] is True and out["new_sha"] == "bbbbbbbb" and out["campaign"]["newest_outcome"] == "absorbed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("campaign,counter,expected", [
|
||||
(None, 1, "no_promotion"), # an every_n tick ran, nothing promoted
|
||||
(None, 0, "no_decision"), # no tick recorded (llm cadence / ineligible)
|
||||
({"status": "active", "transaction_history": [{"cycle_outcome": "no_op"}]}, 1, "cycle_no_op"),
|
||||
({"status": "active", "transaction_history": [{"cycle_outcome": "abandoned"}]}, 1, "cycle_not_absorbed"),
|
||||
({"status": "paused", "transaction_history": [{"cycle_outcome": "no_op"}]}, 1, "campaign_paused"),
|
||||
({"status": "active"}, 1, "cycle_not_enqueued"),
|
||||
])
|
||||
def test_idle_reasons_are_typed_from_the_durable_campaign_state(tmp_path, monkeypatch, campaign, counter, expected):
|
||||
"""The cycle written DURING the wait decides ``cycle_*``: the campaign file is created empty before the wait
|
||||
and the cycle's transaction lands after it started (a fresh one-shot campaign); a paused status wins."""
|
||||
(tmp_path / "state").mkdir(exist_ok=True)
|
||||
if counter:
|
||||
(tmp_path / "state" / "post_task_evolution_counter.json").write_text(json.dumps({"n": counter}), encoding="utf-8")
|
||||
srv = _server(tmp_path, [IDLE], monkeypatch)
|
||||
if campaign is not None:
|
||||
# the campaign appears after the wait's first summary: the first poll writes it, so its history is "new"
|
||||
original = srv._state
|
||||
|
||||
def _state_then_campaign(timeout: float = 5) -> dict:
|
||||
_campaign(tmp_path, **campaign)
|
||||
return original(timeout)
|
||||
|
||||
srv._state = _state_then_campaign
|
||||
out = srv.wait_for_absorb("aaaaaaaa", 0, timeout=1_000, idle_grace=10, idle_polls=3)
|
||||
assert out["reason"] == expected and out["absorbed"] is False
|
||||
assert out["campaign"]["post_task_counter"] == counter and out["campaign"]["present"] is (campaign is not None)
|
||||
|
||||
|
||||
def test_older_cycles_of_a_resumed_campaign_never_speak_for_this_boundary(tmp_path, monkeypatch):
|
||||
"""A campaign resumed across instances (CLB stateful, evolve_smoke --tasks N) carries the previous cycle's
|
||||
``no_op``/``absorbed`` outcome in its history: a boundary that attaches no new cycle is ``cycle_not_enqueued``,
|
||||
not the older cycle's outcome."""
|
||||
_campaign(tmp_path, status="active", source="benchmark", absorbed_cycles_done=1,
|
||||
transaction_history=[{"cycle_outcome": "absorbed"}, {"cycle_outcome": "no_op"}])
|
||||
srv = _server(tmp_path, [IDLE], monkeypatch)
|
||||
out = srv.wait_for_absorb("aaaaaaaa", 1, timeout=1_000, idle_grace=10, idle_polls=2)
|
||||
assert out["reason"] == "cycle_not_enqueued" and out["campaign"]["history_len"] == 2
|
||||
Loading…
Add table
Add a link
Reference in a new issue